public async Task ShouldCreateFingerprintsFromAudioSamplesQueryAndGetTheRightResult()
{
const int secondsToProcess = 10;
const int startAtSecond = 30;
var audioSamples = GetAudioSamples();
var track = new TrackInfo("1234", audioSamples.Origin, audioSamples.Origin);
var fingerprints = await FingerprintCommandBuilder.Instance
.BuildFingerprintCommand()
.From(audioSamples)
.UsingServices(audioService)
.Hash();
var modelService = new InMemoryModelService();
modelService.Insert(track, fingerprints);
var querySamples = GetQuerySamples(GetAudioSamples(), startAtSecond, secondsToProcess);
var queryResult = await QueryCommandBuilder.Instance
.BuildQueryCommand()
.From(new AudioSamples(querySamples, string.Empty, audioSamples.SampleRate))
.UsingServices(modelService, audioService)
.Query();
Assert.IsTrue(queryResult.ContainsMatches);
Assert.AreEqual(1, queryResult.ResultEntries.Count());
var bestMatch = queryResult.BestMatch;
Assert.AreEqual("1234", bestMatch.Track.Id);
Assert.IsTrue(bestMatch.CoverageWithPermittedGapsLength > secondsToProcess - 3, $"QueryCoverageSeconds:{bestMatch.QueryLength}");
Assert.AreEqual(startAtSecond, Math.Abs(bestMatch.TrackStartsAt), 0.1d);
Assert.IsTrue(bestMatch.Confidence > 0.5, $"Confidence:{bestMatch.Confidence}");
}
public async Task ShouldIdentifyMultipleRegionsOfTheSameMatch()
{
float[] match = TestUtilities.GenerateRandomFloatArray(10 * 5512, 1);
float[] withJitter = AddJitter(match, 15, 20);
var modelService = new InMemoryModelService();
var audioService = new SoundFingerprintingAudioService();
await InsertFingerprints(withJitter, audioService, modelService);
var result = await GetQueryResult(match, audioService, modelService);
Assert.IsTrue(result.ContainsMatches);
var entries = result.ResultEntries.OrderBy(entry => entry.TrackMatchStartsAt).ToList();
CollectionAssert.IsOrdered(entries[1]
.Coverage
.BestPath
.Select(_ => _.TrackSequenceNumber));
CollectionAssert.IsOrdered(entries[1]
.Coverage
.BestPath
.Select(_ => _.QuerySequenceNumber));
Assert.AreEqual(2, entries.Count);
Assert.AreEqual(10, entries[0].CoverageWithPermittedGapsLength, 1f);
Assert.AreEqual(10, entries[1].CoverageWithPermittedGapsLength, 1f);
Assert.AreEqual(15d, entries[0].TrackMatchStartsAt, 1f);
Assert.AreEqual(45d, entries[1].TrackMatchStartsAt, 1f);
}
public void ShouldSerializeSpectralImages()
{
var spectrumService = new SpectrumService(new LomontFFT(), new LogUtility());
var spectrums = spectrumService.CreateLogSpectrogram(GetAudioSamples(), new DefaultSpectrogramConfig())
.Select(spectrum => spectrum.ImageRowCols)
.ToList();
var modelService = new InMemoryModelService();
var trackReference = new ModelReference <int>(10);
modelService.InsertSpectralImages(spectrums, trackReference);
var tempFile = Path.GetTempFileName();
modelService.Snapshot(tempFile);
var fromFileService = new InMemoryModelService(tempFile);
File.Delete(tempFile);
var allSpectrums = fromFileService.GetSpectralImagesByTrackReference(trackReference).ToList();
Assert.AreEqual(spectrums.Count, allSpectrums.Count);
}
public async Task ShouldIdentifyMultipleTracksInSameQuery()
{
float[] match = TestUtilities.GenerateRandomFloatArray(10 * 5512, 1);
float[] withJitter = AddJitter(match, 15, 20);
var modelService = new InMemoryModelService();
var audioService = new SoundFingerprintingAudioService();
var hashes = await FingerprintCommandBuilder.Instance
.BuildFingerprintCommand()
.From(new AudioSamples(match, "Queen", 5512))
.UsingServices(audioService)
.Hash();
modelService.Insert(new TrackInfo("123", "Bohemian Rhapsody", "Queen"), new Hashes(hashes, match.Length / 5512f, DateTime.Now, Enumerable.Empty <string>()));
var result = await QueryCommandBuilder.Instance
.BuildQueryCommand()
.From(new AudioSamples(withJitter, "cnn", 5512))
.WithQueryConfig(config =>
{
config.AllowMultipleMatchesOfTheSameTrackInQuery = true;
return(config);
})
.UsingServices(modelService, audioService)
.Query();
Assert.IsTrue(result.ContainsMatches);
var entries = result.ResultEntries.OrderBy(entry => entry.QueryMatchStartsAt).ToList();
Assert.AreEqual(2, entries.Count);
Assert.AreEqual(15d, entries[0].QueryMatchStartsAt, 1f);
Assert.AreEqual(45d, entries[1].QueryMatchStartsAt, 1f);
}
public async Task ShouldSerializeAndDeserialize()
{
var modelService = new InMemoryModelService();
var hashedFingerprints = await FingerprintCommandBuilder.Instance.BuildFingerprintCommand()
.From(GetAudioSamples())
.UsingServices(audioService)
.Hash();
var trackData = new TrackInfo("id", "title", "artist", new Dictionary <string, string> {
{ "key", "value" }
}, MediaType.Audio);
modelService.Insert(trackData, hashedFingerprints);
var tempFile = Path.GetTempFileName();
modelService.Snapshot(tempFile);
var queryResult = await QueryCommandBuilder.Instance.BuildQueryCommand()
.From(GetAudioSamples())
.UsingServices(new InMemoryModelService(tempFile), audioService)
.Query();
File.Delete(tempFile);
Assert.IsTrue(queryResult.ContainsMatches);
AssertTracksAreEqual(trackData, queryResult.BestMatch.Track);
Assert.IsTrue(queryResult.BestMatch.Confidence > 0.9);
}
public void ShouldSerializeAndIncrementNextIdCorrectly()
{
var modelService = new InMemoryModelService();
var firstTrack = new TrackInfo("id1", "title", "artist");
modelService.Insert(firstTrack, new Hashes(new[] { new HashedFingerprint(GenericHashBuckets(), 1, 0f) }, 1.48));
var tempFile = Path.GetTempFileName();
modelService.Snapshot(tempFile);
var fromFileService = new InMemoryModelService(tempFile);
var secondTrack = new TrackInfo("id2", "title", "artist");
fromFileService.Insert(secondTrack, new Hashes(new[] { new HashedFingerprint(GenericHashBuckets(), 1, 0f) }, 1.48));
var tracks = fromFileService.ReadAllTracks().ToList();
File.Delete(tempFile);
var ref1 = tracks.First(track => track.Id == "id1").TrackReference;
var ref2 = tracks.First(track => track.Id == "id2").TrackReference;
Assert.IsTrue(tracks.Any(track => track.Id == "id1"));
Assert.IsTrue(tracks.Any(track => track.Id == "id2"));
Assert.IsTrue(!ref1.Equals(ref2));
}
public async Task ShouldCreateFingerprintsFromAudioSamplesQueryWithPreviouslyCreatedFingerprintsAndGetTheRightResult()
{
var audioSamples = GetAudioSamples();
var track = new TrackInfo("4321", audioSamples.Origin, audioSamples.Origin);
var fingerprints = await FingerprintCommandBuilder.Instance
.BuildFingerprintCommand()
.From(audioSamples)
.UsingServices(audioService)
.Hash();
var modelService = new InMemoryModelService();
modelService.Insert(track, fingerprints);
var queryResult = await QueryCommandBuilder.Instance.BuildQueryCommand()
.From(fingerprints)
.UsingServices(modelService, audioService)
.Query();
Assert.IsTrue(queryResult.ContainsMatches);
Assert.AreEqual(1, queryResult.ResultEntries.Count());
var bestMatch = queryResult.BestMatch;
Assert.AreEqual("4321", bestMatch.Track.Id);
Assert.AreEqual(0, Math.Abs(bestMatch.TrackStartsAt), 0.0001d);
Assert.AreEqual(audioSamples.Duration, bestMatch.CoverageWithPermittedGapsLength, 1.48d);
Assert.AreEqual(1d, bestMatch.RelativeCoverage, 0.005d);
Assert.AreEqual(1, bestMatch.Confidence, 0.01, $"Confidence:{bestMatch.Confidence}");
}
public void ShouldSerializeAndDeserialize()
{
var modelService = new InMemoryModelService();
var hashedFingerprints = FingerprintCommandBuilder.Instance.BuildFingerprintCommand()
.From(GetAudioSamples())
.UsingServices(audioService)
.Hash()
.Result;
var trackData = new TrackData("isrc", "artist", "title", "album", 2017, 200);
var trackReferences = modelService.InsertTrack(trackData);
modelService.InsertHashDataForTrack(hashedFingerprints, trackReferences);
var tempFile = Path.GetTempFileName();
modelService.Snapshot(tempFile);
var queryResult = QueryCommandBuilder.Instance.BuildQueryCommand()
.From(GetAudioSamples())
.UsingServices(new InMemoryModelService(tempFile), audioService)
.Query()
.Result;
File.Delete(tempFile);
Assert.IsTrue(queryResult.ContainsMatches);
AssertTracksAreEqual(trackData, queryResult.BestMatch.Track);
Assert.IsTrue(queryResult.BestMatch.Confidence > 0.9);
}
public async Task ShouldCreateFingerprintsInsertThenQueryAndGetTheRightResult()
{
const int secondsToProcess = 8;
const int startAtSecond = 2;
var track = new TrackInfo("id", "title", "artist");
var fingerprints = await FingerprintCommandBuilder.Instance
.BuildFingerprintCommand()
.From(PathToWav)
.WithFingerprintConfig(new HighPrecisionFingerprintConfiguration())
.UsingServices(audioService)
.Hash();
var modelService = new InMemoryModelService();
modelService.Insert(track, fingerprints);
var queryResult = await QueryCommandBuilder.Instance
.BuildQueryCommand()
.From(PathToWav, secondsToProcess, startAtSecond)
.WithQueryConfig(new HighPrecisionQueryConfiguration())
.UsingServices(modelService, audioService)
.Query();
Assert.IsTrue(queryResult.ContainsMatches);
Assert.AreEqual(1, queryResult.ResultEntries.Count());
var bestMatch = queryResult.BestMatch;
Assert.AreEqual("id", bestMatch.Track.Id);
Assert.IsTrue(bestMatch.CoverageWithPermittedGapsLength > secondsToProcess - 3, $"QueryCoverageSeconds:{bestMatch.QueryLength}");
Assert.AreEqual(startAtSecond, Math.Abs(bestMatch.TrackStartsAt), 0.1d);
Assert.IsTrue(bestMatch.Confidence > 0.7, $"Confidence:{bestMatch.Confidence}");
}
public void Setup()
{
audioService = new SoundFingerprintingAudioService();
lmdbModelService = new LMDBModelService(Path.Combine(Program.databasesPath, "db"));
inMemoryModelService = new InMemoryModelService(Path.Combine(Program.databasesPath, "memory.db"));
}
public void Setup()
{
_logger = new MemoryLogger();
_modelService = new InMemoryModelService();
_audioService = new SoundFingerprintingAudioService();
_indexer = new AudioIndexer(_logger, _modelService, _audioService);
}
public void ShouldSerializeAndIncrementNextIdCorrectly()
{
var modelService = new InMemoryModelService();
var firstTrack = new TrackInfo("id1", "title", "artist");
modelService.Insert(firstTrack, new Hashes(new[] { new HashedFingerprint(GenericHashBuckets(), 1, 0f, Array.Empty <byte>()) }, 1.48, DateTime.Now, Enumerable.Empty <string>()));
var tempFile = Path.GetTempFileName();
modelService.Snapshot(tempFile);
var fromFileService = new InMemoryModelService(tempFile);
var secondTrack = new TrackInfo("id2", "title", "artist");
fromFileService.Insert(secondTrack, new Hashes(new[] { new HashedFingerprint(GenericHashBuckets(), 1, 0f, Array.Empty <byte>()) }, 1.48, DateTime.Now, Enumerable.Empty <string>()));
var tracks = fromFileService.GetTrackIds().ToList();
File.Delete(tempFile);
Assert.IsTrue(tracks.Any(track => track == "id1"));
Assert.IsTrue(tracks.Any(track => track == "id2"));
}
public async Task ShouldPurgeCompletedMatchWhenAsyncCollectionIsExhausted()
{
var modelService = new InMemoryModelService();
const double minSizeChunk = 10240d / 5512; // length in seconds of one query chunk ~1.8577
const double totalTrackLength = 210; // length of the track 3 minutes 30 seconds.
var data = GenerateRandomAudioChunks((int)(totalTrackLength / minSizeChunk), seed: 1, DateTime.UtcNow);
var concatenated = Concatenate(data);
var hashes = await FingerprintCommandBuilder.Instance
.BuildFingerprintCommand()
.From(concatenated)
.UsingServices(new SoundFingerprintingAudioService())
.Hash();
modelService.Insert(new TrackInfo("312", "Bohemian Rhapsody", "Queen"), hashes);
var list = new List <ResultEntry>();
await QueryCommandBuilder.Instance.BuildRealtimeQueryCommand()
.From(SimulateRealtimeQueryData(data, jitterLength: 0))
.WithRealtimeQueryConfig(config =>
{
config.ResultEntryFilter = new TrackRelativeCoverageLengthEntryFilter(0.5, true);
config.SuccessCallback = entry => { list.Add(entry); };
return(config);
})
.UsingServices(modelService)
.Query(CancellationToken.None);
Assert.AreEqual(1, list.Count);
}
public void ShouldSerializeSpectralImages()
{
var spectrumService = new SpectrumService(new LomontFFT(), new LogUtility());
var spectrums = spectrumService.CreateLogSpectrogram(GetAudioSamples(), new DefaultSpectrogramConfig())
.Select(spectrum => spectrum.ImageRowCols)
.ToList();
var modelService = new InMemoryModelService();
var track = new TrackInfo("id", string.Empty, string.Empty);
var hashes = new Hashes(GetGenericHashes(), 10);
modelService.Insert(track, hashes);
modelService.InsertSpectralImages(spectrums, "id");
var tempFile = Path.GetTempFileName();
modelService.Snapshot(tempFile);
var fromFileService = new InMemoryModelService(tempFile);
File.Delete(tempFile);
var allSpectrums = fromFileService.GetSpectralImagesByTrackId("id").ToList();
Assert.AreEqual(spectrums.Count, allSpectrums.Count);
}
public async Task QueryingWithAggregatedHashesShouldResultInTheSameMatches()
{
var audioService = new SoundFingerprintingAudioService();
var modelService = new InMemoryModelService();
int count = 20, testWaitTime = 5000;
var data = GenerateRandomAudioChunks(count, 1, DateTime.UtcNow);
var concatenated = Concatenate(data);
var hashes = await FingerprintCommandBuilder.Instance
.BuildFingerprintCommand()
.From(concatenated)
.WithFingerprintConfig(config => config)
.UsingServices(audioService)
.Hash();
modelService.Insert(new TrackInfo("312", "Bohemian Rhapsody", "Queen"), hashes);
var collection = SimulateRealtimeQueryData(data, jitterLength: 0);
var cancellationTokenSource = new CancellationTokenSource(testWaitTime);
var fingerprints = new List <Hashes>();
var entries = new List <ResultEntry>();
await QueryCommandBuilder.Instance.BuildRealtimeQueryCommand()
.From(collection)
.WithRealtimeQueryConfig(config =>
{
config.SuccessCallback = entry => entries.Add(entry);
config.ResultEntryFilter = new TrackRelativeCoverageLengthEntryFilter(0.8d);
config.Stride = new IncrementalStaticStride(2048);
return(config);
})
.Intercept(queryHashes =>
{
fingerprints.Add(queryHashes);
return(queryHashes);
})
.UsingServices(modelService)
.Query(cancellationTokenSource.Token);
Assert.IsTrue(entries.Any());
Assert.AreEqual(1, entries.Count);
var realtimeResult = entries.First();
var aggregatedHashes = Hashes.Aggregate(fingerprints, 60d).First();
var nonRealtimeResult = await QueryCommandBuilder.Instance
.BuildQueryCommand()
.From(aggregatedHashes)
.UsingServices(modelService, audioService)
.Query();
Assert.IsTrue(nonRealtimeResult.ContainsMatches);
Assert.AreEqual(1, nonRealtimeResult.ResultEntries.Count());
Assert.AreEqual(realtimeResult.MatchedAt, aggregatedHashes.RelativeTo);
Assert.AreEqual(realtimeResult.MatchedAt, nonRealtimeResult.BestMatch.MatchedAt, $"Realtime vs NonRealtime {nonRealtimeResult.BestMatch.Coverage.BestPath.Count()} match time does not match");
}
public async Task HashesShouldMatchExactlyWhenAggregated()
{
var audioService = new SoundFingerprintingAudioService();
var modelService = new InMemoryModelService();
int count = 20;
var data = GenerateRandomAudioChunks(count, seed: 1, DateTime.UtcNow);
var concatenated = Concatenate(data);
var hashes = await FingerprintCommandBuilder.Instance
.BuildFingerprintCommand()
.From(concatenated)
.WithFingerprintConfig(config =>
{
config.Stride = new IncrementalStaticStride(512);
return(config);
})
.UsingServices(audioService)
.Hash();
var collection = SimulateRealtimeQueryData(data, jitterLength: 0);
var list = new List <Hashes>();
await QueryCommandBuilder.Instance.BuildRealtimeQueryCommand()
.From(collection)
.WithRealtimeQueryConfig(config =>
{
config.Stride = new IncrementalStaticStride(512);
return(config);
})
.Intercept(timedHashes =>
{
list.Add(timedHashes);
return(timedHashes);
})
.UsingServices(modelService)
.Query(CancellationToken.None);
Assert.AreEqual(hashes.Count, list.Select(entry => entry.Count).Sum());
var merged = Hashes.Aggregate(list, concatenated.Duration).ToList();
Assert.AreEqual(1, merged.Count, $"Hashes:{string.Join(",", merged.Select(_ => $"{_.RelativeTo},{_.DurationInSeconds:0.00}"))}");
Assert.AreEqual(hashes.Count, merged.Select(entry => entry.Count).Sum());
var aggregated = Hashes.Aggregate(list, double.MaxValue).ToList();
Assert.AreEqual(1, aggregated.Count);
Assert.AreEqual(hashes.Count, aggregated[0].Count);
foreach (var zipped in hashes.OrderBy(h => h.SequenceNumber).Zip(aggregated[0], (a, b) => new { a, b }))
{
Assert.AreEqual(zipped.a.StartsAt, zipped.b.StartsAt, 1d);
Assert.AreEqual(zipped.a.SequenceNumber, zipped.b.SequenceNumber);
CollectionAssert.AreEqual(zipped.a.HashBins, zipped.b.HashBins);
}
}
public async Task ShouldQueryInRealtime()
{
var audioService = new SoundFingerprintingAudioService();
var modelService = new InMemoryModelService();
int count = 10, found = 0, didNotPassThreshold = 0, thresholdVotes = 4, testWaitTime = 5000, fingerprintsCount = 0;
var data = GenerateRandomAudioChunks(count, 1);
var concatenated = Concatenate(data);
var hashes = await FingerprintCommandBuilder.Instance
.BuildFingerprintCommand()
.From(concatenated)
.UsingServices(audioService)
.Hash();
modelService.Insert(new TrackInfo("312", "Bohemian Rhapsody", "Queen"), hashes);
var collection = SimulateRealtimeQueryData(data, true, TimeSpan.FromMilliseconds);
var realtimeConfig = new RealtimeQueryConfiguration(thresholdVotes, new QueryMatchLengthFilter(10),
entry =>
{
Console.WriteLine($"Found Match Starts At {entry.TrackMatchStartsAt:0.000}, Match Length {entry.CoverageWithPermittedGapsLength:0.000}, Query Length {entry.QueryLength:0.000} Track Starts At {entry.TrackStartsAt:0.000}");
Interlocked.Increment(ref found);
},
entry =>
{
Console.WriteLine($"Entry didn't pass filter, Starts At {entry.TrackMatchStartsAt:0.000}, Match Length {entry.CoverageWithPermittedGapsLength:0.000}, Query Length {entry.CoverageWithPermittedGapsLength:0.000}");
Interlocked.Increment(ref didNotPassThreshold);
},
fingerprints => Interlocked.Add(ref fingerprintsCount, fingerprints.Count),
(error, _) => throw error,
() => throw new Exception("Downtime callback called"),
Enumerable.Empty<Hashes>(),
new IncrementalRandomStride(256, 512),
1.48d,
0d,
(int)(10240d/5512) * 1000,
new Dictionary<string, string>());
var cancellationTokenSource = new CancellationTokenSource(testWaitTime);
double processed = await QueryCommandBuilder.Instance.BuildRealtimeQueryCommand()
.From(collection)
.WithRealtimeQueryConfig(realtimeConfig)
.UsingServices(modelService)
.Query(cancellationTokenSource.Token);
Assert.AreEqual(1, found);
Assert.AreEqual(1, didNotPassThreshold);
Assert.AreEqual((count + 10) * 10240 / 5512d, processed, 0.2);
}
public async Task HashesShouldMatchExactlyWhenAggregated()
{
var audioService = new SoundFingerprintingAudioService();
var modelService = new InMemoryModelService();
int count = 20, testWaitTime = 40000;
var data = GenerateRandomAudioChunks(count, 1);
var concatenated = Concatenate(data);
var hashes = await FingerprintCommandBuilder.Instance
.BuildFingerprintCommand()
.From(concatenated)
.WithFingerprintConfig(config =>
{
config.Stride = new IncrementalStaticStride(512);
return(config);
})
.UsingServices(audioService)
.Hash();
var collection = SimulateRealtimeQueryData(data, false, TimeSpan.FromSeconds);
var cancellationTokenSource = new CancellationTokenSource(testWaitTime);
var list = new List <TimedHashes>();
await QueryCommandBuilder.Instance.BuildRealtimeQueryCommand()
.From(collection)
.WithRealtimeQueryConfig(config =>
{
config.QueryFingerprintsCallback += timedHashes => list.Add(timedHashes);
config.Stride = new IncrementalStaticStride(512);
return(config);
})
.UsingServices(modelService)
.Query(cancellationTokenSource.Token);
Assert.AreEqual(hashes.Count, list.Select(entry => entry.HashedFingerprints.Count).Sum());
var merged = TimedHashes.Aggregate(list, 20d).ToList();
Assert.AreEqual(2, merged.Count);
Assert.AreEqual(hashes.Count, merged.Select(entry => entry.HashedFingerprints.Count).Sum());
var aggregated = TimedHashes.Aggregate(list, double.MaxValue).ToList();
Assert.AreEqual(1, aggregated.Count);
Assert.AreEqual(hashes.Count, aggregated[0].HashedFingerprints.Count);
foreach (var zipped in hashes.OrderBy(h => h.SequenceNumber).Zip(aggregated[0].HashedFingerprints, (a, b) => new { a, b }))
{
Assert.AreEqual(zipped.a.StartsAt, zipped.b.StartsAt, 0.5d);
Assert.AreEqual(zipped.a.SequenceNumber, zipped.b.SequenceNumber);
CollectionAssert.AreEqual(zipped.a.HashBins, zipped.b.HashBins);
}
}
public async Task RealtimeQueryShouldMatchOnlySelectedClusters()
{
var audioService = new SoundFingerprintingAudioService();
var modelService = new InMemoryModelService();
int count = 10, foundWithClusters = 0, foundWithWrongClusters = 0, testWaitTime = 3000;
var data = GenerateRandomAudioChunks(count, 1);
var concatenated = Concatenate(data);
var hashes = await FingerprintCommandBuilder.Instance
.BuildFingerprintCommand()
.From(concatenated)
.UsingServices(audioService)
.Hash();
modelService.Insert(new TrackInfo("312", "Bohemian Rhapsody", "Queen", new Dictionary <string, string> {
{ "country", "USA" }
}), hashes);
var cancellationTokenSource = new CancellationTokenSource(testWaitTime);
var wrong = QueryCommandBuilder.Instance.BuildRealtimeQueryCommand()
.From(SimulateRealtimeQueryData(data, false, TimeSpan.FromMilliseconds))
.WithRealtimeQueryConfig(config =>
{
config.ResultEntryFilter = new QueryMatchLengthFilter(15d);
config.SuccessCallback = entry => Interlocked.Increment(ref foundWithWrongClusters);
config.MetaFieldsFilter = new Dictionary <string, string> {
{ "country", "CANADA" }
};
return(config);
})
.UsingServices(modelService)
.Query(cancellationTokenSource.Token);
var right = QueryCommandBuilder.Instance.BuildRealtimeQueryCommand()
.From(SimulateRealtimeQueryData(data, false, TimeSpan.FromMilliseconds))
.WithRealtimeQueryConfig(config =>
{
config.ResultEntryFilter = new QueryMatchLengthFilter(15d);
config.SuccessCallback = entry => Interlocked.Increment(ref foundWithClusters);
config.MetaFieldsFilter = new Dictionary <string, string> {
{ "country", "USA" }
};
return(config);
})
.UsingServices(modelService)
.Query(cancellationTokenSource.Token);
await Task.WhenAll(wrong, right);
Assert.AreEqual(1, foundWithClusters);
Assert.AreEqual(0, foundWithWrongClusters);
}
public async Task ShouldIdentifyMultipleRegionsOfTheSameMatch()
{
float[] match = TestUtilities.GenerateRandomFloatArray(10 * 5512, 1);
float[] withJitter = AddJitter(match, 15, 20);
var modelService = new InMemoryModelService();
var audioService = new SoundFingerprintingAudioService();
var hashes = await FingerprintCommandBuilder.Instance
.BuildFingerprintCommand()
.From(new AudioSamples(withJitter, "Queen", 5512))
.UsingServices(audioService)
.Hash();
modelService.Insert(new TrackInfo("123", "Bohemian Rhapsody", "Queen"), new Hashes(hashes, withJitter.Length / 5512f));
var result = await QueryCommandBuilder.Instance
.BuildQueryCommand()
.From(new AudioSamples(match, "cnn", 5512))
.WithQueryConfig(config =>
{
config.Stride = new IncrementalStaticStride(256);
config.AllowMultipleMatchesOfTheSameTrackInQuery = true;
return(config);
})
.UsingServices(modelService, audioService)
.Query();
Assert.IsTrue(result.ContainsMatches);
var entries = result.ResultEntries.OrderBy(entry => entry.TrackMatchStartsAt).ToList();
CollectionAssert.IsOrdered(entries[1]
.Coverage
.BestPath
.Select(_ => _.TrackSequenceNumber));
CollectionAssert.IsOrdered(entries[1]
.Coverage
.BestPath
.Select(_ => _.QuerySequenceNumber));
Assert.AreEqual(2, entries.Count);
Assert.AreEqual(10, entries[0].CoverageWithPermittedGapsLength, 1f);
Assert.AreEqual(10, entries[1].CoverageWithPermittedGapsLength, 1f);
Assert.AreEqual(15d, entries[0].TrackMatchStartsAt, 1f);
Assert.AreEqual(45d, entries[1].TrackMatchStartsAt, 1f);
}
public async Task QueryingWithAggregatedHashesShouldResultInTheSameMatches()
{
var audioService = new SoundFingerprintingAudioService();
var modelService = new InMemoryModelService();
int count = 20, testWaitTime = 5000;
var data = GenerateRandomAudioChunks(count, 1);
var concatenated = Concatenate(data);
var hashes = await FingerprintCommandBuilder.Instance
.BuildFingerprintCommand()
.From(concatenated)
.WithFingerprintConfig(config => config)
.UsingServices(audioService)
.Hash();
modelService.Insert(new TrackInfo("312", "Bohemian Rhapsody", "Queen"), hashes);
var collection = SimulateRealtimeQueryData(data, false, TimeSpan.FromMilliseconds);
var cancellationTokenSource = new CancellationTokenSource(testWaitTime);
var fingerprints = new List <TimedHashes>();
var entries = new List <ResultEntry>();
await QueryCommandBuilder.Instance.BuildRealtimeQueryCommand()
.From(collection)
.WithRealtimeQueryConfig(config =>
{
config.QueryFingerprintsCallback += timedHashes => fingerprints.Add(timedHashes);
config.SuccessCallback = entry => entries.Add(entry);
config.ResultEntryFilter = new CoverageLengthEntryFilter(0.8d);
return(config);
})
.UsingServices(modelService)
.Query(cancellationTokenSource.Token);
Assert.IsTrue(entries.Any());
Assert.AreEqual(1, entries.Count);
var aggregated = TimedHashes.Aggregate(fingerprints, 60d).ToList();
var result = await QueryCommandBuilder.Instance.BuildQueryCommand()
.From(new Hashes(aggregated[0].HashedFingerprints, aggregated[0].TotalSeconds))
.UsingServices(modelService, audioService)
.Query(aggregated[0].StartsAt);
Assert.IsTrue(result.ContainsMatches);
Assert.AreEqual(entries[0].MatchedAt, result.BestMatch.MatchedAt);
}
public void ShouldSerializeAndIncrementNextIdCorrectly()
{
var modelService = new InMemoryModelService();
var trackData = new TrackData("isrc", "artist", "title", "album", 2017, 200);
var trackReferences = modelService.InsertTrack(trackData);
var tempFile = Path.GetTempFileName();
modelService.Snapshot(tempFile);
var fromFileService = new InMemoryModelService(tempFile);
var newTrackReference = fromFileService.InsertTrack(trackData);
File.Delete(tempFile);
Assert.AreNotEqual(trackReferences, newTrackReference);
}
public async Task RealtimeQueryStrideShouldBeUsed()
{
var audioService = new SoundFingerprintingAudioService();
var modelService = new InMemoryModelService();
int minSize = 8192 + 2048;
int staticStride = 1024;
double permittedGap = (double)minSize / 5512;
int count = 10, found = 0, didNotPassThreshold = 0, fingerprintsCount = 0;
int testWaitTime = 3000;
var data = GenerateRandomAudioChunks(count, 1, DateTime.UtcNow);
var concatenated = Concatenate(data);
var hashes = await FingerprintCommandBuilder.Instance
.BuildFingerprintCommand()
.From(concatenated)
.UsingServices(audioService)
.Hash();
modelService.Insert(new TrackInfo("312", "Bohemian Rhapsody", "Queen"), hashes);
var collection = SimulateRealtimeQueryData(data, jitterLength: 0);
var cancellationTokenSource = new CancellationTokenSource(testWaitTime);
double duration = await QueryCommandBuilder.Instance.BuildRealtimeQueryCommand()
.From(collection)
.WithRealtimeQueryConfig(config =>
{
config.Stride = new IncrementalStaticStride(staticStride);
config.SuccessCallback = _ => Interlocked.Increment(ref found);
config.DidNotPassFilterCallback = _ => Interlocked.Increment(ref didNotPassThreshold);
config.PermittedGap = permittedGap;
return(config);
})
.Intercept(fingerprints =>
{
Interlocked.Add(ref fingerprintsCount, fingerprints.Count);
return(fingerprints);
})
.UsingServices(modelService)
.Query(cancellationTokenSource.Token);
Assert.AreEqual((count - 1) * minSize / staticStride + 1, fingerprintsCount);
Assert.AreEqual((double)count * minSize / 5512, duration, 0.00001);
}
public async Task ShouldIdentifyOnlyOneMatch()
{
float[] match = TestUtilities.GenerateRandomFloatArray(10 * 5512);
float[] withJitter = AddJitter(match);
var modelService = new InMemoryModelService();
var audioService = new SoundFingerprintingAudioService();
await InsertFingerprints(withJitter, audioService, modelService);
var result = await GetQueryResult(withJitter, audioService, modelService);
Assert.IsTrue(result.ContainsMatches);
var entries = result.ResultEntries.OrderBy(entry => entry.QueryMatchStartsAt).ToList();
Assert.AreEqual(1, entries.Count);
Assert.AreEqual(0d, entries[0].QueryMatchStartsAt, 1f);
Assert.AreEqual(0d, entries[0].TrackMatchStartsAt, 1f);
}
public async Task ShouldIdentifyConsecutiveRepeatingSequencesInTrack()
{
float[] match = TestUtilities.GenerateRandomFloatArray(10 * 5512);
float[] twoCopies = new float[match.Length * 2];
match.CopyTo(twoCopies, 0);
match.CopyTo(twoCopies, match.Length);
var modelService = new InMemoryModelService();
var audioService = new SoundFingerprintingAudioService();
await InsertFingerprints(twoCopies, audioService, modelService);
var result = await GetQueryResult(match, audioService, modelService);
Assert.AreEqual(2, result.ResultEntries.Count());
foreach (var entry in result.ResultEntries)
{
Assert.AreEqual(0.95, entry.Confidence, 0.05);
Assert.AreEqual(10, entry.CoverageWithPermittedGapsLength, 1);
}
}
public async Task ShouldIdentifyTwoSeparateMatchesDueToShiftBetweenThem()
{
float[] match = TestUtilities.GenerateRandomFloatArray(10 * 5512);
float[] track = AddJitter(match, beforeSec: 10, betweenSec: 10, afterSec: 10);
float[] query = AddJitter(match, beforeSec: 15, betweenSec: 0, afterSec: 15);
var modelService = new InMemoryModelService();
var audioService = new SoundFingerprintingAudioService();
await InsertFingerprints(track, audioService, modelService);
var result = await GetQueryResult(query, audioService, modelService);
Assert.IsTrue(result.ContainsMatches);
var entries = result.ResultEntries.OrderBy(entry => entry.TrackMatchStartsAt).ToList();
Assert.AreEqual(2, entries.Count);
Assert.AreEqual(15d, entries[0].QueryMatchStartsAt, 1f);
Assert.AreEqual(15d, entries[1].QueryMatchStartsAt, 1f);
Assert.AreEqual(10d, entries[0].TrackMatchStartsAt, 1f);
Assert.AreEqual(30d, entries[1].TrackMatchStartsAt, 1f);
}
public void ShouldFingerprintAndQuerySuccessfully()
{
var modelService = new InMemoryModelService();
for (int i = 0; i < 30; ++i)
{
var samples = new AudioSamples(TestUtilities.GenerateRandomFloatArray(120 * 5512), "${i}", 5512);
var hashes = FingerprintCommandBuilder.Instance.BuildFingerprintCommand()
.From(samples)
.UsingServices(audioService)
.Hash()
.Result;
var track = new TrackData("${i}", "", "", "", 2017, 120);
var trackReference = modelService.InsertTrack(track);
modelService.InsertHashDataForTrack(hashes, trackReference);
}
Console.WriteLine("Fingerprinting Time, Query Time, Candidates Found");
double avgFingerprinting = 0, avgQuery = 0;
int totalRuns = 10;
for (int i = 0; i < totalRuns; ++i)
{
var samples = new AudioSamples(TestUtilities.GenerateRandomFloatArray(120 * 5512), "${i}", 5512);
var queryResult = QueryCommandBuilder.Instance.BuildQueryCommand()
.From(samples)
.UsingServices(modelService, audioService)
.Query()
.Result;
Console.WriteLine("{0,10}ms{1,15}ms{2,15}", queryResult.Stats.FingerprintingDuration, queryResult.Stats.QueryDuration, queryResult.Stats.TotalFingerprintsAnalyzed);
avgFingerprinting += queryResult.Stats.FingerprintingDuration;
avgQuery += queryResult.Stats.QueryDuration;
}
Console.WriteLine("Avg. Fingerprinting: {0,0:000}ms, Avg. Query: {1, 0:000}ms", avgFingerprinting / totalRuns, avgQuery / totalRuns);
}
private static void Main()
{
if (!Directory.Exists(Path.Combine(databasesPath, "db")))
{
// Create databases
Console.WriteLine("Building databases");
var audioService = new SoundFingerprintingAudioService();
var lmdbModelService = new LMDBModelService(Path.Combine(databasesPath, "db"));
var inMemoryModelService = new InMemoryModelService();
foreach (var path in Directory.EnumerateFiles(RecognitionsBenchmark.refs_path, "*.wav"))
{
var filename = Path.GetFileNameWithoutExtension(path);
Console.WriteLine($"Adding track {filename}");
var track = new TrackInfo(filename, string.Empty, string.Empty);
var hashedFingerprints = FingerprintCommandBuilder.Instance
.BuildFingerprintCommand()
.From(path)
.UsingServices(audioService)
.Hash()
.Result;
lmdbModelService.Insert(track, hashedFingerprints);
inMemoryModelService.Insert(track, hashedFingerprints);
}
inMemoryModelService.Snapshot(Path.Combine(databasesPath, "memory.db"));
lmdbModelService.Dispose();
Console.WriteLine("Databases built");
}
// Run benchmarks
var summary = BenchmarkRunner.Run <RecognitionsBenchmark>();
}
public async Task ShouldQueryInRealtime()
{
var audioService = new SoundFingerprintingAudioService();
var modelService = new InMemoryModelService();
const double minSizeChunk = 10240d / 5512; // length in seconds of one query chunk ~1.8577
const double totalTrackLength = 210; // length of the track 3 minutes 30 seconds.
int count = (int)(totalTrackLength / minSizeChunk), fingerprintsCount = 0, queryMatchLength = 10;
var data = GenerateRandomAudioChunks(count, seed: 1);
var concatenated = Concatenate(data);
var hashes = await FingerprintCommandBuilder.Instance
.BuildFingerprintCommand()
.From(concatenated)
.UsingServices(audioService)
.Hash();
// hashes have to be equal to total track length +- 1 second
Assert.AreEqual(totalTrackLength, hashes.DurationInSeconds, delta: 1);
// store track data and associated hashes
modelService.Insert(new TrackInfo("312", "Bohemian Rhapsody", "Queen"), hashes);
var successMatches = new List <ResultEntry>();
var didNotGetToContiguousQueryMatchLengthMatch = new List <ResultEntry>();
var realtimeConfig = new RealtimeQueryConfiguration(thresholdVotes: 4, new TrackMatchLengthEntryFilter(queryMatchLength),
successCallback: entry =>
{
Console.WriteLine($"Found Match Starts At {entry.TrackMatchStartsAt:0.000}, Match Length {entry.TrackCoverageWithPermittedGapsLength:0.000}, Query Length {entry.QueryLength:0.000} Track Starts At {entry.TrackStartsAt:0.000}");
successMatches.Add(entry);
},
didNotPassFilterCallback: entry =>
{
Console.WriteLine($"Entry didn't pass filter, Starts At {entry.TrackMatchStartsAt:0.000}, Match Length {entry.TrackCoverageWithPermittedGapsLength:0.000}, Query Length {entry.TrackCoverageWithPermittedGapsLength:0.000}");
didNotGetToContiguousQueryMatchLengthMatch.Add(entry);
},
queryFingerprintsCallback: fingerprints => Interlocked.Add(ref fingerprintsCount, fingerprints.Count),
errorCallback: (error, _) => throw error,
restoredAfterErrorCallback: () => throw new Exception("Downtime callback called"),
downtimeHashes: Enumerable.Empty <Hashes>(),
stride: new IncrementalRandomStride(256, 512),
permittedGap: 2d,
downtimeCapturePeriod: 0d,
millisecondsDelay: (int)(10240d / 5512) * 1000,
metaFieldFilters: new Dictionary <string, string>());
// simulating realtime query, starting in the middle of the track ~1 min 45 seconds (105 seconds).
// and we query for 35 seconds
const double queryLength = 35;
// track ---------------------------- 210 seconds
// query ---- 35 seconds
// match starts at | 105 second
var realtimeQuery = data.Skip(count / 2).Take((int)(queryLength / minSizeChunk) + 1).ToArray();
Assert.AreEqual(queryLength, realtimeQuery.Sum(_ => _.Duration), 1); // asserting the total length of the query +- 1 second
// adding some jitter before and after the query which should not match
// track ---------------------------- 210 seconds
// q with jitter ^^^----^^^ 10 sec + 35 seconds + 10 sec = 55 sec
// match starts at | 105 second
const double jitterLength = 10;
var collection = SimulateRealtimeQueryData(realtimeQuery, jitterLength, TimeSpan.FromMilliseconds);
double processed = await QueryCommandBuilder.Instance
.BuildRealtimeQueryCommand()
.From(collection)
.WithRealtimeQueryConfig(realtimeConfig)
.UsingServices(modelService)
.Query(CancellationToken.None);
// since we start from the middle of the track ~1 min 45 seconds and query for 35 seconds with 10 seconds filter
// this means we will get 3 successful matches
// start: 105 seconds, query length: 35 seconds, query match filter length: 10
int matchesCount = (int)Math.Floor(queryLength / queryMatchLength);
Assert.AreEqual(matchesCount, successMatches.Count);
// since our realtime query was 35 seconds with 3 successful matches of 10
// there has to be one more purged match of 5 seconds which did not get through successful filter
Assert.AreEqual(1, didNotGetToContiguousQueryMatchLengthMatch.Count);
// verifying that we queried the correct amount of seconds
Assert.AreEqual(queryLength + 2 * jitterLength, processed, 1);
// track starts to match at the middle
// matches |||
// q with jitter ^^^---^^^
// expecting 3 matches at 105th, 115th and 125th second
double[] trackMatches = Enumerable.Repeat(totalTrackLength / 2, matchesCount).Select((matchAt, index) => matchAt + queryMatchLength * index).ToArray();
for (int i = 0; i < trackMatches.Length; ++i)
{
Assert.AreEqual(trackMatches[i], successMatches[i].TrackMatchStartsAt, 2.5);
}
}
public async Task ShouldNotLoseAudioSamplesInCaseIfExceptionIsThrown()
{
var audioService = new SoundFingerprintingAudioService();
var modelService = new InMemoryModelService();
int count = 10, found = 0, didNotPassThreshold = 0, thresholdVotes = 4, testWaitTime = 40000, fingerprintsCount = 0, errored = 0;
var data = GenerateRandomAudioChunks(count, 1);
var concatenated = Concatenate(data);
var hashes = await FingerprintCommandBuilder.Instance
.BuildFingerprintCommand()
.From(concatenated)
.UsingServices(audioService)
.Hash();
modelService.Insert(new TrackInfo("312", "Bohemian Rhapsody", "Queen"), hashes);
var started = DateTime.Now;
var resultEntries = new List<ResultEntry>();
var collection = SimulateRealtimeQueryData(data, true, TimeSpan.FromSeconds);
var cancellationTokenSource = new CancellationTokenSource(testWaitTime);
var offlineStorage = new OfflineStorage(Path.GetTempPath());
var restoreCalled = new bool[1];
double processed = await new RealtimeQueryCommand(FingerprintCommandBuilder.Instance, new FaultyQueryService(count, QueryFingerprintService.Instance))
.From(collection)
.WithRealtimeQueryConfig(config =>
{
config.SuccessCallback = entry =>
{
Interlocked.Increment(ref found);
resultEntries.Add(entry);
};
config.QueryFingerprintsCallback = fingerprints => Interlocked.Increment(ref fingerprintsCount);
config.DidNotPassFilterCallback = entry => Interlocked.Increment(ref didNotPassThreshold);
config.ErrorCallback = (exception, timedHashes) =>
{
Interlocked.Increment(ref errored);
offlineStorage.Save(timedHashes);
};
config.ResultEntryFilter = new QueryMatchLengthFilter(10);
config.RestoredAfterErrorCallback = () => restoreCalled[0] = true;
config.PermittedGap = 1.48d;
config.ThresholdVotes = thresholdVotes;
config.DowntimeHashes = offlineStorage;
config.DowntimeCapturePeriod = 3d;
return config;
})
.UsingServices(modelService)
.Query(cancellationTokenSource.Token);
Assert.AreEqual(count, errored);
Assert.AreEqual(20, fingerprintsCount);
Assert.IsTrue(restoreCalled[0]);
Assert.AreEqual(1, found);
var resultEntry = resultEntries[0];
double jitterLength = 5 * 10240 / 5512d;
Assert.AreEqual(0d, started.AddSeconds(jitterLength + resultEntry.TrackMatchStartsAt).Subtract(resultEntry.MatchedAt).TotalSeconds, 2d);
Assert.AreEqual(1, didNotPassThreshold);
Assert.AreEqual((count + 10) * 10240 / 5512d, processed, 0.2);
}
