public void ShouldReadSubFingerprintsByHashBucketsHavingThreshold() { const int Threshold = 5; TrackData firstTrack = new TrackData("isrc1", "artist", "title", "album", 1986, 200); var firstTrackReference = modelService.InsertTrack(firstTrack); TrackData secondTrack = new TrackData("isrc2", "artist", "title", "album", 1986, 200); var secondTrackReference = modelService.InsertTrack(secondTrack); long[] firstTrackBuckets = new long[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25 }; long[] secondTrackBuckets = new long[] { 2, 2, 4, 5, 6, 7, 7, 9, 10, 11, 12, 13, 14, 14, 16, 17, 18, 19, 20, 20, 22, 23, 24, 25, 26 }; var firstHashData = new HashedFingerprint(GenericSignature, firstTrackBuckets, 1, 0.928); var secondHashData = new HashedFingerprint(GenericSignature, secondTrackBuckets, 1, 0.928); modelService.InsertHashDataForTrack(new[] { firstHashData }, firstTrackReference); modelService.InsertHashDataForTrack(new[] { secondHashData }, secondTrackReference); // query buckets are similar with 5 elements from first track and 4 elements from second track long[] queryBuckets = new long[] { 3, 2, 5, 6, 7, 8, 7, 10, 11, 12, 13, 14, 15, 14, 17, 18, 19, 20, 21, 20, 23, 24, 25, 26, 25 }; var subFingerprints = modelService.ReadSubFingerprintDataByHashBucketsWithThreshold(queryBuckets, Threshold); Assert.IsTrue(subFingerprints.Count == 1); Assert.AreEqual(firstTrackReference, subFingerprints[0].TrackReference); }
public void HammingSimilarityIsSummedUpAccrossAllSubFingerprintsTest() { var queryHash = new HashedFingerprint(GenericSignature(), GenericHashBuckets(), 0, 0, Enumerable.Empty<string>()); const int FirstTrackId = 20; const int FirstSubFingerprintId = 10; const int SecondSubFingerprintId = 11; var firstTrackReference = new ModelReference<int>(FirstTrackId); var firstResult = new SubFingerprintData(GenericHashBuckets(), 1, 0, new ModelReference<int>(FirstSubFingerprintId), firstTrackReference); var secondResult = new SubFingerprintData(GenericHashBuckets(), 2, 0.928, new ModelReference<int>(SecondSubFingerprintId), firstTrackReference); var defaultQueryConfiguration = new DefaultQueryConfiguration(); modelService.Setup(service => service.SupportsBatchedSubFingerprintQuery).Returns(false); modelService.Setup(service => service.ReadSubFingerprints(It.IsAny<long[]>(), defaultQueryConfiguration)) .Returns(new List<SubFingerprintData> { firstResult, secondResult }); modelService.Setup(service => service.ReadTrackByReference(firstTrackReference)) .Returns(new TrackData { ISRC = "isrc", TrackReference = firstTrackReference }); var queryResult = queryFingerprintService.Query(new List<HashedFingerprint> { queryHash }, defaultQueryConfiguration, modelService.Object); Assert.IsTrue(queryResult.ContainsMatches); Assert.AreEqual("isrc", queryResult.BestMatch.Track.ISRC); Assert.AreEqual(firstTrackReference, queryResult.BestMatch.Track.TrackReference); Assert.AreEqual(GenericSignature().Length * 2, queryResult.BestMatch.HammingSimilaritySum); Assert.AreEqual(1, queryResult.ResultEntries.Count()); }
public ResultEntryAccumulator Add(HashedFingerprint hashedFingerprint, SubFingerprintData match, int hammingSimilarity) { HammingSimilaritySum += hammingSimilarity; var matchedPair = new MatchedPair(hashedFingerprint, match, hammingSimilarity); ResetBestMatchIfAppropriate(matchedPair); matches.Add(matchedPair); return this; }
public void MaximumNumberOfReturnedTracksIsLessThanAnalyzedCandidatesResultsTest() { long[] buckets = new long[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 }; var queryHash = new HashedFingerprint(new byte[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 11 }, buckets, 1, 0); const int DefaultThreshold = 5; const int FirstTrackId = 20; const int SecondTrackId = 21; const int ThirdTrackId = 22; const int FirstSubFingerprintId = 10; const int SecondSubFingerprintId = 11; var firstTrackReference = new ModelReference<int>(FirstTrackId); var thirdTrackReference = new ModelReference<int>(ThirdTrackId); var firstResult = new SubFingerprintData( new byte[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 }, 1, 0, new ModelReference<int>(FirstSubFingerprintId), firstTrackReference); SubFingerprintData secondResult = new SubFingerprintData( new byte[] { 11, 2, 13, 4, 15, 6, 7, 8, 10, 12 }, 2, 0.928, new ModelReference<int>(SecondSubFingerprintId), new ModelReference<int>(SecondTrackId)); SubFingerprintData thirdResult = new SubFingerprintData( new byte[] { 1, 2, 3, 4, 5, 15, 7, 8, 10, 12 }, 3, 0.928 * 2, new ModelReference<int>(SecondSubFingerprintId), new ModelReference<int>(ThirdTrackId)); modelService.Setup( service => service.ReadSubFingerprintDataByHashBucketsWithThreshold(buckets, DefaultThreshold)).Returns( new List<SubFingerprintData> { firstResult, secondResult, thirdResult }); modelService.Setup(service => service.ReadTrackByReference(firstTrackReference)).Returns( new TrackData { ISRC = "isrc", TrackReference = firstTrackReference }); modelService.Setup(service => service.ReadTrackByReference(thirdTrackReference)).Returns( new TrackData { ISRC = "isrc_2", TrackReference = thirdTrackReference }); var queryResult = queryFingerprintService.Query( modelService.Object, new List<HashedFingerprint> { queryHash }, new CustomQueryConfiguration { MaximumNumberOfTracksToReturnAsResult = 2, ThresholdVotes = DefaultThreshold }); Assert.IsTrue(queryResult.IsSuccessful); Assert.AreEqual("isrc", queryResult.BestMatch.Track.ISRC); Assert.AreEqual(firstTrackReference, queryResult.BestMatch.Track.TrackReference); Assert.AreEqual(9, queryResult.BestMatch.Similarity); Assert.AreEqual(3, queryResult.AnalyzedCandidatesCount); Assert.AreEqual(2, queryResult.ResultEntries.Count); Assert.AreEqual(firstTrackReference, queryResult.ResultEntries[0].Track.TrackReference); Assert.AreEqual(thirdTrackReference, queryResult.ResultEntries[1].Track.TrackReference); }
public void MaximumNumberOfReturnedTracksIsLessThanAnalyzedCandidatesResultsTest() { var queryHash = new HashedFingerprint(GenericSignature(), GenericHashBuckets(), 1, 0, Enumerable.Empty<string>()); const int DefaultThreshold = 5; const int FirstTrackId = 20; const int SecondTrackId = 21; const int ThirdTrackId = 22; const int FirstSubFingerprintId = 10; const int SecondSubFingerprintId = 11; var firstTrackReference = new ModelReference<int>(FirstTrackId); var secondTrackReference = new ModelReference<int>(SecondTrackId); var firstResult = new SubFingerprintData( GenericHashBuckets(), 1, 0, new ModelReference<int>(FirstSubFingerprintId), firstTrackReference); var secondResult = new SubFingerprintData( GenericHashBuckets(), 2, 0.928, new ModelReference<int>(SecondSubFingerprintId), secondTrackReference); var thirdResult = new SubFingerprintData( GenericHashBuckets(), 3, 0.928 * 2, new ModelReference<int>(SecondSubFingerprintId), new ModelReference<int>(ThirdTrackId)); var customQueryConfiguration = new DefaultQueryConfiguration { MaxTracksToReturn = 2, ThresholdVotes = DefaultThreshold }; modelService.Setup(service => service.SupportsBatchedSubFingerprintQuery).Returns(false); modelService.Setup( service => service.ReadSubFingerprints(It.IsAny<long[]>(), customQueryConfiguration)).Returns( new List<SubFingerprintData> { firstResult, secondResult, thirdResult }); modelService.Setup(service => service.ReadTrackByReference(firstTrackReference)).Returns( new TrackData { ISRC = "isrc", TrackReference = firstTrackReference }); modelService.Setup(service => service.ReadTrackByReference(secondTrackReference)).Returns( new TrackData { ISRC = "isrc_1", TrackReference = secondTrackReference }); var queryResult = queryFingerprintService.Query(new List<HashedFingerprint> { queryHash }, customQueryConfiguration, modelService.Object); Assert.IsTrue(queryResult.ContainsMatches); Assert.AreEqual("isrc", queryResult.BestMatch.Track.ISRC); Assert.AreEqual(firstTrackReference, queryResult.BestMatch.Track.TrackReference); Assert.AreEqual(50, queryResult.BestMatch.HammingSimilaritySum); Assert.AreEqual(2, queryResult.ResultEntries.Count()); var results = queryResult.ResultEntries.ToList(); Assert.AreEqual(firstTrackReference, results[0].Track.TrackReference); Assert.AreEqual(secondTrackReference, results[1].Track.TrackReference); }
public void AccumulateHammingSimilarity(IEnumerable<SubFingerprintData> candidates, HashedFingerprint expected, ConcurrentDictionary<IModelReference, ResultEntryAccumulator> accumulator) { foreach (var subFingerprint in candidates) { byte[] signature = hashConverter.ToBytes(subFingerprint.Hashes, expected.SubFingerprint.Length); int hammingSimilarity = CalculateHammingSimilarity(expected.SubFingerprint, signature); SubFingerprintData fingerprint = subFingerprint; accumulator.AddOrUpdate( subFingerprint.TrackReference, reference => new ResultEntryAccumulator(expected, fingerprint, hammingSimilarity), (reference, entryAccumulator) => entryAccumulator.Add(expected, fingerprint, hammingSimilarity)); } }
public void InsertHashDataTest() { TrackData expectedTrack = new TrackData("isrc", "artist", "title", "album", 1986, 200); var trackReference = modelService.InsertTrack(expectedTrack); var hashedFingerprints = new HashedFingerprint(GenericSignature(), GenericHashBuckets(), 1, 0.928, Enumerable.Empty<string>()); modelService.InsertHashDataForTrack(new[] { hashedFingerprints }, trackReference); var subFingerprints = modelService.ReadSubFingerprints(GenericHashBuckets(), new DefaultQueryConfiguration()); Assert.AreEqual(1, subFingerprints.Count); Assert.AreEqual(trackReference, subFingerprints[0].TrackReference); Assert.AreNotEqual(0, subFingerprints[0].SubFingerprintReference.GetHashCode()); CollectionAssert.AreEqual(GenericHashBuckets(), subFingerprints[0].Hashes); }
public void DeleteTrackTest() { TrackData track = new TrackData("isrc", "artist", "title", "album", 1986, 200); var trackReference = modelService.InsertTrack(track); var hashedFingerprints = new HashedFingerprint(GenericSignature(), GenericHashBuckets(), 1, 0.928, Enumerable.Empty<string>()); modelService.InsertHashDataForTrack(new[] { hashedFingerprints }, trackReference); modelService.DeleteTrack(trackReference); var subFingerprints = modelService.ReadSubFingerprints(GenericHashBuckets(), new DefaultQueryConfiguration()); Assert.IsTrue(subFingerprints.Any() == false); TrackData actualTrack = modelService.ReadTrackByReference(trackReference); Assert.IsNull(actualTrack); }
public void OnlyTracksWithGroupIdAreConsideredAsPotentialCandidatesTest() { long[] buckets = new long[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 }; var queryHash = new HashedFingerprint(new byte[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 11 }, buckets, 0, 0); const int DefaultThreshold = 5; const int FirstTrackId = 20; const int FirstSubFingerprintId = 10; var firstTrackReference = new ModelReference<int>(FirstTrackId); SubFingerprintData firstResult = new SubFingerprintData( new byte[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 }, 1, 0.928, new ModelReference<int>(FirstSubFingerprintId), firstTrackReference); modelService.Setup( service => service.ReadSubFingerprintDataByHashBucketsThresholdWithGroupId(buckets, DefaultThreshold, "group-id")) .Returns(new List<SubFingerprintData> { firstResult }); modelService.Setup(service => service.ReadTrackByReference(firstTrackReference)).Returns( new TrackData { ISRC = "isrc", TrackReference = firstTrackReference }); var queryResult = queryFingerprintService.Query( modelService.Object, new List<HashedFingerprint> { queryHash }, new CustomQueryConfiguration { TrackGroupId = "group-id" }); Assert.IsTrue(queryResult.IsSuccessful); Assert.AreEqual("isrc", queryResult.BestMatch.Track.ISRC); Assert.AreEqual(firstTrackReference, queryResult.BestMatch.Track.TrackReference); }
public void HammingSimilarityIsSummedUpAccrossAllSubFingerprintsTest() { long[] buckets = new long[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 }; var queryHash = new HashedFingerprint(new byte[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 11 }, buckets, 0, 0); const int DefaultThreshold = 5; const int FirstTrackId = 20; const int FirstSubFingerprintId = 10; const int SecondSubFingerprintId = 11; var firstTrackReference = new ModelReference<int>(FirstTrackId); SubFingerprintData firstResult = new SubFingerprintData(new byte[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 }, 1, 0, new ModelReference<int>(FirstSubFingerprintId), firstTrackReference); SubFingerprintData secondResult = new SubFingerprintData(new byte[] { 1, 2, 3, 4, 5, 6, 7, 8, 10, 12 }, 2, 0.928, new ModelReference<int>(SecondSubFingerprintId), firstTrackReference); modelService.Setup(service => service.ReadSubFingerprintDataByHashBucketsWithThreshold(buckets, DefaultThreshold)) .Returns(new List<SubFingerprintData> { firstResult, secondResult }); modelService.Setup(service => service.ReadTrackByReference(firstTrackReference)) .Returns(new TrackData { ISRC = "isrc", TrackReference = firstTrackReference }); var queryResult = queryFingerprintService.Query(modelService.Object, new List<HashedFingerprint> { queryHash }, new DefaultQueryConfiguration()); Assert.IsTrue(queryResult.IsSuccessful); Assert.AreEqual("isrc", queryResult.BestMatch.Track.ISRC); Assert.AreEqual(firstTrackReference, queryResult.BestMatch.Track.TrackReference); Assert.AreEqual(9 + 8, queryResult.BestMatch.Similarity); Assert.AreEqual(1, queryResult.AnalyzedCandidatesCount); Assert.AreEqual(1, queryResult.ResultEntries.Count); }
public void InsertHashDataTest() { const int Threshold = 5; TrackData expectedTrack = new TrackData("isrc", "artist", "title", "album", 1986, 200); var trackReference = ModelService.InsertTrack(expectedTrack); var hashedFingerprints = new HashedFingerprint(GenericSignature, GenericHashBuckets, 1, 0.928); ModelService.InsertHashDataForTrack(new[] { hashedFingerprints }, trackReference); var subFingerprints = ModelService.ReadSubFingerprintDataByHashBucketsWithThreshold(GenericHashBuckets, Threshold); Assert.IsTrue(subFingerprints.Count == 1); Assert.AreEqual(trackReference, subFingerprints[0].TrackReference); Assert.IsFalse(subFingerprints[0].SubFingerprintReference.GetHashCode() == 0); for (int i = 0; i < GenericSignature.Length; i++) { Assert.AreEqual(GenericSignature[i], subFingerprints[0].Signature[i]); } }
public void DeleteTrackTest() { const int Threshold = 5; TrackData track = new TrackData("isrc", "artist", "title", "album", 1986, 200); var trackReference = ModelService.InsertTrack(track); var hashedFingerprints = new HashedFingerprint(GenericSignature, GenericHashBuckets, 1, 0.928); ModelService.InsertHashDataForTrack(new[] { hashedFingerprints }, trackReference); ModelService.DeleteTrack(trackReference); var subFingerprints = ModelService.ReadSubFingerprintDataByHashBucketsWithThreshold(GenericHashBuckets, Threshold); Assert.IsTrue(subFingerprints.Any() == false); TrackData actualTrack = ModelService.ReadTrackByReference(trackReference); Assert.IsNull(actualTrack); }
public void ReadSubFingerprintsByHashBucketsHavingThresholdWithGroupIdTest() { TrackData firstTrack = new TrackData("isrc1", "artist", "title", "album", 1986, 200); var firstTrackReference = modelService.InsertTrack(firstTrack); TrackData secondTrack = new TrackData("isrc2", "artist", "title", "album", 1986, 200); var secondTrackReference = modelService.InsertTrack(secondTrack); Assert.IsFalse(firstTrackReference.Equals(secondTrackReference)); long[] firstTrackBuckets = new long[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25 }; long[] secondTrackBuckets = new long[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25 }; var firstHashData = new HashedFingerprint(GenericSignature(), firstTrackBuckets, 1, 0.928, new[] { "first-group-id" }); var secondHashData = new HashedFingerprint(GenericSignature(), secondTrackBuckets, 1, 0.928, new[] { "second-group-id" }); modelService.InsertHashDataForTrack(new[] { firstHashData }, firstTrackReference); modelService.InsertHashDataForTrack(new[] { secondHashData }, secondTrackReference); // query buckets are similar with 5 elements from first track and 4 elements from second track long[] queryBuckets = new long[] { 3, 2, 5, 6, 7, 8, 7, 10, 11, 12, 13, 14, 15, 14, 17, 18, 19, 20, 21, 20, 23, 24, 25, 26, 25 }; var subFingerprints = modelService.ReadSubFingerprints(queryBuckets, new DefaultQueryConfiguration { Clusters = new[] { "first-group-id" } }); Assert.AreEqual(1, subFingerprints.Count); Assert.AreEqual(firstTrackReference, subFingerprints[0].TrackReference); }
private IEnumerable<SubFingerprintData> GetSubFingerprints(IModelService modelService, HashedFingerprint hash, QueryConfiguration queryConfiguration) { if (!string.IsNullOrEmpty(queryConfiguration.TrackGroupId)) { return modelService.ReadSubFingerprintDataByHashBucketsThresholdWithGroupId(hash.HashBins, queryConfiguration.ThresholdVotes, queryConfiguration.TrackGroupId); } return modelService.ReadSubFingerprintDataByHashBucketsWithThreshold(hash.HashBins, queryConfiguration.ThresholdVotes); }
private void InsertSubFingerprint(HashedFingerprint hashedFingerprint, IModelReference trackReference) { var subFingerprintReference = new ModelReference<long>(Interlocked.Increment(ref counter)); storage.SubFingerprints[subFingerprintReference] = new SubFingerprintData( hashedFingerprint.HashBins, hashedFingerprint.SequenceNumber, hashedFingerprint.StartsAt, subFingerprintReference, trackReference) { Clusters = hashedFingerprint.Clusters }; if (!storage.TracksHashes.ContainsKey(trackReference)) { storage.TracksHashes[trackReference] = new ConcurrentDictionary<IModelReference, HashedFingerprint>(); } storage.TracksHashes[trackReference][subFingerprintReference] = hashedFingerprint; this.InsertHashes(hashedFingerprint.HashBins, subFingerprintReference); }
public void NoResultsReturnedFromUnderlyingStorageTest() { var queryHash = new HashedFingerprint(new byte[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 11 }, new long[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 }, 0, 0, Enumerable.Empty<string>()); var customQueryConfiguration = new DefaultQueryConfiguration { MaxTracksToReturn = 1, ThresholdVotes = 10, FingerprintConfiguration = new DefaultFingerprintConfiguration() }; modelService.Setup(service => service.SupportsBatchedSubFingerprintQuery).Returns(false); modelService.Setup(service => service.ReadSubFingerprints(It.IsAny<long[]>(), customQueryConfiguration)).Returns(new List<SubFingerprintData>()); var queryResult = queryFingerprintService.Query(new List<HashedFingerprint> { queryHash }, customQueryConfiguration, modelService.Object); Assert.IsFalse(queryResult.ContainsMatches); Assert.IsNull(queryResult.BestMatch); Assert.AreEqual(0, queryResult.ResultEntries.Count()); }
public void NoResultsReturnedFromUnderlyingStorageTest() { var queryHash = new HashedFingerprint(new byte[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 11 }, new long[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 }, 0, 0); modelService.Setup(service => service.ReadSubFingerprintDataByHashBucketsWithThreshold(It.IsAny<long[]>(), 10)).Returns(new List<SubFingerprintData>()); var queryResult = queryFingerprintService.Query( modelService.Object, new List<HashedFingerprint> { queryHash }, new CustomQueryConfiguration { MaximumNumberOfTracksToReturnAsResult = 1, ThresholdVotes = 10 }); Assert.IsFalse(queryResult.IsSuccessful); Assert.IsNull(queryResult.BestMatch); Assert.AreEqual(0, queryResult.AnalyzedCandidatesCount); Assert.AreEqual(0, queryResult.ResultEntries.Count); }
public ResultEntryAccumulator(HashedFingerprint hashedFingerprint, SubFingerprintData match, int hammingSimilarity) { BestMatch = new MatchedPair(hashedFingerprint, match, hammingSimilarity); Add(hashedFingerprint, match, hammingSimilarity); }