Exemple #1
0
        public void SameQueryHashGeneratesMultipleTrackMatches()
        {
            var groupedQueryResults = new GroupedQueryResults(10d, DateTime.Now);

            var random = new Random(1);
            int runs   = 100;

            int[] counts   = new int[runs];
            var   trackRef = new ModelReference <uint>(1);
            int   k        = 0;

            Parallel.For(0, runs, i =>
            {
                counts[i]      = random.Next(5, 10);
                var queryPoint = new HashedFingerprint(new int[25], (uint)i, i * 1.48f, Array.Empty <byte>());
                for (int j = 0; j < counts[i]; ++j)
                {
                    var dbPoint = new SubFingerprintData(new int[25], (uint)k, k * 0.01f, new ModelReference <uint>((uint)Interlocked.Increment(ref k)), trackRef);
                    groupedQueryResults.Add(queryPoint, dbPoint, i);
                }
            });

            var allMatches = groupedQueryResults.GetMatchesForTrack(trackRef).ToList();

            Assert.AreEqual(counts.Sum(), allMatches.Count);
            Assert.AreEqual(runs, allMatches.Select(m => m.QuerySequenceNumber).Distinct().Count());
        }
        public void ReadSubFingerprintsByHashBucketsHavingThresholdTest()
        {
            var firstTrack  = new TrackInfo("isrc1", "title", "artist");
            var secondTrack = new TrackInfo("isrc2", "title", "artist");

            int[] firstTrackBuckets =
            {
                1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25
            };
            int[] secondTrackBuckets =
            {
                2, 2, 4, 5, 6, 7, 7, 9, 10, 11, 12, 13, 14, 14, 16, 17, 18, 19, 20, 20, 22, 23, 24, 25, 26
            };
            var firstHashData  = new HashedFingerprint(firstTrackBuckets, 1, 0.928f, Array.Empty <byte>());
            var secondHashData = new HashedFingerprint(secondTrackBuckets, 1, 0.928f, Array.Empty <byte>());

            modelService.Insert(firstTrack, new Hashes(new[] { firstHashData }, 200));
            modelService.Insert(secondTrack, new Hashes(new[] { secondHashData }, 200));

            // query buckets are similar with 5 elements from first track and 4 elements from second track
            int[] queryBuckets =
            {
                3, 2, 5, 6, 7, 8, 7, 10, 11, 12, 13, 14, 15, 14, 17, 18, 19, 20, 21, 20, 23, 24, 25, 26, 25
            };

            var subFingerprints = modelService.Query(
                new Hashes(new[] { new HashedFingerprint(queryBuckets, 0, 0f, Array.Empty <byte>()) }, 200),
                new LowLatencyQueryConfiguration()).ToList();

            subFingerprints.Count.Should().Be(1);
        }
        public void HammingSimilarityIsSummedUpAccrossAllSubFingerprintsTest()
        {
            var       queryHash                 = new HashedFingerprint(GenericHashBuckets(), 0, 0, Enumerable.Empty <string>());
            const int FirstTrackId              = 20;
            const int FirstSubFingerprintId     = 10;
            const int SecondSubFingerprintId    = 11;
            var       firstTrackReference       = new ModelReference <int>(FirstTrackId);
            var       firstResult               = new SubFingerprintData(GenericHashBuckets(), 1, 0, new ModelReference <int>(FirstSubFingerprintId), firstTrackReference);
            var       secondResult              = new SubFingerprintData(GenericHashBuckets(), 2, 0.928f, new ModelReference <int>(SecondSubFingerprintId), firstTrackReference);
            var       defaultQueryConfiguration = new DefaultQueryConfiguration();

            modelService.Setup(service => service.SupportsBatchedSubFingerprintQuery).Returns(false);
            modelService.Setup(service => service.ReadSubFingerprints(It.IsAny <int[]>(), defaultQueryConfiguration))
            .Returns(new List <SubFingerprintData> {
                firstResult, secondResult
            });
            modelService.Setup(service => service.ReadTracksByReferences(new [] { firstTrackReference }))
            .Returns(new List <TrackData> {
                new TrackData {
                    ISRC = "isrc", TrackReference = firstTrackReference
                }
            });

            var queryResult = queryFingerprintService.Query(new List <HashedFingerprint> {
                queryHash
            }, defaultQueryConfiguration, modelService.Object);

            Assert.IsTrue(queryResult.ContainsMatches);
            Assert.AreEqual("isrc", queryResult.BestMatch.Track.ISRC);
            Assert.AreEqual(firstTrackReference, queryResult.BestMatch.Track.TrackReference);
            Assert.AreEqual(200, queryResult.BestMatch.HammingSimilaritySum);
            Assert.AreEqual(1, queryResult.ResultEntries.Count());
        }
Exemple #4
0
        public void ReadSubFingerprintsByHashBucketsHavingThresholdWithClustersTest()
        {
            var firstTrack = new TrackInfo("id1", "title", "artist", new Dictionary <string, string> {
                { "group-id", "first-group-id" }
            });
            var secondTrack = new TrackInfo("id2", "title", "artist", new Dictionary <string, string> {
                { "group-id", "second-group-id" }
            });

            int[] firstTrackBuckets  = { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25 };
            int[] secondTrackBuckets = { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25 };
            var   firstHashData      = new HashedFingerprint(firstTrackBuckets, 1, 0.928f, Array.Empty <byte>());
            var   secondHashData     = new HashedFingerprint(secondTrackBuckets, 1, 0.928f, Array.Empty <byte>());

            modelService.Insert(firstTrack, new Hashes(new[] { firstHashData }, 1.48d, MediaType.Audio, DateTime.Now, Enumerable.Empty <string>()));
            modelService.Insert(secondTrack, new Hashes(new[] { secondHashData }, 1.48d, MediaType.Audio, DateTime.Now, Enumerable.Empty <string>()));

            // query buckets are similar with 5 elements from first track and 4 elements from second track
            int[] queryBuckets    = { 3, 2, 5, 6, 7, 8, 7, 10, 11, 12, 13, 14, 15, 14, 17, 18, 19, 20, 21, 20, 23, 24, 25, 26, 25 };
            var   queryHashes     = new Hashes(new[] { new HashedFingerprint(queryBuckets, 0, 0f, Array.Empty <byte>()), }, 1.48d, MediaType.Audio, DateTime.Now, Enumerable.Empty <string>());
            var   subFingerprints = modelService.Query(queryHashes,
                                                       new DefaultQueryConfiguration
            {
                YesMetaFieldsFilters = new Dictionary <string, string> {
                    { "group-id", "first-group-id" }
                }
            }).ToList();

            Assert.AreEqual(1, subFingerprints.Count);
        }
Exemple #5
0
        public void HammingSimilarityIsSummedUpAccrossAllSubFingerprintsTest()
        {
            long[]             buckets                = new long[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 };
            var                queryHash              = new HashedFingerprint(new byte[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 11 }, buckets, 0, 0);
            const int          DefaultThreshold       = 5;
            const int          FirstTrackId           = 20;
            const int          FirstSubFingerprintId  = 10;
            const int          SecondSubFingerprintId = 11;
            var                firstTrackReference    = new ModelReference <int>(FirstTrackId);
            SubFingerprintData firstResult            = new SubFingerprintData(new byte[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 }, 1, 0, new ModelReference <int>(FirstSubFingerprintId), firstTrackReference);
            SubFingerprintData secondResult           = new SubFingerprintData(new byte[] { 1, 2, 3, 4, 5, 6, 7, 8, 10, 12 }, 2, 0.928, new ModelReference <int>(SecondSubFingerprintId), firstTrackReference);

            modelService.Setup(service => service.ReadSubFingerprintDataByHashBucketsWithThreshold(buckets, DefaultThreshold))
            .Returns(new List <SubFingerprintData> {
                firstResult, secondResult
            });
            modelService.Setup(service => service.ReadTrackByReference(firstTrackReference))
            .Returns(new TrackData {
                ISRC = "isrc", TrackReference = firstTrackReference
            });

            var queryResult = queryFingerprintService.Query(modelService.Object, new List <HashedFingerprint> {
                queryHash
            }, new DefaultQueryConfiguration());

            Assert.IsTrue(queryResult.IsSuccessful);
            Assert.AreEqual("isrc", queryResult.BestMatch.Track.ISRC);
            Assert.AreEqual(firstTrackReference, queryResult.BestMatch.Track.TrackReference);
            Assert.AreEqual(9 + 8, queryResult.BestMatch.Similarity);
            Assert.AreEqual(1, queryResult.AnalyzedCandidatesCount);
            Assert.AreEqual(1, queryResult.ResultEntries.Count);
        }
        public void ShouldInsertEntriesInThreadSafeManner()
        {
            var storage       = new RAMStorage(50);
            var hashConverter = new HashConverter();

            var hashes = Enumerable.Range(0, 100).Select(b => (byte)b).ToArray();
            var longs  = hashConverter.ToInts(hashes, 25);

            int   tracksCount             = 520;
            int   subFingerprintsPerTrack = 33;
            float one = 8192f / 5512;

            Parallel.For(0, tracksCount, i =>
            {
                var trackReference = new ModelReference <int>(i);
                for (int j = 0; j < subFingerprintsPerTrack; ++j)
                {
                    var hashed = new HashedFingerprint(longs, (uint)j, j * one, Array.Empty <byte>());
                    storage.AddHashedFingerprint(hashed, trackReference);
                }
            });

            for (int i = 0; i < 25; ++i)
            {
                var subFingerprints = storage.GetSubFingerprintsByHashTableAndHash(i, longs[i]);
                Assert.AreEqual(tracksCount * subFingerprintsPerTrack, subFingerprints.Count);
            }
        }
        public void ReadSubFingerprintsByHashBucketsHavingThresholdWithGroupIdTest()
        {
            const int Threshold  = 5;
            TrackData firstTrack = new TrackData("isrc1", "artist", "title", "album", 1986, 200)
            {
                GroupId = "first-group-id"
            };
            var       firstTrackReference = ModelService.InsertTrack(firstTrack);
            TrackData secondTrack         = new TrackData("isrc2", "artist", "title", "album", 1986, 200)
            {
                GroupId = "second-group-id"
            };
            var secondTrackReference = ModelService.InsertTrack(secondTrack);

            Assert.IsFalse(firstTrackReference.Equals(secondTrackReference));
            long[] firstTrackBuckets  = new long[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25 };
            long[] secondTrackBuckets = new long[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25 };
            var    firstHashData      = new HashedFingerprint(GenericSignature, firstTrackBuckets, 1, 0.928);
            var    secondHashData     = new HashedFingerprint(GenericSignature, secondTrackBuckets, 1, 0.928);

            ModelService.InsertHashDataForTrack(new[] { firstHashData }, firstTrackReference);
            ModelService.InsertHashDataForTrack(new[] { secondHashData }, secondTrackReference);

            // query buckets are similar with 5 elements from first track and 4 elements from second track
            long[] queryBuckets = new long[] { 3, 2, 5, 6, 7, 8, 7, 10, 11, 12, 13, 14, 15, 14, 17, 18, 19, 20, 21, 20, 23, 24, 25, 26, 25 };

            var subFingerprints = ModelService.ReadSubFingerprintDataByHashBucketsThresholdWithGroupId(queryBuckets, Threshold, "first-group-id");

            Assert.IsTrue(subFingerprints.Count == 1);
            Assert.AreEqual(firstTrackReference, subFingerprints[0].TrackReference);
        }
        public void HammingSimilarityIsSummedUpAcrossAllSubFingerprintsTest()
        {
            var       queryHash                 = new HashedFingerprint(GenericHashBuckets(), 0, 0, Array.Empty <byte>());
            const int firstTrackId              = 20;
            const int firstSubFingerprintId     = 10;
            const int secondSubFingerprintId    = 11;
            var       firstTrackReference       = new ModelReference <int>(firstTrackId);
            var       firstResult               = new SubFingerprintData(GenericHashBuckets(), 1, 0, new ModelReference <int>(firstSubFingerprintId), firstTrackReference);
            var       secondResult              = new SubFingerprintData(GenericHashBuckets(), 2, 0.928f, new ModelReference <int>(secondSubFingerprintId), firstTrackReference);
            var       defaultQueryConfiguration = new DefaultQueryConfiguration();

            modelService.Setup(service => service.Query(
                                   It.IsAny <Hashes>(),
                                   It.IsAny <QueryConfiguration>())).Returns(new[] { firstResult, secondResult });

            modelService.Setup(service => service.ReadTracksByReferences(new[] { firstTrackReference })).Returns(
                new List <TrackData>
            {
                new TrackData("id", string.Empty, string.Empty, 0d, firstTrackReference)
            });

            var hashes = new Hashes(new List <HashedFingerprint> {
                queryHash
            }, 1.48f, DateTime.Now, Enumerable.Empty <string>());
            var queryResult = queryFingerprintService.Query(hashes, defaultQueryConfiguration, modelService.Object);

            Assert.IsTrue(queryResult.ContainsMatches);
            Assert.AreEqual("id", queryResult.BestMatch.Track.Id);
            Assert.AreEqual(firstTrackReference, queryResult.BestMatch.Track.TrackReference);
            Assert.AreEqual(200, queryResult.BestMatch.Score);
            Assert.AreEqual(1, queryResult.ResultEntries.Count());
        }
Exemple #9
0
        public void ReadSubFingerprintsByHashBucketsHavingThresholdWithGroupIdTest()
        {
            TrackData firstTrack           = new TrackData("isrc1", "artist", "title", "album", 1986, 200);
            var       firstTrackReference  = modelService.InsertTrack(firstTrack);
            TrackData secondTrack          = new TrackData("isrc2", "artist", "title", "album", 1986, 200);
            var       secondTrackReference = modelService.InsertTrack(secondTrack);

            Assert.IsFalse(firstTrackReference.Equals(secondTrackReference));
            int[] firstTrackBuckets =
            {
                1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25
            };
            int[] secondTrackBuckets =
            {
                1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25
            };
            var firstHashData  = new HashedFingerprint(firstTrackBuckets, 1, 0.928f, new[] { "first-group-id" });
            var secondHashData = new HashedFingerprint(secondTrackBuckets, 1, 0.928f, new[] { "second-group-id" });

            modelService.InsertHashDataForTrack(new[] { firstHashData }, firstTrackReference);
            modelService.InsertHashDataForTrack(new[] { secondHashData }, secondTrackReference);

            // query buckets are similar with 5 elements from first track and 4 elements from second track
            int[] queryBuckets =
            {
                3, 2, 5, 6, 7, 8, 7, 10, 11, 12, 13, 14, 15, 14, 17, 18, 19, 20, 21, 20, 23, 24, 25, 26, 25
            };

            var subFingerprints = modelService.ReadSubFingerprints(queryBuckets, new DefaultQueryConfiguration {
                Clusters = new[] { "first-group-id" }
            });

            Assert.AreEqual(1, subFingerprints.Count);
            Assert.AreEqual(firstTrackReference, subFingerprints[0].TrackReference);
        }
Exemple #10
0
        public void ReadSubFingerprintsByHashBucketsHavingThresholdTest()
        {
            TrackData firstTrack           = new TrackData("isrc1", "artist", "title", "album", 1986, 200);
            var       firstTrackReference  = modelService.InsertTrack(firstTrack);
            TrackData secondTrack          = new TrackData("isrc2", "artist", "title", "album", 1986, 200);
            var       secondTrackReference = modelService.InsertTrack(secondTrack);

            Assert.IsFalse(firstTrackReference.Equals(secondTrackReference));
            long[] firstTrackBuckets = new long[]
            {
                1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25
            };
            long[] secondTrackBuckets = new long[]
            {
                2, 2, 4, 5, 6, 7, 7, 9, 10, 11, 12, 13, 14, 14, 16, 17, 18, 19, 20, 20, 22, 23, 24, 25, 26
            };
            var firstHashData  = new HashedFingerprint(GenericSignature(), firstTrackBuckets, 1, 0.928, Enumerable.Empty <string>());
            var secondHashData = new HashedFingerprint(GenericSignature(), secondTrackBuckets, 1, 0.928, Enumerable.Empty <string>());

            modelService.InsertHashDataForTrack(new[] { firstHashData }, firstTrackReference);
            modelService.InsertHashDataForTrack(new[] { secondHashData }, secondTrackReference);

            // query buckets are similar with 5 elements from first track and 4 elements from second track
            long[] queryBuckets = new long[]
            {
                3, 2, 5, 6, 7, 8, 7, 10, 11, 12, 13, 14, 15, 14, 17, 18, 19, 20, 21, 20, 23, 24, 25, 26, 25
            };

            var subFingerprints = modelService.ReadSubFingerprints(queryBuckets, new DefaultQueryConfiguration());

            Assert.AreEqual(1, subFingerprints.Count);
            Assert.AreEqual(firstTrackReference, subFingerprints[0].TrackReference);
        }
Exemple #11
0
        public ResultEntryAccumulator Add(HashedFingerprint hashedFingerprint, SubFingerprintData match, int hammingSimilarity)
        {
            HammingSimilaritySum += hammingSimilarity;
            var matchedPair = new MatchedPair(hashedFingerprint, match, hammingSimilarity);

            ResetBestMatchIfAppropriate(matchedPair);
            matches.Add(matchedPair);
            return(this);
        }
        public void MaximumNumberOfReturnedTracksIsLessThanAnalyzedCandidatesResultsTest()
        {
            var       queryHash              = new HashedFingerprint(GenericHashBuckets(), 1, 0, Array.Empty <byte>());
            const int defaultThreshold       = 5;
            const int firstTrackId           = 20;
            const int secondTrackId          = 21;
            const int thirdTrackId           = 22;
            const int firstSubFingerprintId  = 10;
            const int secondSubFingerprintId = 11;
            var       firstTrackReference    = new ModelReference <int>(firstTrackId);
            var       secondTrackReference   = new ModelReference <int>(secondTrackId);
            var       firstResult            = new SubFingerprintData(GenericHashBuckets(), 1, 0,
                                                                      new ModelReference <int>(firstSubFingerprintId),
                                                                      firstTrackReference);
            var secondResult = new SubFingerprintData(GenericHashBuckets(), 2, 0.928f,
                                                      new ModelReference <int>(secondSubFingerprintId),
                                                      secondTrackReference);
            var thirdResult = new SubFingerprintData(GenericHashBuckets(), 3, 0.928f * 2,
                                                     new ModelReference <int>(secondSubFingerprintId),
                                                     new ModelReference <int>(thirdTrackId));

            var customQueryConfiguration = new DefaultQueryConfiguration {
                MaxTracksToReturn = 2, ThresholdVotes = defaultThreshold
            };

            modelService
            .Setup(service => service.Query(It.IsAny <Hashes>(), customQueryConfiguration))
            .Returns(new[] { firstResult, secondResult, thirdResult });

            modelService.Setup(service => service.ReadTracksByReferences(new[] { firstTrackReference, secondTrackReference }))
            .Returns(new List <TrackData>
            {
                new TrackData("id", string.Empty, string.Empty, 0d, firstTrackReference),
                new TrackData("id_1", string.Empty, string.Empty, 0d, secondTrackReference)
            });

            var hashes = new Hashes(new List <HashedFingerprint> {
                queryHash
            }, 1.48f, DateTime.Now, Enumerable.Empty <string>());
            var queryResult = queryFingerprintService.Query(hashes, customQueryConfiguration, modelService.Object);

            Assert.IsTrue(queryResult.ContainsMatches);
            Assert.AreEqual("id", queryResult.BestMatch.Track.Id);
            Assert.AreEqual(firstTrackReference, queryResult.BestMatch.Track.TrackReference);
            Assert.AreEqual(100, queryResult.BestMatch.Score);
            Assert.AreEqual(2, queryResult.ResultEntries.Count());
            var results = queryResult.ResultEntries.ToList();

            Assert.AreEqual(firstTrackReference, results[0].Track.TrackReference);
            Assert.AreEqual(secondTrackReference, results[1].Track.TrackReference);
        }
Exemple #13
0
        public void Add(HashedFingerprint hashedFingerprint, SubFingerprintData subFingerprintData, int hammingSimilarity)
        {
            similaritySumPerTrack.AddOrUpdate(subFingerprintData.TrackReference, hammingSimilarity, (key, oldHamming) => oldHamming + hammingSimilarity);
            var matchedWith = new MatchedWith(hashedFingerprint.StartsAt, subFingerprintData.SequenceAt, hammingSimilarity);

            if (!matches.TryGetValue(hashedFingerprint.SequenceNumber, out var matched))
            {
                matches.Add(hashedFingerprint.SequenceNumber, new Candidates(subFingerprintData.TrackReference, matchedWith));
            }
            else
            {
                matched.AddOrUpdateNewMatch(subFingerprintData.TrackReference, matchedWith);
            }
        }
Exemple #14
0
        public void InsertHashDataTest()
        {
            TrackData expectedTrack      = new TrackData("isrc", "artist", "title", "album", 1986, 200);
            var       trackReference     = modelService.InsertTrack(expectedTrack);
            var       hashedFingerprints = new HashedFingerprint(GenericHashBuckets(), 1, 0.928f, Enumerable.Empty <string>());

            modelService.InsertHashDataForTrack(new[] { hashedFingerprints }, trackReference);

            var subFingerprints = modelService.ReadSubFingerprints(GenericHashBuckets(), new DefaultQueryConfiguration());

            Assert.AreEqual(1, subFingerprints.Count);
            Assert.AreEqual(trackReference, subFingerprints[0].TrackReference);
            Assert.AreNotEqual(0, subFingerprints[0].SubFingerprintReference.GetHashCode());
            CollectionAssert.AreEqual(GenericHashBuckets(), subFingerprints[0].Hashes);
        }
Exemple #15
0
        private ConcurrentDictionary <IModelReference, ResultEntryAccumulator> GetSimilaritiesUsingBatchedStrategy(IEnumerable <HashedFingerprint> queryFingerprints, QueryConfiguration configuration, IModelService modelService)
        {
            var hashedFingerprints  = queryFingerprints as List <HashedFingerprint> ?? queryFingerprints.ToList();
            var allCandidates       = modelService.ReadSubFingerprints(hashedFingerprints.Select(querySubfingerprint => querySubfingerprint.HashBins), configuration);
            var hammingSimilarities = new ConcurrentDictionary <IModelReference, ResultEntryAccumulator>();

            foreach (var hashedFingerprint in hashedFingerprints)
            {
                HashedFingerprint queryFingerprint = hashedFingerprint;
                var subFingerprints = allCandidates.Where(candidate => queryMath.IsCandidatePassingThresholdVotes(queryFingerprint, candidate, configuration.ThresholdVotes));
                similarityUtility.AccumulateHammingSimilarity(subFingerprints, queryFingerprint, hammingSimilarities);
            }

            return(hammingSimilarities);
        }
Exemple #16
0
 public void Add(HashedFingerprint queryFingerprint, SubFingerprintData resultSubFingerprint, double score)
 {
     lock (lockObject)
     {
         scoreSumPerTrack.AddOrUpdate(resultSubFingerprint.TrackReference, score, (key, old) => old + score);
         var matchedWith = new MatchedWith(queryFingerprint.SequenceNumber, queryFingerprint.StartsAt, resultSubFingerprint.SequenceNumber, resultSubFingerprint.SequenceAt, score);
         if (!sequenceToCandidates.TryGetValue(queryFingerprint.SequenceNumber, out Candidates candidates))
         {
             sequenceToCandidates.Add(queryFingerprint.SequenceNumber, new Candidates(resultSubFingerprint.TrackReference, matchedWith));
         }
         else
         {
             candidates.AddNewMatchForTrack(resultSubFingerprint.TrackReference, matchedWith);
         }
     }
 }
Exemple #17
0
        public void AddSubfingerprint(HashedFingerprint hashedFingerprint, IModelReference trackReference)
        {
            var subFingerprintReference = new ModelReference <ulong>((ulong)Interlocked.Increment(ref subFingerprintReferenceCounter));
            var subFingerprintData      = new SubFingerprintData(
                hashedFingerprint.HashBins,
                hashedFingerprint.SequenceNumber,
                hashedFingerprint.StartsAt,
                subFingerprintReference,
                trackReference)
            {
                Clusters = hashedFingerprint.Clusters
            };

            SubFingerprints[(ulong)subFingerprintData.SubFingerprintReference.Id] = subFingerprintData;
            InsertHashes(hashedFingerprint.HashBins, subFingerprintReference.Id);
        }
        public void ShouldReadSubFingerprintsByHashBucketsHavingThreshold()
        {
            var firstTrack           = new TrackData("isrc1", "artist", "title", "album", 1986, 200);
            var firstTrackReference  = modelService.InsertTrack(firstTrack);
            var secondTrack          = new TrackData("isrc2", "artist", "title", "album", 1986, 200);
            var secondTrackReference = modelService.InsertTrack(secondTrack);
            var firstHashData        = new HashedFingerprint(GenericSignature(), firstTrackBuckets, 1, 0.928, Enumerable.Empty <string>());
            var secondHashData       = new HashedFingerprint(GenericSignature(), secondTrackBuckets, 1, 0.928, Enumerable.Empty <string>());

            modelService.InsertHashDataForTrack(new[] { firstHashData }, firstTrackReference);
            modelService.InsertHashDataForTrack(new[] { secondHashData }, secondTrackReference);

            var subFingerprints = modelService.ReadSubFingerprints(queryBuckets, new DefaultQueryConfiguration());

            Assert.AreEqual(1, subFingerprints.Count);
            Assert.AreEqual(firstTrackReference, subFingerprints[0].TrackReference);
        }
Exemple #19
0
        public void DeleteTrackTest()
        {
            TrackData track              = new TrackData("isrc", "artist", "title", "album", 1986, 200);
            var       trackReference     = modelService.InsertTrack(track);
            var       hashedFingerprints = new HashedFingerprint(GenericHashBuckets(), 1, 0.928f, Enumerable.Empty <string>());

            modelService.InsertHashDataForTrack(new[] { hashedFingerprints }, trackReference);

            modelService.DeleteTrack(trackReference);

            var subFingerprints = modelService.ReadSubFingerprints(GenericHashBuckets(), new DefaultQueryConfiguration());

            Assert.IsTrue(subFingerprints.Any() == false);
            TrackData actualTrack = modelService.ReadTrackByReference(trackReference);

            Assert.IsNull(actualTrack);
        }
Exemple #20
0
        public void ReadSubFingerprintsByHashBucketsHavingThresholdWithGroupIdTest()
        {
            var firstTrack = new TrackInfo("isrc1", "title", "artist",
                                           new Dictionary <string, string> {
                { "group-id", "first-group-id" }
            });
            var secondTrack = new TrackInfo("isrc2", "title", "artist",
                                            new Dictionary <string, string> {
                { "group-id", "second-group-id" }
            });

            int[] firstTrackBuckets =
            {
                1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25
            };
            int[] secondTrackBuckets =
            {
                1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25
            };
            var firstHashData  = new HashedFingerprint(firstTrackBuckets, 1, 0.928f, Array.Empty <byte>());
            var secondHashData = new HashedFingerprint(secondTrackBuckets, 1, 0.928f, Array.Empty <byte>());

            modelService.Insert(firstTrack, new Hashes(new[] { firstHashData }, 200));
            modelService.Insert(secondTrack, new Hashes(new[] { secondHashData }, 200));

            var firstTrackReference  = modelService.ReadAllTracks().First(t => t.Id == firstTrack.Id).TrackReference;
            var secondTrackReference = modelService.ReadAllTracks().First(t => t.Id == secondTrack.Id).TrackReference;

            firstTrackReference.Should().NotBe(secondTrackReference);

            // query buckets are similar with 5 elements from first track and 4 elements from second track
            int[] queryBuckets =
            {
                3, 2, 5, 6, 7, 8, 7, 10, 11, 12, 13, 14, 15, 14, 17, 18, 19, 20, 21, 20, 23, 24, 25, 26, 25
            };

            var subFingerprints = modelService.Query(
                new Hashes(new[] { new HashedFingerprint(queryBuckets, 0, 0f, Array.Empty <byte>()) }, 200),
                new DefaultQueryConfiguration
            {
                MetaFieldsFilter = firstTrack.MetaFields
            }).ToList();

            subFingerprints.Count.Should().Be(1);
            firstTrackReference.Should().Be(subFingerprints[0].TrackReference);
        }
        public void NoResultsReturnedFromUnderlyingStorageTest()
        {
            var queryHash = new HashedFingerprint(new[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 }, 0, 0, Array.Empty <byte>());
            var customQueryConfiguration = new DefaultQueryConfiguration {
                MaxTracksToReturn = 1, ThresholdVotes = 10, FingerprintConfiguration = new DefaultFingerprintConfiguration()
            };

            modelService.Setup(service => service.Query(It.IsAny <Hashes>(), customQueryConfiguration)).Returns(new List <SubFingerprintData>());

            var hashes = new Hashes(new List <HashedFingerprint> {
                queryHash
            }, 148f, DateTime.Now, Enumerable.Empty <string>());
            var queryResult = queryFingerprintService.Query(hashes, customQueryConfiguration, modelService.Object);

            Assert.IsFalse(queryResult.ContainsMatches);
            Assert.IsNull(queryResult.BestMatch);
        }
Exemple #22
0
        public void ShouldCalculateQueryLengthCorrectly()
        {
            var   config = new DefaultFingerprintConfiguration();
            float delta  = 0.05f;
            int   runs   = 1000;
            var   bag    = new ConcurrentBag <HashedFingerprint>();

            Parallel.For(0, runs, i =>
            {
                var hashed = new HashedFingerprint(new int[0], (uint)i, i * delta);
                bag.Add(hashed);
            });


            double length = bag.QueryLength(config);

            Assert.AreEqual(length, delta * (runs - 1) + config.FingerprintLengthInSeconds, 0.0001);
        }
        public void NoResultsReturnedFromUnderlyingStorageTest()
        {
            var queryHash = new HashedFingerprint(new int[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 }, 0, 0, Enumerable.Empty <string>());
            var customQueryConfiguration = new DefaultQueryConfiguration {
                MaxTracksToReturn = 1, ThresholdVotes = 10, FingerprintConfiguration = new DefaultFingerprintConfiguration()
            };

            modelService.Setup(service => service.SupportsBatchedSubFingerprintQuery).Returns(false);
            modelService.Setup(service => service.ReadSubFingerprints(It.IsAny <int[]>(), customQueryConfiguration)).Returns(new List <SubFingerprintData>());

            var queryResult = queryFingerprintService.Query(new List <HashedFingerprint> {
                queryHash
            }, customQueryConfiguration, modelService.Object);

            Assert.IsFalse(queryResult.ContainsMatches);
            Assert.IsNull(queryResult.BestMatch);
            Assert.AreEqual(0, queryResult.ResultEntries.Count());
        }
        public void DeleteTrackTest()
        {
            const int Threshold          = 5;
            TrackData track              = new TrackData("isrc", "artist", "title", "album", 1986, 200);
            var       trackReference     = ModelService.InsertTrack(track);
            var       hashedFingerprints = new HashedFingerprint(GenericSignature, GenericHashBuckets, 1, 0.928);

            ModelService.InsertHashDataForTrack(new[] { hashedFingerprints }, trackReference);

            ModelService.DeleteTrack(trackReference);

            var subFingerprints = ModelService.ReadSubFingerprintDataByHashBucketsWithThreshold(GenericHashBuckets, Threshold);

            Assert.IsTrue(subFingerprints.Any() == false);
            TrackData actualTrack = ModelService.ReadTrackByReference(trackReference);

            Assert.IsNull(actualTrack);
        }
        public void InsertHashDataTest()
        {
            const int Threshold          = 5;
            TrackData expectedTrack      = new TrackData("isrc", "artist", "title", "album", 1986, 200);
            var       trackReference     = ModelService.InsertTrack(expectedTrack);
            var       hashedFingerprints = new HashedFingerprint(GenericSignature, GenericHashBuckets, 1, 0.928);

            ModelService.InsertHashDataForTrack(new[] { hashedFingerprints }, trackReference);

            var subFingerprints = ModelService.ReadSubFingerprintDataByHashBucketsWithThreshold(GenericHashBuckets, Threshold);

            Assert.IsTrue(subFingerprints.Count == 1);
            Assert.AreEqual(trackReference, subFingerprints[0].TrackReference);
            Assert.IsFalse(subFingerprints[0].SubFingerprintReference.GetHashCode() == 0);
            for (int i = 0; i < GenericSignature.Length; i++)
            {
                Assert.AreEqual(GenericSignature[i], subFingerprints[0].Signature[i]);
            }
        }
Exemple #26
0
        public void MatchesShouldBeOrderedByQueryAt()
        {
            int runs = 1000;

            var groupedQueryResults = new GroupedQueryResults(Enumerable.Empty <HashedFingerprint>());
            var reference           = new ModelReference <int>(1);

            Parallel.For(0, runs, i =>
            {
                var hashed    = new HashedFingerprint(new int[0], (uint)i, i, new string[0]);
                var candidate = new SubFingerprintData(new int[0], (uint)i, runs - i, new ModelReference <uint>((uint)i), reference);
                groupedQueryResults.Add(hashed, candidate, i);
            });

            var matchedWith = groupedQueryResults.GetMatchesForTrackOrderedByQueryAt(reference);

            var ordered = matchedWith.Select(with => (int)with.QueryAt).ToList();

            CollectionAssert.AreEqual(Enumerable.Range(0, runs), ordered);
        }
Exemple #27
0
        public void AccumulateHammingSimilarity(
            IEnumerable <SubFingerprintData> candidates,
            HashedFingerprint expected,
            ConcurrentDictionary <IModelReference, ResultEntryAccumulator> accumulator,
            int keysPerHash)
        {
            foreach (var subFingerprint in candidates)
            {
                int hammingSimilarity = CalculateHammingSimilarity(
                    expected.HashBins,
                    subFingerprint.Hashes,
                    keysPerHash);

                SubFingerprintData fingerprint = subFingerprint;
                accumulator.AddOrUpdate(
                    subFingerprint.TrackReference,
                    reference => new ResultEntryAccumulator(expected, fingerprint, hammingSimilarity),
                    (reference, entryAccumulator) => entryAccumulator.Add(expected, fingerprint, hammingSimilarity));
            }
        }
Exemple #28
0
        public void NoResultsReturnedFromUnderlyingStorageTest()
        {
            var queryHash = new HashedFingerprint(new byte[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 11 }, new long[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 }, 0, 0);

            modelService.Setup(service => service.ReadSubFingerprintDataByHashBucketsWithThreshold(It.IsAny <long[]>(), 10)).Returns(new List <SubFingerprintData>());

            var queryResult = queryFingerprintService.Query(
                modelService.Object,
                new List <HashedFingerprint> {
                queryHash
            },
                new CustomQueryConfiguration {
                MaximumNumberOfTracksToReturnAsResult = 1, ThresholdVotes = 10
            });

            Assert.IsFalse(queryResult.IsSuccessful);
            Assert.IsNull(queryResult.BestMatch);
            Assert.AreEqual(0, queryResult.AnalyzedCandidatesCount);
            Assert.AreEqual(0, queryResult.ResultEntries.Count);
        }
Exemple #29
0
        public bool IsCandidatePassingThresholdVotes(HashedFingerprint queryFingerprint, SubFingerprintData candidate, int thresholdVotes)
        {
            int[] query  = queryFingerprint.HashBins;
            int[] result = candidate.Hashes;
            int   count  = 0;

            for (int i = 0; i < query.Length; ++i)
            {
                if (query[i] == result[i])
                {
                    count++;
                }

                if (count >= thresholdVotes)
                {
                    return(true);
                }
            }

            return(false);
        }
Exemple #30
0
        public void CanSerializeAndDeserialize()
        {
            var list = GetHashedFingerprints();

            var timed        = new TimedHashes(list, DateTime.Now);
            var buffer       = Serialize(timed);
            var deserialized = Deserialize(buffer);

            Assert.AreEqual(timed.HashedFingerprints.Count, deserialized.HashedFingerprints.Count);
            Assert.AreEqual(timed.StartsAt, deserialized.StartsAt);

            for (int i = 0; i < timed.HashedFingerprints.Count; ++i)
            {
                HashedFingerprint a = timed.HashedFingerprints[i];
                HashedFingerprint b = deserialized.HashedFingerprints[i];

                Assert.AreEqual(a.StartsAt, b.StartsAt);
                Assert.AreEqual(a.SequenceNumber, b.SequenceNumber);
                CollectionAssert.AreEqual(a.HashBins, b.HashBins);
            }
        }