public void SameQueryHashGeneratesMultipleTrackMatches()
{
var groupedQueryResults = new GroupedQueryResults(10d, DateTime.Now);
var random = new Random(1);
int runs = 100;
int[] counts = new int[runs];
var trackRef = new ModelReference <uint>(1);
int k = 0;
Parallel.For(0, runs, i =>
{
counts[i] = random.Next(5, 10);
var queryPoint = new HashedFingerprint(new int[25], (uint)i, i * 1.48f, Array.Empty <byte>());
for (int j = 0; j < counts[i]; ++j)
{
var dbPoint = new SubFingerprintData(new int[25], (uint)k, k * 0.01f, new ModelReference <uint>((uint)Interlocked.Increment(ref k)), trackRef);
groupedQueryResults.Add(queryPoint, dbPoint, i);
}
});
var allMatches = groupedQueryResults.GetMatchesForTrack(trackRef).ToList();
Assert.AreEqual(counts.Sum(), allMatches.Count);
Assert.AreEqual(runs, allMatches.Select(m => m.QuerySequenceNumber).Distinct().Count());
}
public void ReadSubFingerprintsByHashBucketsHavingThresholdTest()
{
var firstTrack = new TrackInfo("isrc1", "title", "artist");
var secondTrack = new TrackInfo("isrc2", "title", "artist");
int[] firstTrackBuckets =
{
1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25
};
int[] secondTrackBuckets =
{
2, 2, 4, 5, 6, 7, 7, 9, 10, 11, 12, 13, 14, 14, 16, 17, 18, 19, 20, 20, 22, 23, 24, 25, 26
};
var firstHashData = new HashedFingerprint(firstTrackBuckets, 1, 0.928f, Array.Empty <byte>());
var secondHashData = new HashedFingerprint(secondTrackBuckets, 1, 0.928f, Array.Empty <byte>());
modelService.Insert(firstTrack, new Hashes(new[] { firstHashData }, 200));
modelService.Insert(secondTrack, new Hashes(new[] { secondHashData }, 200));
// query buckets are similar with 5 elements from first track and 4 elements from second track
int[] queryBuckets =
{
3, 2, 5, 6, 7, 8, 7, 10, 11, 12, 13, 14, 15, 14, 17, 18, 19, 20, 21, 20, 23, 24, 25, 26, 25
};
var subFingerprints = modelService.Query(
new Hashes(new[] { new HashedFingerprint(queryBuckets, 0, 0f, Array.Empty <byte>()) }, 200),
new LowLatencyQueryConfiguration()).ToList();
subFingerprints.Count.Should().Be(1);
}
public void HammingSimilarityIsSummedUpAccrossAllSubFingerprintsTest()
{
var queryHash = new HashedFingerprint(GenericHashBuckets(), 0, 0, Enumerable.Empty <string>());
const int FirstTrackId = 20;
const int FirstSubFingerprintId = 10;
const int SecondSubFingerprintId = 11;
var firstTrackReference = new ModelReference <int>(FirstTrackId);
var firstResult = new SubFingerprintData(GenericHashBuckets(), 1, 0, new ModelReference <int>(FirstSubFingerprintId), firstTrackReference);
var secondResult = new SubFingerprintData(GenericHashBuckets(), 2, 0.928f, new ModelReference <int>(SecondSubFingerprintId), firstTrackReference);
var defaultQueryConfiguration = new DefaultQueryConfiguration();
modelService.Setup(service => service.SupportsBatchedSubFingerprintQuery).Returns(false);
modelService.Setup(service => service.ReadSubFingerprints(It.IsAny <int[]>(), defaultQueryConfiguration))
.Returns(new List <SubFingerprintData> {
firstResult, secondResult
});
modelService.Setup(service => service.ReadTracksByReferences(new [] { firstTrackReference }))
.Returns(new List <TrackData> {
new TrackData {
ISRC = "isrc", TrackReference = firstTrackReference
}
});
var queryResult = queryFingerprintService.Query(new List <HashedFingerprint> {
queryHash
}, defaultQueryConfiguration, modelService.Object);
Assert.IsTrue(queryResult.ContainsMatches);
Assert.AreEqual("isrc", queryResult.BestMatch.Track.ISRC);
Assert.AreEqual(firstTrackReference, queryResult.BestMatch.Track.TrackReference);
Assert.AreEqual(200, queryResult.BestMatch.HammingSimilaritySum);
Assert.AreEqual(1, queryResult.ResultEntries.Count());
}
public void ReadSubFingerprintsByHashBucketsHavingThresholdWithClustersTest()
{
var firstTrack = new TrackInfo("id1", "title", "artist", new Dictionary <string, string> {
{ "group-id", "first-group-id" }
});
var secondTrack = new TrackInfo("id2", "title", "artist", new Dictionary <string, string> {
{ "group-id", "second-group-id" }
});
int[] firstTrackBuckets = { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25 };
int[] secondTrackBuckets = { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25 };
var firstHashData = new HashedFingerprint(firstTrackBuckets, 1, 0.928f, Array.Empty <byte>());
var secondHashData = new HashedFingerprint(secondTrackBuckets, 1, 0.928f, Array.Empty <byte>());
modelService.Insert(firstTrack, new Hashes(new[] { firstHashData }, 1.48d, MediaType.Audio, DateTime.Now, Enumerable.Empty <string>()));
modelService.Insert(secondTrack, new Hashes(new[] { secondHashData }, 1.48d, MediaType.Audio, DateTime.Now, Enumerable.Empty <string>()));
// query buckets are similar with 5 elements from first track and 4 elements from second track
int[] queryBuckets = { 3, 2, 5, 6, 7, 8, 7, 10, 11, 12, 13, 14, 15, 14, 17, 18, 19, 20, 21, 20, 23, 24, 25, 26, 25 };
var queryHashes = new Hashes(new[] { new HashedFingerprint(queryBuckets, 0, 0f, Array.Empty <byte>()), }, 1.48d, MediaType.Audio, DateTime.Now, Enumerable.Empty <string>());
var subFingerprints = modelService.Query(queryHashes,
new DefaultQueryConfiguration
{
YesMetaFieldsFilters = new Dictionary <string, string> {
{ "group-id", "first-group-id" }
}
}).ToList();
Assert.AreEqual(1, subFingerprints.Count);
}
public void HammingSimilarityIsSummedUpAccrossAllSubFingerprintsTest()
{
long[] buckets = new long[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 };
var queryHash = new HashedFingerprint(new byte[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 11 }, buckets, 0, 0);
const int DefaultThreshold = 5;
const int FirstTrackId = 20;
const int FirstSubFingerprintId = 10;
const int SecondSubFingerprintId = 11;
var firstTrackReference = new ModelReference <int>(FirstTrackId);
SubFingerprintData firstResult = new SubFingerprintData(new byte[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 }, 1, 0, new ModelReference <int>(FirstSubFingerprintId), firstTrackReference);
SubFingerprintData secondResult = new SubFingerprintData(new byte[] { 1, 2, 3, 4, 5, 6, 7, 8, 10, 12 }, 2, 0.928, new ModelReference <int>(SecondSubFingerprintId), firstTrackReference);
modelService.Setup(service => service.ReadSubFingerprintDataByHashBucketsWithThreshold(buckets, DefaultThreshold))
.Returns(new List <SubFingerprintData> {
firstResult, secondResult
});
modelService.Setup(service => service.ReadTrackByReference(firstTrackReference))
.Returns(new TrackData {
ISRC = "isrc", TrackReference = firstTrackReference
});
var queryResult = queryFingerprintService.Query(modelService.Object, new List <HashedFingerprint> {
queryHash
}, new DefaultQueryConfiguration());
Assert.IsTrue(queryResult.IsSuccessful);
Assert.AreEqual("isrc", queryResult.BestMatch.Track.ISRC);
Assert.AreEqual(firstTrackReference, queryResult.BestMatch.Track.TrackReference);
Assert.AreEqual(9 + 8, queryResult.BestMatch.Similarity);
Assert.AreEqual(1, queryResult.AnalyzedCandidatesCount);
Assert.AreEqual(1, queryResult.ResultEntries.Count);
}
public void ShouldInsertEntriesInThreadSafeManner()
{
var storage = new RAMStorage(50);
var hashConverter = new HashConverter();
var hashes = Enumerable.Range(0, 100).Select(b => (byte)b).ToArray();
var longs = hashConverter.ToInts(hashes, 25);
int tracksCount = 520;
int subFingerprintsPerTrack = 33;
float one = 8192f / 5512;
Parallel.For(0, tracksCount, i =>
{
var trackReference = new ModelReference <int>(i);
for (int j = 0; j < subFingerprintsPerTrack; ++j)
{
var hashed = new HashedFingerprint(longs, (uint)j, j * one, Array.Empty <byte>());
storage.AddHashedFingerprint(hashed, trackReference);
}
});
for (int i = 0; i < 25; ++i)
{
var subFingerprints = storage.GetSubFingerprintsByHashTableAndHash(i, longs[i]);
Assert.AreEqual(tracksCount * subFingerprintsPerTrack, subFingerprints.Count);
}
}
public void ReadSubFingerprintsByHashBucketsHavingThresholdWithGroupIdTest()
{
const int Threshold = 5;
TrackData firstTrack = new TrackData("isrc1", "artist", "title", "album", 1986, 200)
{
GroupId = "first-group-id"
};
var firstTrackReference = ModelService.InsertTrack(firstTrack);
TrackData secondTrack = new TrackData("isrc2", "artist", "title", "album", 1986, 200)
{
GroupId = "second-group-id"
};
var secondTrackReference = ModelService.InsertTrack(secondTrack);
Assert.IsFalse(firstTrackReference.Equals(secondTrackReference));
long[] firstTrackBuckets = new long[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25 };
long[] secondTrackBuckets = new long[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25 };
var firstHashData = new HashedFingerprint(GenericSignature, firstTrackBuckets, 1, 0.928);
var secondHashData = new HashedFingerprint(GenericSignature, secondTrackBuckets, 1, 0.928);
ModelService.InsertHashDataForTrack(new[] { firstHashData }, firstTrackReference);
ModelService.InsertHashDataForTrack(new[] { secondHashData }, secondTrackReference);
// query buckets are similar with 5 elements from first track and 4 elements from second track
long[] queryBuckets = new long[] { 3, 2, 5, 6, 7, 8, 7, 10, 11, 12, 13, 14, 15, 14, 17, 18, 19, 20, 21, 20, 23, 24, 25, 26, 25 };
var subFingerprints = ModelService.ReadSubFingerprintDataByHashBucketsThresholdWithGroupId(queryBuckets, Threshold, "first-group-id");
Assert.IsTrue(subFingerprints.Count == 1);
Assert.AreEqual(firstTrackReference, subFingerprints[0].TrackReference);
}
public void HammingSimilarityIsSummedUpAcrossAllSubFingerprintsTest()
{
var queryHash = new HashedFingerprint(GenericHashBuckets(), 0, 0, Array.Empty <byte>());
const int firstTrackId = 20;
const int firstSubFingerprintId = 10;
const int secondSubFingerprintId = 11;
var firstTrackReference = new ModelReference <int>(firstTrackId);
var firstResult = new SubFingerprintData(GenericHashBuckets(), 1, 0, new ModelReference <int>(firstSubFingerprintId), firstTrackReference);
var secondResult = new SubFingerprintData(GenericHashBuckets(), 2, 0.928f, new ModelReference <int>(secondSubFingerprintId), firstTrackReference);
var defaultQueryConfiguration = new DefaultQueryConfiguration();
modelService.Setup(service => service.Query(
It.IsAny <Hashes>(),
It.IsAny <QueryConfiguration>())).Returns(new[] { firstResult, secondResult });
modelService.Setup(service => service.ReadTracksByReferences(new[] { firstTrackReference })).Returns(
new List <TrackData>
{
new TrackData("id", string.Empty, string.Empty, 0d, firstTrackReference)
});
var hashes = new Hashes(new List <HashedFingerprint> {
queryHash
}, 1.48f, DateTime.Now, Enumerable.Empty <string>());
var queryResult = queryFingerprintService.Query(hashes, defaultQueryConfiguration, modelService.Object);
Assert.IsTrue(queryResult.ContainsMatches);
Assert.AreEqual("id", queryResult.BestMatch.Track.Id);
Assert.AreEqual(firstTrackReference, queryResult.BestMatch.Track.TrackReference);
Assert.AreEqual(200, queryResult.BestMatch.Score);
Assert.AreEqual(1, queryResult.ResultEntries.Count());
}
public void ReadSubFingerprintsByHashBucketsHavingThresholdWithGroupIdTest()
{
TrackData firstTrack = new TrackData("isrc1", "artist", "title", "album", 1986, 200);
var firstTrackReference = modelService.InsertTrack(firstTrack);
TrackData secondTrack = new TrackData("isrc2", "artist", "title", "album", 1986, 200);
var secondTrackReference = modelService.InsertTrack(secondTrack);
Assert.IsFalse(firstTrackReference.Equals(secondTrackReference));
int[] firstTrackBuckets =
{
1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25
};
int[] secondTrackBuckets =
{
1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25
};
var firstHashData = new HashedFingerprint(firstTrackBuckets, 1, 0.928f, new[] { "first-group-id" });
var secondHashData = new HashedFingerprint(secondTrackBuckets, 1, 0.928f, new[] { "second-group-id" });
modelService.InsertHashDataForTrack(new[] { firstHashData }, firstTrackReference);
modelService.InsertHashDataForTrack(new[] { secondHashData }, secondTrackReference);
// query buckets are similar with 5 elements from first track and 4 elements from second track
int[] queryBuckets =
{
3, 2, 5, 6, 7, 8, 7, 10, 11, 12, 13, 14, 15, 14, 17, 18, 19, 20, 21, 20, 23, 24, 25, 26, 25
};
var subFingerprints = modelService.ReadSubFingerprints(queryBuckets, new DefaultQueryConfiguration {
Clusters = new[] { "first-group-id" }
});
Assert.AreEqual(1, subFingerprints.Count);
Assert.AreEqual(firstTrackReference, subFingerprints[0].TrackReference);
}
public void ReadSubFingerprintsByHashBucketsHavingThresholdTest()
{
TrackData firstTrack = new TrackData("isrc1", "artist", "title", "album", 1986, 200);
var firstTrackReference = modelService.InsertTrack(firstTrack);
TrackData secondTrack = new TrackData("isrc2", "artist", "title", "album", 1986, 200);
var secondTrackReference = modelService.InsertTrack(secondTrack);
Assert.IsFalse(firstTrackReference.Equals(secondTrackReference));
long[] firstTrackBuckets = new long[]
{
1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25
};
long[] secondTrackBuckets = new long[]
{
2, 2, 4, 5, 6, 7, 7, 9, 10, 11, 12, 13, 14, 14, 16, 17, 18, 19, 20, 20, 22, 23, 24, 25, 26
};
var firstHashData = new HashedFingerprint(GenericSignature(), firstTrackBuckets, 1, 0.928, Enumerable.Empty <string>());
var secondHashData = new HashedFingerprint(GenericSignature(), secondTrackBuckets, 1, 0.928, Enumerable.Empty <string>());
modelService.InsertHashDataForTrack(new[] { firstHashData }, firstTrackReference);
modelService.InsertHashDataForTrack(new[] { secondHashData }, secondTrackReference);
// query buckets are similar with 5 elements from first track and 4 elements from second track
long[] queryBuckets = new long[]
{
3, 2, 5, 6, 7, 8, 7, 10, 11, 12, 13, 14, 15, 14, 17, 18, 19, 20, 21, 20, 23, 24, 25, 26, 25
};
var subFingerprints = modelService.ReadSubFingerprints(queryBuckets, new DefaultQueryConfiguration());
Assert.AreEqual(1, subFingerprints.Count);
Assert.AreEqual(firstTrackReference, subFingerprints[0].TrackReference);
}
public ResultEntryAccumulator Add(HashedFingerprint hashedFingerprint, SubFingerprintData match, int hammingSimilarity)
{
HammingSimilaritySum += hammingSimilarity;
var matchedPair = new MatchedPair(hashedFingerprint, match, hammingSimilarity);
ResetBestMatchIfAppropriate(matchedPair);
matches.Add(matchedPair);
return(this);
}
public void MaximumNumberOfReturnedTracksIsLessThanAnalyzedCandidatesResultsTest()
{
var queryHash = new HashedFingerprint(GenericHashBuckets(), 1, 0, Array.Empty <byte>());
const int defaultThreshold = 5;
const int firstTrackId = 20;
const int secondTrackId = 21;
const int thirdTrackId = 22;
const int firstSubFingerprintId = 10;
const int secondSubFingerprintId = 11;
var firstTrackReference = new ModelReference <int>(firstTrackId);
var secondTrackReference = new ModelReference <int>(secondTrackId);
var firstResult = new SubFingerprintData(GenericHashBuckets(), 1, 0,
new ModelReference <int>(firstSubFingerprintId),
firstTrackReference);
var secondResult = new SubFingerprintData(GenericHashBuckets(), 2, 0.928f,
new ModelReference <int>(secondSubFingerprintId),
secondTrackReference);
var thirdResult = new SubFingerprintData(GenericHashBuckets(), 3, 0.928f * 2,
new ModelReference <int>(secondSubFingerprintId),
new ModelReference <int>(thirdTrackId));
var customQueryConfiguration = new DefaultQueryConfiguration {
MaxTracksToReturn = 2, ThresholdVotes = defaultThreshold
};
modelService
.Setup(service => service.Query(It.IsAny <Hashes>(), customQueryConfiguration))
.Returns(new[] { firstResult, secondResult, thirdResult });
modelService.Setup(service => service.ReadTracksByReferences(new[] { firstTrackReference, secondTrackReference }))
.Returns(new List <TrackData>
{
new TrackData("id", string.Empty, string.Empty, 0d, firstTrackReference),
new TrackData("id_1", string.Empty, string.Empty, 0d, secondTrackReference)
});
var hashes = new Hashes(new List <HashedFingerprint> {
queryHash
}, 1.48f, DateTime.Now, Enumerable.Empty <string>());
var queryResult = queryFingerprintService.Query(hashes, customQueryConfiguration, modelService.Object);
Assert.IsTrue(queryResult.ContainsMatches);
Assert.AreEqual("id", queryResult.BestMatch.Track.Id);
Assert.AreEqual(firstTrackReference, queryResult.BestMatch.Track.TrackReference);
Assert.AreEqual(100, queryResult.BestMatch.Score);
Assert.AreEqual(2, queryResult.ResultEntries.Count());
var results = queryResult.ResultEntries.ToList();
Assert.AreEqual(firstTrackReference, results[0].Track.TrackReference);
Assert.AreEqual(secondTrackReference, results[1].Track.TrackReference);
}
public void Add(HashedFingerprint hashedFingerprint, SubFingerprintData subFingerprintData, int hammingSimilarity)
{
similaritySumPerTrack.AddOrUpdate(subFingerprintData.TrackReference, hammingSimilarity, (key, oldHamming) => oldHamming + hammingSimilarity);
var matchedWith = new MatchedWith(hashedFingerprint.StartsAt, subFingerprintData.SequenceAt, hammingSimilarity);
if (!matches.TryGetValue(hashedFingerprint.SequenceNumber, out var matched))
{
matches.Add(hashedFingerprint.SequenceNumber, new Candidates(subFingerprintData.TrackReference, matchedWith));
}
else
{
matched.AddOrUpdateNewMatch(subFingerprintData.TrackReference, matchedWith);
}
}
public void InsertHashDataTest()
{
TrackData expectedTrack = new TrackData("isrc", "artist", "title", "album", 1986, 200);
var trackReference = modelService.InsertTrack(expectedTrack);
var hashedFingerprints = new HashedFingerprint(GenericHashBuckets(), 1, 0.928f, Enumerable.Empty <string>());
modelService.InsertHashDataForTrack(new[] { hashedFingerprints }, trackReference);
var subFingerprints = modelService.ReadSubFingerprints(GenericHashBuckets(), new DefaultQueryConfiguration());
Assert.AreEqual(1, subFingerprints.Count);
Assert.AreEqual(trackReference, subFingerprints[0].TrackReference);
Assert.AreNotEqual(0, subFingerprints[0].SubFingerprintReference.GetHashCode());
CollectionAssert.AreEqual(GenericHashBuckets(), subFingerprints[0].Hashes);
}
private ConcurrentDictionary <IModelReference, ResultEntryAccumulator> GetSimilaritiesUsingBatchedStrategy(IEnumerable <HashedFingerprint> queryFingerprints, QueryConfiguration configuration, IModelService modelService)
{
var hashedFingerprints = queryFingerprints as List <HashedFingerprint> ?? queryFingerprints.ToList();
var allCandidates = modelService.ReadSubFingerprints(hashedFingerprints.Select(querySubfingerprint => querySubfingerprint.HashBins), configuration);
var hammingSimilarities = new ConcurrentDictionary <IModelReference, ResultEntryAccumulator>();
foreach (var hashedFingerprint in hashedFingerprints)
{
HashedFingerprint queryFingerprint = hashedFingerprint;
var subFingerprints = allCandidates.Where(candidate => queryMath.IsCandidatePassingThresholdVotes(queryFingerprint, candidate, configuration.ThresholdVotes));
similarityUtility.AccumulateHammingSimilarity(subFingerprints, queryFingerprint, hammingSimilarities);
}
return(hammingSimilarities);
}
public void Add(HashedFingerprint queryFingerprint, SubFingerprintData resultSubFingerprint, double score)
{
lock (lockObject)
{
scoreSumPerTrack.AddOrUpdate(resultSubFingerprint.TrackReference, score, (key, old) => old + score);
var matchedWith = new MatchedWith(queryFingerprint.SequenceNumber, queryFingerprint.StartsAt, resultSubFingerprint.SequenceNumber, resultSubFingerprint.SequenceAt, score);
if (!sequenceToCandidates.TryGetValue(queryFingerprint.SequenceNumber, out Candidates candidates))
{
sequenceToCandidates.Add(queryFingerprint.SequenceNumber, new Candidates(resultSubFingerprint.TrackReference, matchedWith));
}
else
{
candidates.AddNewMatchForTrack(resultSubFingerprint.TrackReference, matchedWith);
}
}
}
public void AddSubfingerprint(HashedFingerprint hashedFingerprint, IModelReference trackReference)
{
var subFingerprintReference = new ModelReference <ulong>((ulong)Interlocked.Increment(ref subFingerprintReferenceCounter));
var subFingerprintData = new SubFingerprintData(
hashedFingerprint.HashBins,
hashedFingerprint.SequenceNumber,
hashedFingerprint.StartsAt,
subFingerprintReference,
trackReference)
{
Clusters = hashedFingerprint.Clusters
};
SubFingerprints[(ulong)subFingerprintData.SubFingerprintReference.Id] = subFingerprintData;
InsertHashes(hashedFingerprint.HashBins, subFingerprintReference.Id);
}
public void ShouldReadSubFingerprintsByHashBucketsHavingThreshold()
{
var firstTrack = new TrackData("isrc1", "artist", "title", "album", 1986, 200);
var firstTrackReference = modelService.InsertTrack(firstTrack);
var secondTrack = new TrackData("isrc2", "artist", "title", "album", 1986, 200);
var secondTrackReference = modelService.InsertTrack(secondTrack);
var firstHashData = new HashedFingerprint(GenericSignature(), firstTrackBuckets, 1, 0.928, Enumerable.Empty <string>());
var secondHashData = new HashedFingerprint(GenericSignature(), secondTrackBuckets, 1, 0.928, Enumerable.Empty <string>());
modelService.InsertHashDataForTrack(new[] { firstHashData }, firstTrackReference);
modelService.InsertHashDataForTrack(new[] { secondHashData }, secondTrackReference);
var subFingerprints = modelService.ReadSubFingerprints(queryBuckets, new DefaultQueryConfiguration());
Assert.AreEqual(1, subFingerprints.Count);
Assert.AreEqual(firstTrackReference, subFingerprints[0].TrackReference);
}
public void DeleteTrackTest()
{
TrackData track = new TrackData("isrc", "artist", "title", "album", 1986, 200);
var trackReference = modelService.InsertTrack(track);
var hashedFingerprints = new HashedFingerprint(GenericHashBuckets(), 1, 0.928f, Enumerable.Empty <string>());
modelService.InsertHashDataForTrack(new[] { hashedFingerprints }, trackReference);
modelService.DeleteTrack(trackReference);
var subFingerprints = modelService.ReadSubFingerprints(GenericHashBuckets(), new DefaultQueryConfiguration());
Assert.IsTrue(subFingerprints.Any() == false);
TrackData actualTrack = modelService.ReadTrackByReference(trackReference);
Assert.IsNull(actualTrack);
}
public void ReadSubFingerprintsByHashBucketsHavingThresholdWithGroupIdTest()
{
var firstTrack = new TrackInfo("isrc1", "title", "artist",
new Dictionary <string, string> {
{ "group-id", "first-group-id" }
});
var secondTrack = new TrackInfo("isrc2", "title", "artist",
new Dictionary <string, string> {
{ "group-id", "second-group-id" }
});
int[] firstTrackBuckets =
{
1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25
};
int[] secondTrackBuckets =
{
1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25
};
var firstHashData = new HashedFingerprint(firstTrackBuckets, 1, 0.928f, Array.Empty <byte>());
var secondHashData = new HashedFingerprint(secondTrackBuckets, 1, 0.928f, Array.Empty <byte>());
modelService.Insert(firstTrack, new Hashes(new[] { firstHashData }, 200));
modelService.Insert(secondTrack, new Hashes(new[] { secondHashData }, 200));
var firstTrackReference = modelService.ReadAllTracks().First(t => t.Id == firstTrack.Id).TrackReference;
var secondTrackReference = modelService.ReadAllTracks().First(t => t.Id == secondTrack.Id).TrackReference;
firstTrackReference.Should().NotBe(secondTrackReference);
// query buckets are similar with 5 elements from first track and 4 elements from second track
int[] queryBuckets =
{
3, 2, 5, 6, 7, 8, 7, 10, 11, 12, 13, 14, 15, 14, 17, 18, 19, 20, 21, 20, 23, 24, 25, 26, 25
};
var subFingerprints = modelService.Query(
new Hashes(new[] { new HashedFingerprint(queryBuckets, 0, 0f, Array.Empty <byte>()) }, 200),
new DefaultQueryConfiguration
{
MetaFieldsFilter = firstTrack.MetaFields
}).ToList();
subFingerprints.Count.Should().Be(1);
firstTrackReference.Should().Be(subFingerprints[0].TrackReference);
}
public void NoResultsReturnedFromUnderlyingStorageTest()
{
var queryHash = new HashedFingerprint(new[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 }, 0, 0, Array.Empty <byte>());
var customQueryConfiguration = new DefaultQueryConfiguration {
MaxTracksToReturn = 1, ThresholdVotes = 10, FingerprintConfiguration = new DefaultFingerprintConfiguration()
};
modelService.Setup(service => service.Query(It.IsAny <Hashes>(), customQueryConfiguration)).Returns(new List <SubFingerprintData>());
var hashes = new Hashes(new List <HashedFingerprint> {
queryHash
}, 148f, DateTime.Now, Enumerable.Empty <string>());
var queryResult = queryFingerprintService.Query(hashes, customQueryConfiguration, modelService.Object);
Assert.IsFalse(queryResult.ContainsMatches);
Assert.IsNull(queryResult.BestMatch);
}
public void ShouldCalculateQueryLengthCorrectly()
{
var config = new DefaultFingerprintConfiguration();
float delta = 0.05f;
int runs = 1000;
var bag = new ConcurrentBag <HashedFingerprint>();
Parallel.For(0, runs, i =>
{
var hashed = new HashedFingerprint(new int[0], (uint)i, i * delta);
bag.Add(hashed);
});
double length = bag.QueryLength(config);
Assert.AreEqual(length, delta * (runs - 1) + config.FingerprintLengthInSeconds, 0.0001);
}
public void NoResultsReturnedFromUnderlyingStorageTest()
{
var queryHash = new HashedFingerprint(new int[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 }, 0, 0, Enumerable.Empty <string>());
var customQueryConfiguration = new DefaultQueryConfiguration {
MaxTracksToReturn = 1, ThresholdVotes = 10, FingerprintConfiguration = new DefaultFingerprintConfiguration()
};
modelService.Setup(service => service.SupportsBatchedSubFingerprintQuery).Returns(false);
modelService.Setup(service => service.ReadSubFingerprints(It.IsAny <int[]>(), customQueryConfiguration)).Returns(new List <SubFingerprintData>());
var queryResult = queryFingerprintService.Query(new List <HashedFingerprint> {
queryHash
}, customQueryConfiguration, modelService.Object);
Assert.IsFalse(queryResult.ContainsMatches);
Assert.IsNull(queryResult.BestMatch);
Assert.AreEqual(0, queryResult.ResultEntries.Count());
}
public void DeleteTrackTest()
{
const int Threshold = 5;
TrackData track = new TrackData("isrc", "artist", "title", "album", 1986, 200);
var trackReference = ModelService.InsertTrack(track);
var hashedFingerprints = new HashedFingerprint(GenericSignature, GenericHashBuckets, 1, 0.928);
ModelService.InsertHashDataForTrack(new[] { hashedFingerprints }, trackReference);
ModelService.DeleteTrack(trackReference);
var subFingerprints = ModelService.ReadSubFingerprintDataByHashBucketsWithThreshold(GenericHashBuckets, Threshold);
Assert.IsTrue(subFingerprints.Any() == false);
TrackData actualTrack = ModelService.ReadTrackByReference(trackReference);
Assert.IsNull(actualTrack);
}
public void InsertHashDataTest()
{
const int Threshold = 5;
TrackData expectedTrack = new TrackData("isrc", "artist", "title", "album", 1986, 200);
var trackReference = ModelService.InsertTrack(expectedTrack);
var hashedFingerprints = new HashedFingerprint(GenericSignature, GenericHashBuckets, 1, 0.928);
ModelService.InsertHashDataForTrack(new[] { hashedFingerprints }, trackReference);
var subFingerprints = ModelService.ReadSubFingerprintDataByHashBucketsWithThreshold(GenericHashBuckets, Threshold);
Assert.IsTrue(subFingerprints.Count == 1);
Assert.AreEqual(trackReference, subFingerprints[0].TrackReference);
Assert.IsFalse(subFingerprints[0].SubFingerprintReference.GetHashCode() == 0);
for (int i = 0; i < GenericSignature.Length; i++)
{
Assert.AreEqual(GenericSignature[i], subFingerprints[0].Signature[i]);
}
}
public void MatchesShouldBeOrderedByQueryAt()
{
int runs = 1000;
var groupedQueryResults = new GroupedQueryResults(Enumerable.Empty <HashedFingerprint>());
var reference = new ModelReference <int>(1);
Parallel.For(0, runs, i =>
{
var hashed = new HashedFingerprint(new int[0], (uint)i, i, new string[0]);
var candidate = new SubFingerprintData(new int[0], (uint)i, runs - i, new ModelReference <uint>((uint)i), reference);
groupedQueryResults.Add(hashed, candidate, i);
});
var matchedWith = groupedQueryResults.GetMatchesForTrackOrderedByQueryAt(reference);
var ordered = matchedWith.Select(with => (int)with.QueryAt).ToList();
CollectionAssert.AreEqual(Enumerable.Range(0, runs), ordered);
}
public void AccumulateHammingSimilarity(
IEnumerable <SubFingerprintData> candidates,
HashedFingerprint expected,
ConcurrentDictionary <IModelReference, ResultEntryAccumulator> accumulator,
int keysPerHash)
{
foreach (var subFingerprint in candidates)
{
int hammingSimilarity = CalculateHammingSimilarity(
expected.HashBins,
subFingerprint.Hashes,
keysPerHash);
SubFingerprintData fingerprint = subFingerprint;
accumulator.AddOrUpdate(
subFingerprint.TrackReference,
reference => new ResultEntryAccumulator(expected, fingerprint, hammingSimilarity),
(reference, entryAccumulator) => entryAccumulator.Add(expected, fingerprint, hammingSimilarity));
}
}
public void NoResultsReturnedFromUnderlyingStorageTest()
{
var queryHash = new HashedFingerprint(new byte[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 11 }, new long[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 }, 0, 0);
modelService.Setup(service => service.ReadSubFingerprintDataByHashBucketsWithThreshold(It.IsAny <long[]>(), 10)).Returns(new List <SubFingerprintData>());
var queryResult = queryFingerprintService.Query(
modelService.Object,
new List <HashedFingerprint> {
queryHash
},
new CustomQueryConfiguration {
MaximumNumberOfTracksToReturnAsResult = 1, ThresholdVotes = 10
});
Assert.IsFalse(queryResult.IsSuccessful);
Assert.IsNull(queryResult.BestMatch);
Assert.AreEqual(0, queryResult.AnalyzedCandidatesCount);
Assert.AreEqual(0, queryResult.ResultEntries.Count);
}
public bool IsCandidatePassingThresholdVotes(HashedFingerprint queryFingerprint, SubFingerprintData candidate, int thresholdVotes)
{
int[] query = queryFingerprint.HashBins;
int[] result = candidate.Hashes;
int count = 0;
for (int i = 0; i < query.Length; ++i)
{
if (query[i] == result[i])
{
count++;
}
if (count >= thresholdVotes)
{
return(true);
}
}
return(false);
}
public void CanSerializeAndDeserialize()
{
var list = GetHashedFingerprints();
var timed = new TimedHashes(list, DateTime.Now);
var buffer = Serialize(timed);
var deserialized = Deserialize(buffer);
Assert.AreEqual(timed.HashedFingerprints.Count, deserialized.HashedFingerprints.Count);
Assert.AreEqual(timed.StartsAt, deserialized.StartsAt);
for (int i = 0; i < timed.HashedFingerprints.Count; ++i)
{
HashedFingerprint a = timed.HashedFingerprints[i];
HashedFingerprint b = deserialized.HashedFingerprints[i];
Assert.AreEqual(a.StartsAt, b.StartsAt);
Assert.AreEqual(a.SequenceNumber, b.SequenceNumber);
CollectionAssert.AreEqual(a.HashBins, b.HashBins);
}
}