public void HammingSimilarityIsSummedUpAccrossAllSubFingerprintsTest() { long[] buckets = new long[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 }; var queryHash = new HashData(new byte[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 11 }, buckets); const int DefaultThreshold = 5; const int FirstTrackId = 20; const int FirstSubFingerprintId = 10; const int SecondSubFingerprintId = 11; var firstTrackReference = new ModelReference <int>(FirstTrackId); SubFingerprintData firstResult = new SubFingerprintData(new byte[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 }, new ModelReference <int>(FirstSubFingerprintId), firstTrackReference); SubFingerprintData secondResult = new SubFingerprintData(new byte[] { 1, 2, 3, 4, 5, 6, 7, 8, 10, 12 }, new ModelReference <int>(SecondSubFingerprintId), firstTrackReference); modelService.Setup(service => service.ReadSubFingerprintDataByHashBucketsWithThreshold(buckets, DefaultThreshold)) .Returns(new List <SubFingerprintData> { firstResult, secondResult }); modelService.Setup(service => service.ReadTrackByReference(firstTrackReference)) .Returns(new TrackData { ISRC = "isrc", TrackReference = firstTrackReference }); var queryResult = queryFingerprintService.Query(new List <HashData> { queryHash }, new DefaultQueryConfiguration()); Assert.IsTrue(queryResult.IsSuccessful); Assert.AreEqual("isrc", queryResult.BestMatch.Track.ISRC); Assert.AreEqual(firstTrackReference, queryResult.BestMatch.Track.TrackReference); Assert.AreEqual(9 + 8, queryResult.BestMatch.Similarity); Assert.AreEqual(1, queryResult.AnalyzedCandidatesCount); Assert.AreEqual(1, queryResult.ResultEntries.Count); }
protected MetaCompilation Compile(string filePath, bool assertEmptyDiagnostics = true, bool compileMetaModelCore = false) { MetaDescriptor.Initialize(); string text = File.ReadAllText(filePath); var st = MetaSyntaxTree.ParseText(text); BinderFlags binderFlags = BinderFlags.IgnoreAccessibility; if (compileMetaModelCore) { BinderFlags binderFlags2 = BinderFlags.IgnoreMetaLibraryDuplicatedTypes; binderFlags = binderFlags.UnionWith(binderFlags2); } MetaCompilationOptions options = new MetaCompilationOptions(MetaLanguage.Instance, OutputKind.NetModule, deterministic: true, concurrentBuild: false, topLevelBinderFlags: binderFlags); var compilation = MetaCompilation. Create("MetaModelCompilation"). AddSyntaxTrees(st). AddReferences( ModelReference.CreateFromModel(MetaInstance.MModel) ). WithOptions(options); compilation.ForceComplete(); if (assertEmptyDiagnostics) { AssertEmptyDiagnostics(compilation); } return(compilation); }
public void OnlyTracksWithGroupIdAreConsideredAsPotentialCandidatesTest() { long[] buckets = new long[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 }; var queryHash = new HashData(new byte[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 11 }, buckets); const int DefaultThreshold = 5; const int FirstTrackId = 20; const int FirstSubFingerprintId = 10; var firstTrackReference = new ModelReference <int>(FirstTrackId); SubFingerprintData firstResult = new SubFingerprintData( new byte[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 }, new ModelReference <int>(FirstSubFingerprintId), firstTrackReference); modelService.Setup( service => service.ReadSubFingerprintDataByHashBucketsThresholdWithGroupId(buckets, DefaultThreshold, "group-id")) .Returns(new List <SubFingerprintData> { firstResult }); modelService.Setup(service => service.ReadTrackByReference(firstTrackReference)).Returns( new TrackData { ISRC = "isrc", TrackReference = firstTrackReference }); var queryResult = queryFingerprintService.Query( new List <HashData> { queryHash }, new CustomQueryConfiguration { TrackGroupId = "group-id" }); Assert.IsTrue(queryResult.IsSuccessful); Assert.AreEqual("isrc", queryResult.BestMatch.Track.ISRC); Assert.AreEqual(firstTrackReference, queryResult.BestMatch.Track.TrackReference); }
public IModelReference AddTrack(TrackData track) { var trackReference = new ModelReference <int>(Interlocked.Increment(ref trackReferenceCounter)); Tracks[trackReference.Id] = track; return(track.TrackReference = trackReference); }
public void HammingSimilarityIsSummedUpAcrossAllSubFingerprintsTest() { var queryHash = new HashedFingerprint(GenericHashBuckets(), 0, 0, Array.Empty <byte>()); const int firstTrackId = 20; const int firstSubFingerprintId = 10; const int secondSubFingerprintId = 11; var firstTrackReference = new ModelReference <int>(firstTrackId); var firstResult = new SubFingerprintData(GenericHashBuckets(), 1, 0, new ModelReference <int>(firstSubFingerprintId), firstTrackReference); var secondResult = new SubFingerprintData(GenericHashBuckets(), 2, 0.928f, new ModelReference <int>(secondSubFingerprintId), firstTrackReference); var defaultQueryConfiguration = new DefaultQueryConfiguration(); modelService.Setup(service => service.Query( It.IsAny <Hashes>(), It.IsAny <QueryConfiguration>())).Returns(new[] { firstResult, secondResult }); modelService.Setup(service => service.ReadTracksByReferences(new[] { firstTrackReference })).Returns( new List <TrackData> { new TrackData("id", string.Empty, string.Empty, 0d, firstTrackReference) }); var hashes = new Hashes(new List <HashedFingerprint> { queryHash }, 1.48f, DateTime.Now, Enumerable.Empty <string>()); var queryResult = queryFingerprintService.Query(hashes, defaultQueryConfiguration, modelService.Object); Assert.IsTrue(queryResult.ContainsMatches); Assert.AreEqual("id", queryResult.BestMatch.Track.Id); Assert.AreEqual(firstTrackReference, queryResult.BestMatch.Track.TrackReference); Assert.AreEqual(200, queryResult.BestMatch.Score); Assert.AreEqual(1, queryResult.ResultEntries.Count()); }
public void ShouldAccumulateHammingDistanceSumForMultipleTracks() { const int CandidatesCount = 10000; var trackReference0 = new ModelReference<int>(0); var subFingerprints0 = GetSubFingerprintsForTrack(trackReference0, CandidatesCount); var trackReference1 = new ModelReference<int>(1); var subFingerprints1 = GetSubFingerprintsForTrack(trackReference1, CandidatesCount); var trackReference2 = new ModelReference<int>(2); var subFingerprints2 = GetSubFingerprintsForTrack(trackReference2, CandidatesCount); var allSubs = subFingerprints0.Concat(subFingerprints1).Concat(subFingerprints2); var acumulator = new ConcurrentDictionary<IModelReference, ResultEntryAccumulator>(); Parallel.ForEach( allSubs, new ParallelOptions { MaxDegreeOfParallelism = 10 }, sub => similarityUtility.AccumulateHammingSimilarity( new List<SubFingerprintData> { sub }, new HashedFingerprint(GenericSignature(), GenericHashBuckets(), 1, 0d, Enumerable.Empty<string>()), acumulator)); int expectedHammingSimilarity = (GenericSignature().Length * CandidatesCount) - CandidatesCount + 1; Assert.AreEqual(expectedHammingSimilarity, acumulator[trackReference0].HammingSimilaritySum); Assert.AreEqual(expectedHammingSimilarity, acumulator[trackReference1].HammingSimilaritySum); Assert.AreEqual(expectedHammingSimilarity, acumulator[trackReference2].HammingSimilaritySum); Assert.AreEqual(CandidatesCount, acumulator[trackReference0].BestMatch.SubFingerprint.SubFingerprintReference.Id); Assert.AreEqual(CandidatesCount, acumulator[trackReference1].BestMatch.SubFingerprint.SubFingerprintReference.Id); Assert.AreEqual(CandidatesCount, acumulator[trackReference2].BestMatch.SubFingerprint.SubFingerprintReference.Id); }
public void HammingSimilarityIsSummedUpAccrossAllSubFingerprintsTest() { var queryHash = new HashedFingerprint(GenericHashBuckets(), 0, 0, Enumerable.Empty <string>()); const int FirstTrackId = 20; const int FirstSubFingerprintId = 10; const int SecondSubFingerprintId = 11; var firstTrackReference = new ModelReference <int>(FirstTrackId); var firstResult = new SubFingerprintData(GenericHashBuckets(), 1, 0, new ModelReference <int>(FirstSubFingerprintId), firstTrackReference); var secondResult = new SubFingerprintData(GenericHashBuckets(), 2, 0.928f, new ModelReference <int>(SecondSubFingerprintId), firstTrackReference); var defaultQueryConfiguration = new DefaultQueryConfiguration(); modelService.Setup(service => service.SupportsBatchedSubFingerprintQuery).Returns(false); modelService.Setup(service => service.ReadSubFingerprints(It.IsAny <int[]>(), defaultQueryConfiguration)) .Returns(new List <SubFingerprintData> { firstResult, secondResult }); modelService.Setup(service => service.ReadTracksByReferences(new [] { firstTrackReference })) .Returns(new List <TrackData> { new TrackData { ISRC = "isrc", TrackReference = firstTrackReference } }); var queryResult = queryFingerprintService.Query(new List <HashedFingerprint> { queryHash }, defaultQueryConfiguration, modelService.Object); Assert.IsTrue(queryResult.ContainsMatches); Assert.AreEqual("isrc", queryResult.BestMatch.Track.ISRC); Assert.AreEqual(firstTrackReference, queryResult.BestMatch.Track.TrackReference); Assert.AreEqual(200, queryResult.BestMatch.HammingSimilaritySum); Assert.AreEqual(1, queryResult.ResultEntries.Count()); }
/// <inheritdoc /> public override BigQueryModel PatchModel(ModelReference modelReference, Model resource, PatchModelOptions options = null) { var request = CreatePatchModelRequest(modelReference, resource, options); var model = request.Execute(); return(new BigQueryModel(this, model)); }
public void Compile() { if (_compilation == null) { ImmutableModel coreModel = MetaInstance.MModel; string text = File.ReadAllText(_inputFilePath); var tree = MetaSyntaxTree.ParseText(text, path: _inputFilePath); BinderFlags binderFlags = BinderFlags.IgnoreAccessibility; if (_compileMetaModelCore) { BinderFlags binderFlags2 = BinderFlags.IgnoreMetaLibraryDuplicatedTypes; binderFlags = binderFlags.UnionWith(binderFlags2); } MetaCompilationOptions options = new MetaCompilationOptions(MetaLanguage.Instance, OutputKind.NetModule, deterministic: true, concurrentBuild: true, topLevelBinderFlags: binderFlags); //MetaCompilationOptions options = new MetaCompilationOptions(MetaLanguage.Instance, OutputKind.NetModule, deterministic: true, concurrentBuild: false); var compilation = MetaCompilation. Create("MetaModelCompilation"). AddSyntaxTrees(tree). AddReferences( ModelReference.CreateFromModel(coreModel) ). WithOptions(options); Interlocked.CompareExchange(ref _compilation, compilation, null); } _compilation.ForceComplete(); }
public IModelReference InsertTrack(TrackData track) { var trackReference = new ModelReference <int>(Interlocked.Increment(ref counter)); storage.Tracks[trackReference] = track; return(track.TrackReference = trackReference); }
/// <inheritdoc /> public override BigQueryModel GetModel(ModelReference modelReference, GetModelOptions options = null) { var request = CreateGetModelRequest(modelReference, options); var resource = request.Execute(); return(new BigQueryModel(this, resource)); }
public void ShouldInsertEntriesInThreadSafeManner() { var storage = new RAMStorage(50); var hashConverter = new HashConverter(); var hashes = Enumerable.Range(0, 100).Select(b => (byte)b).ToArray(); var longs = hashConverter.ToInts(hashes, 25); int tracksCount = 520; int subFingerprintsPerTrack = 33; float one = 8192f / 5512; Parallel.For(0, tracksCount, i => { var trackReference = new ModelReference <int>(i); for (int j = 0; j < subFingerprintsPerTrack; ++j) { var hashed = new HashedFingerprint(longs, (uint)j, j * one, Array.Empty <byte>()); storage.AddHashedFingerprint(hashed, trackReference); } }); for (int i = 0; i < 25; ++i) { var subFingerprints = storage.GetSubFingerprintsByHashTableAndHash(i, longs[i]); Assert.AreEqual(tracksCount * subFingerprintsPerTrack, subFingerprints.Count); } }
public void HammingSimilarityIsSummedUpAccrossAllSubFingerprintsTest() { long[] buckets = new long[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 }; var queryHash = new HashData(new byte[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 11 }, buckets); const int DefaultThreshold = 5; const int FirstTrackId = 20; const int FirstSubFingerprintId = 10; const int SecondSubFingerprintId = 11; var firstTrackReference = new ModelReference<int>(FirstTrackId); SubFingerprintData firstResult = new SubFingerprintData(new byte[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 }, new ModelReference<int>(FirstSubFingerprintId), firstTrackReference); SubFingerprintData secondResult = new SubFingerprintData(new byte[] { 1, 2, 3, 4, 5, 6, 7, 8, 10, 12 }, new ModelReference<int>(SecondSubFingerprintId), firstTrackReference); modelService.Setup(service => service.ReadSubFingerprintDataByHashBucketsWithThreshold(buckets, DefaultThreshold)) .Returns(new List<SubFingerprintData> { firstResult, secondResult }); modelService.Setup(service => service.ReadTrackByReference(firstTrackReference)) .Returns(new TrackData { ISRC = "isrc", TrackReference = firstTrackReference }); var queryResult = queryFingerprintService.Query(modelService.Object, new List<HashData> { queryHash }, new DefaultQueryConfiguration()); Assert.IsTrue(queryResult.IsSuccessful); Assert.AreEqual("isrc", queryResult.BestMatch.Track.ISRC); Assert.AreEqual(firstTrackReference, queryResult.BestMatch.Track.TrackReference); Assert.AreEqual(9 + 8, queryResult.BestMatch.Similarity); Assert.AreEqual(1, queryResult.AnalyzedCandidatesCount); Assert.AreEqual(1, queryResult.ResultEntries.Count); }
public void SameQueryHashGeneratesMultipleTrackMatches() { var groupedQueryResults = new GroupedQueryResults(10d, DateTime.Now); var random = new Random(1); int runs = 100; int[] counts = new int[runs]; var trackRef = new ModelReference <uint>(1); int k = 0; Parallel.For(0, runs, i => { counts[i] = random.Next(5, 10); var queryPoint = new HashedFingerprint(new int[25], (uint)i, i * 1.48f, Array.Empty <byte>()); for (int j = 0; j < counts[i]; ++j) { var dbPoint = new SubFingerprintData(new int[25], (uint)k, k * 0.01f, new ModelReference <uint>((uint)Interlocked.Increment(ref k)), trackRef); groupedQueryResults.Add(queryPoint, dbPoint, i); } }); var allMatches = groupedQueryResults.GetMatchesForTrack(trackRef).ToList(); Assert.AreEqual(counts.Sum(), allMatches.Count); Assert.AreEqual(runs, allMatches.Select(m => m.QuerySequenceNumber).Distinct().Count()); }
public void DistributionOfHashesHasToBeUniform() { var lshAlgorithm = LocalitySensitiveHashingAlgorithm.Instance; var random = new Random(); var storage = new RAMStorage(25); float one = 8192f / 5512; var config = new DefaultHashingConfig { NumberOfLSHTables = 25, NumberOfMinHashesPerTable = 4, HashBuckets = 0 }; var track = new ModelReference <int>(1); int l = 100000; for (int i = 0; i < l; ++i) { var schema = TestUtilities.GenerateRandomFingerprint(random, 200, 128, 32); var hash = lshAlgorithm.Hash(new Fingerprint(schema, i * one, (uint)i, Array.Empty <byte>()), config); var subFingerprint = new SubFingerprintData(hash.HashBins, hash.SequenceNumber, hash.StartsAt, new ModelReference <uint>((uint)i), track); storage.AddSubFingerprint(subFingerprint); } var distribution = storage.HashCountsPerTable; foreach (var hashPerTable in distribution) { double collisions = (double)(l - hashPerTable) / l; Assert.IsTrue(collisions <= 0.01d, $"Less than 1% of collisions across 100K hashes: {collisions}"); } }
public void HammingSimilarityIsSummedUpAccrossAllSubFingerprintsTest() { var queryHash = new HashedFingerprint(GenericSignature(), GenericHashBuckets(), 0, 0, Enumerable.Empty<string>()); const int FirstTrackId = 20; const int FirstSubFingerprintId = 10; const int SecondSubFingerprintId = 11; var firstTrackReference = new ModelReference<int>(FirstTrackId); var firstResult = new SubFingerprintData(GenericHashBuckets(), 1, 0, new ModelReference<int>(FirstSubFingerprintId), firstTrackReference); var secondResult = new SubFingerprintData(GenericHashBuckets(), 2, 0.928, new ModelReference<int>(SecondSubFingerprintId), firstTrackReference); var defaultQueryConfiguration = new DefaultQueryConfiguration(); modelService.Setup(service => service.SupportsBatchedSubFingerprintQuery).Returns(false); modelService.Setup(service => service.ReadSubFingerprints(It.IsAny<long[]>(), defaultQueryConfiguration)) .Returns(new List<SubFingerprintData> { firstResult, secondResult }); modelService.Setup(service => service.ReadTrackByReference(firstTrackReference)) .Returns(new TrackData { ISRC = "isrc", TrackReference = firstTrackReference }); var queryResult = queryFingerprintService.Query(new List<HashedFingerprint> { queryHash }, defaultQueryConfiguration, modelService.Object); Assert.IsTrue(queryResult.ContainsMatches); Assert.AreEqual("isrc", queryResult.BestMatch.Track.ISRC); Assert.AreEqual(firstTrackReference, queryResult.BestMatch.Track.TrackReference); Assert.AreEqual(GenericSignature().Length * 2, queryResult.BestMatch.HammingSimilaritySum); Assert.AreEqual(1, queryResult.ResultEntries.Count()); }
public void FingerprintsCantMatchUniformlyAtRandom() { var lshAlgorithm = LocalitySensitiveHashingAlgorithm.Instance; var random = new Random(); var storage = new RAMStorage(25); float one = 8192f / 5512; var config = new DefaultHashingConfig { NumberOfLSHTables = 25, NumberOfMinHashesPerTable = 4, HashBuckets = 0 }; var track = new ModelReference <int>(1); for (int i = 0; i < 100; ++i) { var schema = TestUtilities.GenerateRandomFingerprint(random, 200, 128, 32); var hash = lshAlgorithm.Hash(new Fingerprint(schema, i * one, (uint)i, Array.Empty <byte>()), config); var subFingerprint = new SubFingerprintData(hash.HashBins, hash.SequenceNumber, hash.StartsAt, new ModelReference <uint>((uint)i), track); storage.AddSubFingerprint(subFingerprint); } for (int i = 0; i < 10; ++i) { var schema = TestUtilities.GenerateRandomFingerprint(random, 200, 128, 32); var hash = lshAlgorithm.Hash(new Fingerprint(schema, i * one, (uint)i, Array.Empty <byte>()), config); for (int j = 0; j < 25; ++j) { var ids = storage.GetSubFingerprintsByHashTableAndHash(j, hash.HashBins[j]); Assert.IsFalse(ids.Any()); } } }
/// <inheritdoc /> public override async Task <BigQueryModel> PatchModelAsync(ModelReference modelReference, Model resource, PatchModelOptions options = null, CancellationToken cancellationToken = default) { var request = CreatePatchModelRequest(modelReference, resource, options); var model = await request.ExecuteAsync().ConfigureAwait(false); return(new BigQueryModel(this, model)); }
public void ShouldSerializeSpectralImages() { var spectrumService = new SpectrumService(new LomontFFT(), new LogUtility()); var spectrums = spectrumService.CreateLogSpectrogram(GetAudioSamples(), new DefaultSpectrogramConfig()) .Select(spectrum => spectrum.ImageRowCols) .ToList(); var modelService = new InMemoryModelService(); var trackReference = new ModelReference <int>(10); modelService.InsertSpectralImages(spectrums, trackReference); var tempFile = Path.GetTempFileName(); modelService.Snapshot(tempFile); var fromFileService = new InMemoryModelService(tempFile); File.Delete(tempFile); var allSpectrums = fromFileService.GetSpectralImagesByTrackReference(trackReference).ToList(); Assert.AreEqual(spectrums.Count, allSpectrums.Count); }
private static void SetModel(Entity entity, ModelReference model) { entity.MetaData.Set(ModelMetaKey, model); entity.MetaData.Set(ModelNameMetaKey, GetModelName(entity)); entity.MetaData.Set(ModelBoundingBoxMetaKey, model.Model.GetBoundingBox()); entity.UpdateBoundingBox(); }
/// <summary> /// Stores the fingerprints of the given file into the ModelService. /// Uses HighPrecisionFingerprintConfiguration. /// </summary> /// <param name="waveFile"></param> private void StoreNewAudioFileData(IAudioFile waveFile) { TrackData track = new TrackData( isrc: "TD100INPROG", // International Standart Recording Code. artist: "The TD's", title: waveFile.FilePath.NameWithoutExtension, album: waveFile.FilePath.DirectoryPath, releaseYear: DateTime.Today.Year, length: waveFile.DurationInSeconds); // Stores track metadata in the datasource. //TRack trackReference = new TrackReference<int>(mModelService.InsertTrack(track).Id); IModelReference trackReference = mModelService.InsertTrack(track); ModelReference <int> trackReference2 = trackReference as ModelReference <int>; // Creates hashed fingerprints. List <HashedFingerprint> hashedFingerprints = FingerprintCommandBuilder.Instance .BuildFingerprintCommand() .From(waveFile.FilePath.FileFullPath) .WithFingerprintConfig(new DefaultFingerprintConfiguration()) .UsingServices(mAudioService) .Hash() .Result; // Stores hashes in the database. //SaveFingerPrintsInMemory(hashedFingerprints, trackReference2, waveFile.FilePath.NameWithoutExtension); // Currenntly not using fp. mModelService.InsertHashDataForTrack(hashedFingerprints, trackReference); mLogger.WriteLine($"Stored {hashedFingerprints.Count} hashed fingerprints from {waveFile.FilePath.Name}"); }
public void ShouldReturnEmptySinceNoSpectralImagesArePresentForTrack() { var trackReference = new ModelReference<int>(10); var results = spectralImageDao.GetSpectralImagesByTrackReference(trackReference); Assert.IsEmpty(results); }
private DeleteRequest CreateDeleteModelRequest(ModelReference modelReference, DeleteModelOptions options) { GaxPreconditions.CheckNotNull(modelReference, nameof(modelReference)); var request = Service.Models.Delete(modelReference.ProjectId, modelReference.DatasetId, modelReference.ModelId); options?.ModifyRequest(request); return(request); }
public void ShouldReturnEmptySinceNoSpectralImagesArePresentForTrack() { var trackReference = new ModelReference <int>(10); var results = spectralImageDao.GetSpectralImagesByTrackReference(trackReference); Assert.IsEmpty(results); }
public void MaximumNumberOfReturnedTracksIsLessThanAnalyzedCandidatesResultsTest() { long[] buckets = new long[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 }; var queryHash = new HashData(new byte[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 11 }, buckets); const int DefaultThreshold = 5; const int FirstTrackId = 20; const int SecondTrackId = 21; const int ThirdTrackId = 22; const int FirstSubFingerprintId = 10; const int SecondSubFingerprintId = 11; var firstTrackReference = new ModelReference <int>(FirstTrackId); var thirdTrackReference = new ModelReference <int>(ThirdTrackId); SubFingerprintData firstResult = new SubFingerprintData( new byte[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 }, new ModelReference <int>(FirstSubFingerprintId), firstTrackReference); SubFingerprintData secondResult = new SubFingerprintData( new byte[] { 11, 2, 13, 4, 15, 6, 7, 8, 10, 12 }, new ModelReference <int>(SecondSubFingerprintId), new ModelReference <int>(SecondTrackId)); SubFingerprintData thirdResult = new SubFingerprintData( new byte[] { 1, 2, 3, 4, 5, 15, 7, 8, 10, 12 }, new ModelReference <int>(SecondSubFingerprintId), new ModelReference <int>(ThirdTrackId)); modelService.Setup( service => service.ReadSubFingerprintDataByHashBucketsWithThreshold(buckets, DefaultThreshold)).Returns( new List <SubFingerprintData> { firstResult, secondResult, thirdResult }); modelService.Setup(service => service.ReadTrackByReference(firstTrackReference)).Returns( new TrackData { ISRC = "isrc", TrackReference = firstTrackReference }); modelService.Setup(service => service.ReadTrackByReference(thirdTrackReference)).Returns( new TrackData { ISRC = "isrc_2", TrackReference = thirdTrackReference }); var queryResult = queryFingerprintService.Query( modelService.Object, new List <HashData> { queryHash }, new CustomQueryConfiguration { MaximumNumberOfTracksToReturnAsResult = 2, ThresholdVotes = DefaultThreshold }); Assert.IsTrue(queryResult.IsSuccessful); Assert.AreEqual("isrc", queryResult.BestMatch.Track.ISRC); Assert.AreEqual(firstTrackReference, queryResult.BestMatch.Track.TrackReference); Assert.AreEqual(9, queryResult.BestMatch.Similarity); Assert.AreEqual(3, queryResult.AnalyzedCandidatesCount); Assert.AreEqual(2, queryResult.ResultEntries.Count); Assert.AreEqual(firstTrackReference, queryResult.ResultEntries[0].Track.TrackReference); Assert.AreEqual(thirdTrackReference, queryResult.ResultEntries[1].Track.TrackReference); }
public void LevelChanged(MsdLevelChangeType ChangeType, Level TheLevel, ModelReference TheModel) { if (MsdLevelChangeType.AfterCreate == ChangeType || MsdLevelChangeType.AfterDelete == ChangeType || MsdLevelChangeType.ChangeName == ChangeType) { PopulateLevelList(); } }
public void MaximumNumberOfReturnedTracksIsLessThanAnalyzedCandidatesResultsTest() { long[] buckets = new long[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 }; var queryHash = new HashedFingerprint(new byte[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 11 }, buckets, 1, 0); const int DefaultThreshold = 5; const int FirstTrackId = 20; const int SecondTrackId = 21; const int ThirdTrackId = 22; const int FirstSubFingerprintId = 10; const int SecondSubFingerprintId = 11; var firstTrackReference = new ModelReference<int>(FirstTrackId); var thirdTrackReference = new ModelReference<int>(ThirdTrackId); var firstResult = new SubFingerprintData( new byte[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 }, 1, 0, new ModelReference<int>(FirstSubFingerprintId), firstTrackReference); SubFingerprintData secondResult = new SubFingerprintData( new byte[] { 11, 2, 13, 4, 15, 6, 7, 8, 10, 12 }, 2, 0.928, new ModelReference<int>(SecondSubFingerprintId), new ModelReference<int>(SecondTrackId)); SubFingerprintData thirdResult = new SubFingerprintData( new byte[] { 1, 2, 3, 4, 5, 15, 7, 8, 10, 12 }, 3, 0.928 * 2, new ModelReference<int>(SecondSubFingerprintId), new ModelReference<int>(ThirdTrackId)); modelService.Setup( service => service.ReadSubFingerprintDataByHashBucketsWithThreshold(buckets, DefaultThreshold)).Returns( new List<SubFingerprintData> { firstResult, secondResult, thirdResult }); modelService.Setup(service => service.ReadTrackByReference(firstTrackReference)).Returns( new TrackData { ISRC = "isrc", TrackReference = firstTrackReference }); modelService.Setup(service => service.ReadTrackByReference(thirdTrackReference)).Returns( new TrackData { ISRC = "isrc_2", TrackReference = thirdTrackReference }); var queryResult = queryFingerprintService.Query( modelService.Object, new List<HashedFingerprint> { queryHash }, new CustomQueryConfiguration { MaximumNumberOfTracksToReturnAsResult = 2, ThresholdVotes = DefaultThreshold }); Assert.IsTrue(queryResult.IsSuccessful); Assert.AreEqual("isrc", queryResult.BestMatch.Track.ISRC); Assert.AreEqual(firstTrackReference, queryResult.BestMatch.Track.TrackReference); Assert.AreEqual(9, queryResult.BestMatch.Similarity); Assert.AreEqual(3, queryResult.AnalyzedCandidatesCount); Assert.AreEqual(2, queryResult.ResultEntries.Count); Assert.AreEqual(firstTrackReference, queryResult.ResultEntries[0].Track.TrackReference); Assert.AreEqual(thirdTrackReference, queryResult.ResultEntries[1].Track.TrackReference); }
private PatchRequest CreatePatchModelRequest(ModelReference modelReference, Model resource, PatchModelOptions options) { GaxPreconditions.CheckNotNull(modelReference, nameof(modelReference)); GaxPreconditions.CheckNotNull(resource, nameof(resource)); var request = Service.Models.Patch(resource, modelReference.ProjectId, modelReference.DatasetId, modelReference.ModelId); options?.ModifyRequest(request); RetryIfETagPresent(request, resource); return(request); }
private GetRequest CreateGetModelRequest(ModelReference modelReference, GetModelOptions options) { GaxPreconditions.CheckNotNull(modelReference, nameof(modelReference)); var request = Service.Models.Get(modelReference.ProjectId, modelReference.DatasetId, modelReference.ModelId); options?.ModifyRequest(request); RetryHandler.MarkAsRetriable(request); return(request); }
public static String GenererReference(String element) { ModelReference model = SearchModelReference(element); if ((model != null) ? model.Id > 0 : false) { return(ReferenceElement(model)); } Messages.ShowErreur("Cet élément n'a pas de modele de reference!"); return(""); }
/// <summary> /// Finds a model reference (0x14) with the given name /// </summary> /// <param name="modelName">The model actor name</param> /// <param name="modelReference">The out reference parameter</param> /// <returns>Whether or not we have found the model</returns> private bool FindModelReference(string modelName, out ModelReference modelReference) { if (!_fragmentNameDictionary.ContainsKey(modelName)) { modelReference = null; return(false); } modelReference = _fragmentNameDictionary[modelName] as ModelReference; return(true); }
public void DeleteOneTrackTest() { var trackReference = new ModelReference <int>(101); var track = GetTrack(trackReference); trackDao.InsertTrack(track); trackDao.DeleteTrack(trackReference); Assert.IsEmpty(trackDao.ReadTracksByReferences(new [] { trackReference })); }
public void ReadTrackByIdTest() { var trackReference = new ModelReference <int>(101); var expectedTrack = GetTrack(trackReference, 10); trackDao.InsertTrack(expectedTrack); TrackData actualTrack = trackDao.ReadTrackById(expectedTrack.Id); AssertTracksAreEqual(expectedTrack, actualTrack); }
public int Insert(TrackData track) { int id = PrepareStoredProcedure(SpInsertTrack) .WithParametersFromModel(track) .Execute() .AsScalar <int>(); var trackReference = new ModelReference <int>(id); track.TrackReference = trackReference; return(id); }
public IModelReference InsertSubFingerprint(byte[] signature, int sequenceNumber, double sequenceAt, IModelReference trackReference) { var subFingerprintReference = new ModelReference<long>(Interlocked.Increment(ref counter)); storage.SubFingerprints[subFingerprintReference] = new SubFingerprintData(signature, sequenceNumber, sequenceAt, subFingerprintReference, trackReference); if (!storage.TracksHashes.ContainsKey(trackReference)) { storage.TracksHashes[trackReference] = new ConcurrentDictionary<IModelReference, HashedFingerprint>(); } storage.TracksHashes[trackReference][subFingerprintReference] = new HashedFingerprint(signature, null, sequenceNumber, sequenceAt); return subFingerprintReference; }
public void ShouldSerializeModelReference() { var @ref = new ModelReference <int>(42); using var stream = new MemoryStream(); Serializer.SerializeWithLengthPrefix <IModelReference>(stream, @ref, PrefixStyle.Fixed32); byte[] serialized = stream.ToArray(); using var deserializedStream = new MemoryStream(serialized); var reference = Serializer.DeserializeWithLengthPrefix <IModelReference>(deserializedStream, PrefixStyle.Fixed32); Assert.NotNull(reference); }
public IModelReference InsertSubFingerprint(byte[] signature, IModelReference trackReference) { var subFingerprintReference = new ModelReference<long>(Interlocked.Increment(ref counter)); storage.SubFingerprints[subFingerprintReference] = new SubFingerprintData(signature, subFingerprintReference, trackReference); if (!storage.TracksHashes.ContainsKey(trackReference)) { storage.TracksHashes[trackReference] = new ConcurrentDictionary<IModelReference, HashData>(); } storage.TracksHashes[trackReference][subFingerprintReference] = new HashData { SubFingerprint = signature }; return subFingerprintReference; }
public void MaximumNumberOfReturnedTracksIsLessThanAnalyzedCandidatesResultsTest() { var queryHash = new HashedFingerprint(GenericSignature(), GenericHashBuckets(), 1, 0, Enumerable.Empty<string>()); const int DefaultThreshold = 5; const int FirstTrackId = 20; const int SecondTrackId = 21; const int ThirdTrackId = 22; const int FirstSubFingerprintId = 10; const int SecondSubFingerprintId = 11; var firstTrackReference = new ModelReference<int>(FirstTrackId); var secondTrackReference = new ModelReference<int>(SecondTrackId); var firstResult = new SubFingerprintData( GenericHashBuckets(), 1, 0, new ModelReference<int>(FirstSubFingerprintId), firstTrackReference); var secondResult = new SubFingerprintData( GenericHashBuckets(), 2, 0.928, new ModelReference<int>(SecondSubFingerprintId), secondTrackReference); var thirdResult = new SubFingerprintData( GenericHashBuckets(), 3, 0.928 * 2, new ModelReference<int>(SecondSubFingerprintId), new ModelReference<int>(ThirdTrackId)); var customQueryConfiguration = new DefaultQueryConfiguration { MaxTracksToReturn = 2, ThresholdVotes = DefaultThreshold }; modelService.Setup(service => service.SupportsBatchedSubFingerprintQuery).Returns(false); modelService.Setup( service => service.ReadSubFingerprints(It.IsAny<long[]>(), customQueryConfiguration)).Returns( new List<SubFingerprintData> { firstResult, secondResult, thirdResult }); modelService.Setup(service => service.ReadTrackByReference(firstTrackReference)).Returns( new TrackData { ISRC = "isrc", TrackReference = firstTrackReference }); modelService.Setup(service => service.ReadTrackByReference(secondTrackReference)).Returns( new TrackData { ISRC = "isrc_1", TrackReference = secondTrackReference }); var queryResult = queryFingerprintService.Query(new List<HashedFingerprint> { queryHash }, customQueryConfiguration, modelService.Object); Assert.IsTrue(queryResult.ContainsMatches); Assert.AreEqual("isrc", queryResult.BestMatch.Track.ISRC); Assert.AreEqual(firstTrackReference, queryResult.BestMatch.Track.TrackReference); Assert.AreEqual(50, queryResult.BestMatch.HammingSimilaritySum); Assert.AreEqual(2, queryResult.ResultEntries.Count()); var results = queryResult.ResultEntries.ToList(); Assert.AreEqual(firstTrackReference, results[0].Track.TrackReference); Assert.AreEqual(secondTrackReference, results[1].Track.TrackReference); }
public void ShouldAccumulateHammingDistanceSum() { const int CandidatesCount = 5; var trackReference = new ModelReference<int>(0); var subFingerprints = GetSubFingerprintsForTrack(trackReference, CandidatesCount); var acumulator = new ConcurrentDictionary<IModelReference, ResultEntryAccumulator>(); similarityUtility.AccumulateHammingSimilarity(subFingerprints, new HashedFingerprint(GenericSignature(), GenericHashBuckets(), 1, 0d, Enumerable.Empty<string>()), acumulator); int expectedHammingSimilaritySum = (GenericSignature().Length * CandidatesCount) - CandidatesCount + 1; Assert.AreEqual(expectedHammingSimilaritySum, acumulator[trackReference].HammingSimilaritySum); Assert.AreEqual(CandidatesCount, acumulator[trackReference].BestMatch.SubFingerprint.SubFingerprintReference.Id); }
public void ShouldInsertSpectralImages() { var images = new List<float[]>() { new float[0], new float[0], new float[0] }; var trackReference = new ModelReference<int>(10); spectralImageDao.InsertSpectralImages(images, trackReference); Assert.AreEqual(3, spectralImageDao.GetSpectralImagesByTrackReference(trackReference).Count()); var ids = spectralImageDao.GetSpectralImagesByTrackReference(trackReference) .Select(dto => (int)dto.SpectralImageReference.Id) .ToList(); CollectionAssert.AreEqual(Enumerable.Range(1, 3), ids); }
public void ShouldGetBestCandidatesByHammingDistance() { var modelService = new Mock<IModelService>(MockBehavior.Strict); var trackReference = new ModelReference<int>(3); modelService.Setup(s => s.ReadTrackByReference(trackReference)).Returns( new TrackData { ISRC = "isrc-1234-1234" }); var queryConfiguration = new DefaultQueryConfiguration { MaxTracksToReturn = 1 }; var query = new List<HashedFingerprint> { new HashedFingerprint(null, null, 1, 0d, Enumerable.Empty<string>()), new HashedFingerprint(null, null, 1, 4d, Enumerable.Empty<string>()), new HashedFingerprint(null, null, 1, 8d, Enumerable.Empty<string>()) }; var first = new ResultEntryAccumulator(query[0], new SubFingerprintData(null, 1, 0d, null, null), 100); var second = new ResultEntryAccumulator(query[1], new SubFingerprintData(null, 1, 4d, null, null), 99); var third = new ResultEntryAccumulator(query[2], new SubFingerprintData(null, 1, 8d, null, null), 101); var hammingSimilarties = new Dictionary<IModelReference, ResultEntryAccumulator> { { new ModelReference<int>(1), first }, { new ModelReference<int>(2), second }, { new ModelReference<int>(3), third }, }; var best = queryMath.GetBestCandidates( query, hammingSimilarties, queryConfiguration.MaxTracksToReturn, modelService.Object, queryConfiguration.FingerprintConfiguration); Assert.AreEqual(1, best.Count); Assert.AreEqual("isrc-1234-1234", best[0].Track.ISRC); Assert.AreEqual(9.48d, best[0].QueryLength, 0.01); Assert.AreEqual(0d, best[0].TrackStartsAt); modelService.VerifyAll(); }
public void ShouldSortMatchesProperly() { List<SubFingerprintData> subFingerprints = new List<SubFingerprintData>(); var trackReference = new ModelReference<int>(0); const double OneFingerprintLength = 1.0d; const int CandidatesCount = 20; for (int i = 0; i < CandidatesCount; ++i) { var sub = new SubFingerprintData(GenericHashBuckets(), i, OneFingerprintLength * (CandidatesCount - i), new ModelReference<int>(i), trackReference); subFingerprints.Add(sub); } var acumulator = new ConcurrentDictionary<IModelReference, ResultEntryAccumulator>(); similarityUtility.AccumulateHammingSimilarity(subFingerprints, new HashedFingerprint(GenericSignature(), GenericHashBuckets(), 1, 0d, Enumerable.Empty<string>()), acumulator); var expected = Enumerable.Range(1, 20); var actual = acumulator[trackReference].Matches.Select(m => m.SubFingerprint.SequenceAt).ToList(); CollectionAssert.AreEqual(expected, actual); }
public void RenderModel(WLD modelWld,ModelReference model,int textureNumber=0,int headNumber=0,int face=0) { var wld = modelWld; TrackAnimationBuilder animation = null; List<MeshReference> meshrefs = new List<MeshReference>(); foreach (var refs in model.References) { var meshref = wld.MeshReferences.Where(x => x.FragmentNumber == refs).FirstOrDefault(); if (meshref != null) { meshrefs.Add(meshref); continue; } var skel = wld.SkeletonTrackReferences.Where(x => x.FragmentNumber == refs).FirstOrDefault(); if (skel != null) { var skelset = wld.SkeletonTrackSet.Where(x => x.FragmentNumber == skel.SkeletonTrackSetReference).FirstOrDefault(); if (skelset != null) { animation = new TrackAnimationBuilder(skelset, wld); foreach (var ms in skelset.MeshReferences) { var m = wld.MeshReferences.Where(x => x.FragmentNumber == ms).FirstOrDefault(); if (m != null) meshrefs.Add(m); } } } } List<Mesh> meshes = new List<Mesh>(); foreach (var m in meshrefs) { var mesh = wld.ZoneMeshes.FirstOrDefault(x => x.FragmentNumber == m.FragmentReference); if (headNumber > 0) { //see if it's a head mesh int pos = mesh.FragmentName.IndexOf("HE00"); if (pos > 0) { string hNum = String.Format("HE{0:d2}", headNumber); string hMeshName = mesh.FragmentName.Substring(0, pos) + hNum + mesh.FragmentName.Substring(pos + hNum.Length); var hMesh = wld.ZoneMeshes.FirstOrDefault(x => x.FragmentName == hMeshName); if (hMesh != null) mesh = hMesh; } } if (mesh != null) meshes.Add(mesh); } if (meshes.Count > 0) { RenderMesh(wld,meshes, animation, textureNumber,face); } }
public static WLD Load(Stream stream,string name="default.wld") { WLD wld = new WLD(); wld.Name = name; int size = Marshal.SizeOf(typeof(WLDHeader)); var barray = new byte[size]; stream.Read(barray, 0, size); var header = Functions.ByteArrayToStructure<WLDHeader>(barray); if (header.Magic != 0x54503d02) { throw new Exception("Invalid file format"); } if (header.Version == 0x00015500) { wld._format = Format.Old; } else if (header.Version == 0x1000C800) { wld._format = Format.New; } else { throw new Exception("Unknown file version"); } //var shash = stream.Position; barray = new byte[header.StringHashSize]; stream.Read(barray, 0, (int)header.StringHashSize); wld._sHash = WLD.DecodeFileName(barray); wld._strings = wld._sHash.Split('\0'); var fragCount = header.FragmentCount; stream.Seek(size + header.StringHashSize, SeekOrigin.Begin); int fragSize = Marshal.SizeOf(typeof(BasicWLDFragment)); for (int i = 0; i < header.FragmentCount; i++) { barray = new byte[fragSize]; stream.Read(barray, 0, fragSize); var fragment = Functions.ByteArrayToStructure<BasicWLDFragment>(barray); int nameRef = (int)fragment.NameRef; var position = stream.Position; switch (fragment.Id) { case 0x03: var bmpname = new BitmapName(i, nameRef); bmpname.Handler(stream); wld._fragments.Add(bmpname); break; case 0x04: var binfo = new BitmapInfo(i, nameRef); binfo.Handler(stream); wld._fragments.Add(binfo); break; case 0x05: var bitmapInfoRef = new BitmapInfoReference(i, nameRef); bitmapInfoRef.Handler(stream); wld._fragments.Add(bitmapInfoRef); break; case 0x09: break; case 0x10: var skelset = new SkeletonTrackSet(i, nameRef); skelset.Handler(stream); skelset.FragmentName = wld.GetStringAtHashIndex(0-skelset.FragmentNameRef); wld._fragments.Add(skelset); break; case 0x11: var skeltrackRef = new SkeletonTrackReference(i, nameRef); skeltrackRef.Handler(stream); wld._fragments.Add(skeltrackRef); break; case 0x12: var skelpiece = new SkeletonPieceTrack(i, nameRef); skelpiece.Handler(stream); wld._fragments.Add(skelpiece); break; case 0x13: var skelpref = new SkeletonPieceTrackReference(i, nameRef); skelpref.Handler(stream); //skelpref.FragmentName = wld.GetStringAtHashIndex(0 - skelpref.FragmentNameRef); wld._fragments.Add(skelpref); break; case 0x14: var modelref = new ModelReference(i, nameRef); modelref.Handler(stream); modelref.FragmentName = wld.GetStringAtHashIndex(0 - modelref.FragmentNameRef); wld._fragments.Add(modelref); modelref.MagicString = wld.GetStringAtHashIndex(modelref.MagicStringRef); break; case 0x15: var objlocation = new ObjectLocation(i, nameRef); objlocation.Handler(stream); wld._fragments.Add(objlocation); break; case 0x22: //num_0x22++; break; case 0x2d: var meshref = new MeshReference(i, nameRef); meshref.Handler(stream); wld._fragments.Add(meshref); break; case 0x31: var tlist = new TextureList(i, nameRef); tlist.Handler(stream); wld._fragments.Add(tlist); break; case 0x30: var texture = new Texture(i, nameRef); texture.Handler(stream); wld._fragments.Add(texture); break; // Grab the number of vertices and polygons case 0x36: var mesh = new Mesh(i, nameRef); mesh.Handler(stream); wld._fragments.Add(mesh); break; } stream.Seek(position + fragment.Size - 4, SeekOrigin.Begin); } return wld; }
public void OnlyTracksWithGroupIdAreConsideredAsPotentialCandidatesTest() { long[] buckets = new long[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 }; var queryHash = new HashedFingerprint(new byte[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 11 }, buckets, 0, 0); const int DefaultThreshold = 5; const int FirstTrackId = 20; const int FirstSubFingerprintId = 10; var firstTrackReference = new ModelReference<int>(FirstTrackId); SubFingerprintData firstResult = new SubFingerprintData( new byte[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10 }, 1, 0.928, new ModelReference<int>(FirstSubFingerprintId), firstTrackReference); modelService.Setup( service => service.ReadSubFingerprintDataByHashBucketsThresholdWithGroupId(buckets, DefaultThreshold, "group-id")) .Returns(new List<SubFingerprintData> { firstResult }); modelService.Setup(service => service.ReadTrackByReference(firstTrackReference)).Returns( new TrackData { ISRC = "isrc", TrackReference = firstTrackReference }); var queryResult = queryFingerprintService.Query( modelService.Object, new List<HashedFingerprint> { queryHash }, new CustomQueryConfiguration { TrackGroupId = "group-id" }); Assert.IsTrue(queryResult.IsSuccessful); Assert.AreEqual("isrc", queryResult.BestMatch.Track.ISRC); Assert.AreEqual(firstTrackReference, queryResult.BestMatch.Track.TrackReference); }
private void InsertSubFingerprint(HashedFingerprint hashedFingerprint, IModelReference trackReference) { var subFingerprintReference = new ModelReference<long>(Interlocked.Increment(ref counter)); storage.SubFingerprints[subFingerprintReference] = new SubFingerprintData( hashedFingerprint.HashBins, hashedFingerprint.SequenceNumber, hashedFingerprint.StartsAt, subFingerprintReference, trackReference) { Clusters = hashedFingerprint.Clusters }; if (!storage.TracksHashes.ContainsKey(trackReference)) { storage.TracksHashes[trackReference] = new ConcurrentDictionary<IModelReference, HashedFingerprint>(); } storage.TracksHashes[trackReference][subFingerprintReference] = hashedFingerprint; this.InsertHashes(hashedFingerprint.HashBins, subFingerprintReference); }
private IEnumerable<SubFingerprintData> GetSubFingerprintsForTrack(ModelReference<int> trackReference, int candidatesCount) { List<SubFingerprintData> subFingerprints = new List<SubFingerprintData>(); const double OneFingerprintLength = 0.256; for (int i = 0; i < candidatesCount - 1; ++i) { var sub = new SubFingerprintData(GenericHashBuckets(), i, OneFingerprintLength * i, new ModelReference<int>(i), trackReference); sub.Hashes[0] = 0; subFingerprints.Add(sub); } subFingerprints.Add(new SubFingerprintData(GenericHashBuckets(), candidatesCount - 1, OneFingerprintLength * candidatesCount, new ModelReference<int>(candidatesCount), trackReference)); return subFingerprints; }
public IModelReference InsertTrack(TrackData track) { var trackReference = new ModelReference<int>(Interlocked.Increment(ref counter)); storage.Tracks[trackReference] = track; return track.TrackReference = trackReference; }
public void RenderModel(ModelReference model,int textureNumber,int headNumber=0) { if (model == null) return; _display3d.RenderModel(model, textureNumber, headNumber); }