private void NormalizeAudioIfNecessary(AudioSamples samples, FingerprintConfiguration configuration)
{
if (configuration.NormalizeSignal)
{
audioSamplesNormalizer.NormalizeInPlace(samples.Samples);
}
}
public List <Fingerprint> CreateFingerprints(AudioSamples samples, FingerprintConfiguration configuration)
{
NormalizeAudioIfNecessary(samples, configuration);
var spectrum = spectrumService.CreateLogSpectrogram(samples, configuration.SpectrogramConfig);
return(CreateFingerprintsFromLogSpectrum(spectrum, configuration));
}
private ResultEntry GetResultEntry(FingerprintConfiguration configuration, TrackData track, ResultEntryAccumulator acc, double queryLength)
{
var coverage = queryResultCoverageCalculator.GetCoverage(
acc.Matches,
queryLength,
configuration);
double confidence = confidenceCalculator.CalculateConfidence(
coverage.SourceMatchStartsAt,
coverage.SourceMatchLength,
queryLength,
coverage.OriginMatchStartsAt,
track.Length);
return(new ResultEntry(
track,
coverage.SourceMatchStartsAt,
coverage.SourceMatchLength,
coverage.OriginMatchStartsAt,
GetTrackStartsAt(acc.BestMatch),
confidence,
acc.HammingSimilaritySum,
queryLength,
acc.BestMatch));
}
private ResultEntry GetResultEntry(IModelService modelService, FingerprintConfiguration configuration, KeyValuePair <IModelReference, ResultEntryAccumulator> pair, double queryLength)
{
var track = modelService.ReadTrackByReference(pair.Key);
var coverage = queryResultCoverageCalculator.GetCoverage(
pair.Value.Matches,
queryLength,
configuration);
double confidence = confidenceCalculator.CalculateConfidence(
coverage.SourceMatchStartsAt,
coverage.SourceMatchLength,
queryLength,
coverage.OriginMatchStartsAt,
track.Length);
return(new ResultEntry(
track,
coverage.SourceMatchStartsAt,
coverage.SourceMatchLength,
coverage.OriginMatchStartsAt,
GetTrackStartsAt(pair.Value.BestMatch),
confidence,
pair.Value.HammingSimilaritySum,
queryLength,
pair.Value.BestMatch));
}
public List <HashedFingerprint> CreateFingerprints(AudioSamples samples, FingerprintConfiguration configuration)
{
var spectrum = spectrumService.CreateLogSpectrogram(samples, configuration.SpectrogramConfig);
var fingerprints = CreateFingerprintsFromLogSpectrum(spectrum, configuration);
return(HashFingerprints(fingerprints, configuration));
}
public IUsingFingerprintServices WithFingerprintConfig(Action <CustomFingerprintConfiguration> functor)
{
CustomFingerprintConfiguration customFingerprintConfiguration = new CustomFingerprintConfiguration();
FingerprintConfiguration = customFingerprintConfiguration;
functor(customFingerprintConfiguration);
return(this);
}
public Hashes CreateFingerprintsFromImageFrames(Frames imageFrames, FingerprintConfiguration configuration)
{
var frames = imageFrames.ToList();
var hashes = CreateOriginalFingerprintsFromFrames(frames, configuration)
.AsParallel()
.Select(fingerprint => lshAlgorithm.HashImage(fingerprint, configuration.HashingConfig))
.ToList();
return(new Hashes(hashes, imageFrames.Duration, MediaType.Video, imageFrames.RelativeTo, new[] { imageFrames.Origin }));
}
public Hashes CreateFingerprintsFromAudioSamples(AudioSamples samples, FingerprintConfiguration configuration)
{
var spectrumFrames = spectrumService.CreateLogSpectrogram(samples, configuration.SpectrogramConfig);
var hashes = CreateOriginalFingerprintsFromFrames(spectrumFrames, configuration)
.AsParallel()
.ToList()
.Select(fingerprint => lshAlgorithm.Hash(fingerprint, configuration.HashingConfig))
.ToList();
return(new Hashes(hashes, samples.Duration, MediaType.Audio, samples.RelativeTo, new[] { samples.Origin }));
}
public List <ResultEntry> GetBestCandidates(
List <HashedFingerprint> hashedFingerprints,
IDictionary <IModelReference, ResultEntryAccumulator> hammingSimilarites,
int maxNumberOfMatchesToReturn,
IModelService modelService,
FingerprintConfiguration fingerprintConfiguration)
{
double queryLength = CalculateExactQueryLength(hashedFingerprints, fingerprintConfiguration);
return(hammingSimilarites.OrderByDescending(e => e.Value.HammingSimilaritySum)
.Take(maxNumberOfMatchesToReturn)
.Select(e => GetResultEntry(modelService, fingerprintConfiguration, e, queryLength))
.ToList());
}
public List <ResultEntry> GetBestCandidates(
List <HashedFingerprint> hashedFingerprints,
IDictionary <IModelReference, ResultEntryAccumulator> hammingSimilarites,
int maxNumberOfMatchesToReturn,
IModelService modelService,
FingerprintConfiguration fingerprintConfiguration)
{
double queryLength = CalculateExactQueryLength(hashedFingerprints, fingerprintConfiguration);
var trackIds = hammingSimilarites.OrderByDescending(e => e.Value.HammingSimilaritySum)
.Take(maxNumberOfMatchesToReturn)
.Select(p => p.Key)
.ToList();
var tracks = modelService.ReadTracksByReferences(trackIds);
return(tracks
.Select(track => GetResultEntry(fingerprintConfiguration, track, hammingSimilarites[track.TrackReference], queryLength))
.ToList());
}
public IUsingFingerprintServices WithFingerprintConfig(FingerprintConfiguration configuration)
{
FingerprintConfiguration = configuration;
return(this);
}
private static double GetLength(FingerprintConfiguration configuration, IReadOnlyCollection <HashedFingerprint> hashes)
{
return(hashes.Any() ? hashes.Max(_ => _.StartsAt) + configuration.FingerprintLengthInSeconds : 0d);
}
public IUsingQueryServices WithFingerprintConfig(FingerprintConfiguration fingerprintConfiguration)
{
FingerprintConfiguration = fingerprintConfiguration;
return(this);
}
public static double AdjustLengthToSeconds(double endsAt, double startsAt, FingerprintConfiguration configuration)
{
return(endsAt - startsAt + configuration.FingerprintLengthInSeconds);
}
public IUsingQueryServices WithConfigs(FingerprintConfiguration fingerprintConfiguration, QueryConfiguration configuration)
{
QueryConfiguration = configuration;
FingerprintConfiguration = fingerprintConfiguration;
return(this);
}
public Hashes CreateFingerprintsFromImageFrames(IEnumerable <Frame> imageFrames, FingerprintConfiguration configuration)
{
var frames = imageFrames.ToList();
var hashes = CreateOriginalFingerprintsFromFrames(frames, configuration)
.AsParallel()
.Select(fingerprint => lshAlgorithm.HashImage(fingerprint, configuration.HashingConfig))
.ToList()
.Join(frames, hashed => hashed.SequenceNumber, frame => frame.SequenceNumber, (hash, frame) =>
{
byte[] transformed = configuration.OriginalPointSaveTransform != null ? configuration.OriginalPointSaveTransform(frame) : Array.Empty <byte>();
return(new HashedFingerprint(hash.HashBins, hash.SequenceNumber, hash.StartsAt, transformed));
})
.ToList();
return(new Hashes(hashes, GetDuration(hashes, configuration.FingerprintLengthInSeconds)));
}
public IUsingFingerprintServices WithFingerprintConfig(Func <FingerprintConfiguration, FingerprintConfiguration> amendFunctor)
{
FingerprintConfiguration = amendFunctor(FingerprintConfiguration);
return(this);
}
public Coverage GetCoverage(SortedSet <MatchedPair> matches, double queryLength, FingerprintConfiguration configuration)
{
int minI = 0, maxI = 0, curMinI = 0, maxLength = 0;
var sortedMatches = matches.ToList();
for (int i = 1; i < sortedMatches.Count; ++i)
{
if (ConsecutiveMatchesAreLongerThanTheQuery(queryLength, sortedMatches, i, configuration))
{
// potentialy a new start of best matched sequence
curMinI = i;
}
if (i - curMinI > maxLength)
{
maxLength = i - curMinI;
maxI = i;
minI = curMinI;
}
}
double notCovered = 0d;
for (int i = minI + 1; i <= maxI; ++i)
{
if (sortedMatches[i].SubFingerprint.SequenceAt - sortedMatches[i - 1].SubFingerprint.SequenceAt > configuration.FingerprintLengthInSeconds)
{
notCovered += sortedMatches[i].SubFingerprint.SequenceAt - (sortedMatches[i - 1].SubFingerprint.SequenceAt + configuration.FingerprintLengthInSeconds);
}
}
double sourceMatchLength = SubFingerprintsToSeconds.AdjustLengthToSeconds(
sortedMatches[maxI].SubFingerprint.SequenceAt,
sortedMatches[minI].SubFingerprint.SequenceAt,
configuration) - notCovered;
double sourceMatchStartsAt = sortedMatches[minI].HashedFingerprint.StartsAt;
double originMatchStartsAt = sortedMatches[minI].SubFingerprint.SequenceAt;
return(new Coverage(sourceMatchStartsAt, sourceMatchLength, originMatchStartsAt));
}
private bool ConsecutiveMatchesAreLongerThanTheQuery(double queryLength, List <MatchedPair> sortedMatches, int index, FingerprintConfiguration config)
{
return(SubFingerprintsToSeconds.AdjustLengthToSeconds(
sortedMatches[index].SubFingerprint.SequenceAt,
sortedMatches[index - 1].SubFingerprint.SequenceAt,
config) > queryLength);
}
private List <Fingerprint> CreateFingerprintsFromLogSpectrum(IEnumerable <SpectralImage> spectralImages, FingerprintConfiguration configuration)
{
var fingerprints = new List <Fingerprint>();
foreach (var spectralImage in spectralImages)
{
waveletDecomposition.DecomposeImageInPlace(spectralImage.Image);
bool[] image = fingerprintDescriptor.ExtractTopWavelets(spectralImage.Image, configuration.TopWavelets);
if (!IsSilence(image))
{
fingerprints.Add(new Fingerprint {
Signature = image, Timestamp = spectralImage.Timestamp, SequenceNumber = spectralImage.SequenceNumber
});
}
}
return(fingerprints);
}
public List <HashedFingerprint> CreateFingerprints(AudioSamples samples, FingerprintConfiguration configuration)
{
// Explode samples to the range of 16 bit shorts (–32,768 to 32,767)
// Matlab multiplies with 2^15 (32768)
const int AUDIO_MULTIPLIER = 65536; // 32768 still makes alot of mfcc feature computations fail!
// Explode samples to the range of 16 bit shorts (–32,768 to 32,767)
// Matlab multiplies with 2^15 (32768)
// e.g. if( max(abs(speech))<=1 ), speech = speech * 2^15; end;
float[] audiodata = samples.Samples;
MathUtils.Multiply(ref audiodata, AUDIO_MULTIPLIER);
// zero pad if the audio file is too short to perform a fft
if (audiodata.Length < (configuration.SpectrogramConfig.WdftSize + configuration.SpectrogramConfig.Overlap))
{
int lenNew = configuration.SpectrogramConfig.WdftSize + configuration.SpectrogramConfig.Overlap;
Array.Resize <float>(ref audiodata, lenNew);
}
samples.Samples = audiodata;
if (configuration.SpectrogramConfig.Verbosity == Verbosity.Verbose)
{
WriteOutputUtils.WriteCSV(audiodata, Path.Combine(SoundFingerprinter.DEBUG_DIRECTORY_PATH, (Path.GetFileNameWithoutExtension(samples.Origin) + "_audiodata.csv")));
}
// create log spectrogram
var spectralImages = spectrumService.CreateLogSpectrogram(samples, configuration.SpectrogramConfig);
if (configuration.SpectrogramConfig.Verbosity == Verbosity.Verbose)
{
if (spectralImages.Count > 0)
{
var imageService = new FindSimilarImageService();
using (Image image = imageService.GetLogSpectralImages(spectralImages, spectralImages.Count > 5 ? 5 : spectralImages.Count))
{
var fileName = Path.Combine(SoundFingerprinter.DEBUG_DIRECTORY_PATH, (Path.GetFileNameWithoutExtension(samples.Origin) + "_spectral_images.png"));
if (fileName != null)
{
image.Save(fileName, ImageFormat.Png);
}
}
}
}
var fingerprints = CreateFingerprintsFromLogSpectrum(spectralImages, configuration);
if (configuration.SpectrogramConfig.Verbosity == Verbosity.Verbose)
{
if (fingerprints.Count > 0)
{
var imageService = new FindSimilarImageService();
using (Image image = imageService.GetImageForFingerprints(fingerprints, 128, 32, fingerprints.Count > 5 ? 5 : fingerprints.Count))
{
var fileName = Path.Combine(SoundFingerprinter.DEBUG_DIRECTORY_PATH, (Path.GetFileNameWithoutExtension(samples.Origin) + "_fingerprints.png"));
if (fileName != null)
{
image.Save(fileName, ImageFormat.Png);
}
}
}
}
var hashedFingerprints = HashFingerprints(fingerprints, configuration);
if (configuration.SpectrogramConfig.Verbosity == Verbosity.Verbose)
{
if (hashedFingerprints.Count > 0)
{
var hashedFingerprintList = new List <int[]>();
foreach (var hashedFingerprint in hashedFingerprints)
{
hashedFingerprintList.Add(hashedFingerprint.HashBins);
}
var hashedFingerprinArray = hashedFingerprintList.ToArray();
WriteOutputUtils.WriteCSV(hashedFingerprinArray, Path.Combine(SoundFingerprinter.DEBUG_DIRECTORY_PATH, (Path.GetFileNameWithoutExtension(samples.Origin) + "_hashbins.csv")), ";");
}
}
return(hashedFingerprints);
}
private static double CalculateExactQueryLength(IEnumerable <HashedFingerprint> hashedFingerprints, FingerprintConfiguration fingerprintConfiguration)
{
double startsAt = double.MaxValue, endsAt = double.MinValue;
foreach (var hashedFingerprint in hashedFingerprints)
{
startsAt = System.Math.Min(startsAt, hashedFingerprint.StartsAt);
endsAt = System.Math.Max(endsAt, hashedFingerprint.StartsAt);
}
return(SubFingerprintsToSeconds.AdjustLengthToSeconds(endsAt, startsAt, fingerprintConfiguration.FingerprintLengthInSeconds));
}
internal IEnumerable <Fingerprint> CreateOriginalFingerprintsFromFrames(IEnumerable <Frame> frames, FingerprintConfiguration configuration)
{
var normalized = configuration.FrameNormalizationTransform.Normalize(frames);
var blurred = configuration.GaussianBlurConfiguration.GaussianFilter switch
{
GaussianFilter.None => normalized,
_ => BlurFrames(normalized, configuration.GaussianBlurConfiguration)
};
var images = blurred.ToList();
if (!images.Any())
{
return(Enumerable.Empty <Fingerprint>());
}
var fingerprints = new ConcurrentBag <Fingerprint>();
var length = images.First().Length;
var saveTransform = configuration.OriginalPointSaveTransform;
Parallel.ForEach(images, () => new ushort[length], (frame, loop, cachedIndexes) =>
{
byte[] originalPoint = saveTransform(frame);
float[] rowCols = frame.ImageRowCols;
waveletDecomposition.DecomposeImageInPlace(rowCols, frame.Rows, frame.Cols, configuration.HaarWaveletNorm);
RangeUtils.PopulateIndexes(length, cachedIndexes);
var image = fingerprintDescriptor.ExtractTopWavelets(rowCols, configuration.TopWavelets, cachedIndexes);
if (!image.IsSilence())
{
fingerprints.Add(new Fingerprint(image, frame.StartsAt, frame.SequenceNumber, originalPoint));
}
return(cachedIndexes);
},
cachedIndexes => { });
return(fingerprints.ToList());
}
internal FingerprintCommand(IFingerprintService fingerprintService)
{
this.fingerprintService = fingerprintService;
fingerprintConfiguration = new DefaultFingerprintConfiguration();
}
public IUsingFingerprintServices WithFingerprintConfig <T>() where T : FingerprintConfiguration, new()
{
FingerprintConfiguration = new T();
return(this);
}
public static double QueryLength(this IEnumerable <HashedFingerprint> hashedFingerprints, FingerprintConfiguration configuration)
{
double startsAt = double.MaxValue, endsAt = double.MinValue;
foreach (var hashedFingerprint in hashedFingerprints)
{
startsAt = System.Math.Min(startsAt, hashedFingerprint.StartsAt);
endsAt = System.Math.Max(endsAt, hashedFingerprint.StartsAt);
}
return(SubFingerprintsToSeconds.MatchLengthToSeconds(endsAt, startsAt, configuration.FingerprintLengthInSeconds));
}
public List <Fingerprint> CreateFingerprintsFromLogSpectrum(IEnumerable <SpectralImage> spectralImages, FingerprintConfiguration configuration)
{
var fingerprints = new ConcurrentBag <Fingerprint>();
var spectrumLength = configuration.SpectrogramConfig.ImageLength * configuration.SpectrogramConfig.LogBins;
Parallel.ForEach(spectralImages, () => new ushort[spectrumLength], (spectralImage, loop, cachedIndexes) =>
{
waveletDecomposition.DecomposeImageInPlace(spectralImage.Image, spectralImage.Rows, spectralImage.Cols, configuration.HaarWaveletNorm);
RangeUtils.PopulateIndexes(spectrumLength, cachedIndexes);
var image = fingerprintDescriptor.ExtractTopWavelets(spectralImage.Image, configuration.TopWavelets, cachedIndexes);
if (!image.IsSilence())
{
fingerprints.Add(new Fingerprint(image, spectralImage.StartsAt, spectralImage.SequenceNumber));
}
return(cachedIndexes);
},
cachedIndexes => { });
return(fingerprints.ToList());
}
internal IEnumerable <Fingerprint> CreateOriginalFingerprintsFromFrames(IEnumerable <Frame> frames, FingerprintConfiguration configuration)
{
var fingerprints = new ConcurrentBag <Fingerprint>();
var images = frames.ToList();
if (!images.Any())
{
return(Enumerable.Empty <Fingerprint>());
}
var length = images.First().Length;
Parallel.ForEach(images, () => new ushort[length], (frame, loop, cachedIndexes) =>
{
float[] rowCols = configuration.OriginalPointSaveTransform != null ? frame.GetImageRowColsCopy() : frame.ImageRowCols;
waveletDecomposition.DecomposeImageInPlace(rowCols, frame.Rows, frame.Cols, configuration.HaarWaveletNorm);
RangeUtils.PopulateIndexes(length, cachedIndexes);
var image = fingerprintDescriptor.ExtractTopWavelets(rowCols, configuration.TopWavelets, cachedIndexes);
if (!image.IsSilence())
{
fingerprints.Add(new Fingerprint(image, frame.StartsAt, frame.SequenceNumber));
}
return(cachedIndexes);
},
cachedIndexes => { });
return(fingerprints.ToList());
}
public double GetQueryLength(FingerprintConfiguration configuration)
{
return(CalculateExactQueryLength(queryFingerprints, configuration));
}
private List <HashedFingerprint> HashFingerprints(IEnumerable <Fingerprint> fingerprints, FingerprintConfiguration configuration)
{
var hashedFingerprints = new ConcurrentBag <HashedFingerprint>();
Parallel.ForEach(fingerprints, (fingerprint, state, index) =>
{
var hashedFingerprint = lshAlgorithm.Hash(fingerprint, configuration.HashingConfig, configuration.Clusters);
hashedFingerprints.Add(hashedFingerprint);
});
return(hashedFingerprints.ToList());
}
