//##################################################################################################################################
//####### STATIC METHODS ###########################################################################################################
//##################################################################################################################################
public static SpectrogramStandard GetSpectralSonogram(string recordingFileName, int frameSize, double windowOverlap, int bitsPerSample, double windowPower, int sr,
TimeSpan duration, NoiseReductionType nrt, double[,] amplitudeSpectrogram)
{
SonogramConfig sonoConfig = new SonogramConfig
{
SourceFName = recordingFileName,
WindowSize = frameSize,
WindowOverlap = windowOverlap,
NoiseReductionType = nrt,
epsilon = Math.Pow(0.5, bitsPerSample - 1),
WindowPower = windowPower,
SampleRate = sr,
Duration = duration,
}; //default values config
var sonogram = new SpectrogramStandard(sonoConfig, amplitudeSpectrogram);
sonogram.SetTimeScale(duration);
return(sonogram);
}
/// <summary>
/// Does the Analysis
/// Returns a DataTable
/// </summary>
/// <param name="fiSegmentOfSourceFile"></param>
/// <param name="configDict"></param>
/// <param name="diOutputDir"></param>
/// <param name="opFileName"></param>
/// <param name="segmentStartOffset"></param>
/// <param name="config"></param>
/// <param name="segmentAudioFile"></param>
public static Tuple <BaseSonogram, double[, ], double[], List <AcousticEvent>, TimeSpan> Analysis(FileInfo fiSegmentOfSourceFile, Dictionary <string, string> configDict, DirectoryInfo diOutputDir, string opFileName, TimeSpan segmentStartOffset)
{
//set default values
int bandWidth = 500; //detect bars in bands of this width.
int frameSize = 1024;
double windowOverlap = 0.0;
double intensityThreshold = double.Parse(configDict[key_INTENSITY_THRESHOLD]);
//intensityThreshold = 0.01;
AudioRecording recording = AudioRecording.GetAudioRecording(fiSegmentOfSourceFile, RESAMPLE_RATE, diOutputDir.FullName, opFileName);
if (recording == null)
{
LoggedConsole.WriteLine("############ WARNING: Recording could not be obtained - likely file does not exist.");
return(null);
}
int sr = recording.SampleRate;
double binWidth = recording.SampleRate / (double)frameSize;
double frameDuration = frameSize / (double)sr;
double frameOffset = frameDuration * (1 - windowOverlap); //seconds between start of each frame
double framesPerSecond = 1 / frameOffset;
TimeSpan tsRecordingtDuration = recording.Duration;
int colStep = (int)Math.Round(bandWidth / binWidth);
//i: GET SONOGRAM AS MATRIX
double epsilon = Math.Pow(0.5, recording.BitsPerSample - 1);
var results2 = DSP_Frames.ExtractEnvelopeAndAmplSpectrogram(recording.WavReader.Samples, sr, epsilon, frameSize, windowOverlap);
double[] avAbsolute = results2.Average; //average absolute value over the minute recording
//double[] envelope = results2.Item2;
double[,] spectrogram = results2.AmplitudeSpectrogram; //amplitude spectrogram. Note that column zero is the DC or average energy value and can be ignored.
double windowPower = results2.WindowPower;
//############################ NEXT LINE FOR DEBUGGING ONLY
//spectrogram = GetTestSpectrogram(spectrogram.GetLength(0), spectrogram.GetLength(1), 0.01, 0.03);
var output = DetectGratingEvents(spectrogram, colStep, intensityThreshold);
var amplitudeArray = output.Item2; //for debug purposes only
//convert List of Dictionary events to List of ACousticevents.
//also set up the hits matrix.
int rowCount = spectrogram.GetLength(0);
int colCount = spectrogram.GetLength(1);
var hitsMatrix = new double[rowCount, colCount];
var acousticEvents = new List <AcousticEvent>();
double minFrameCount = 8; //this assumes that the minimum grid is 2 * 4 = 8 long
foreach (Dictionary <string, double> item in output.Item1)
{
int minRow = (int)item[key_START_FRAME];
int maxRow = (int)item[key_END_FRAME];
int frameCount = maxRow - minRow + 1;
if (frameCount < minFrameCount)
{
continue; //only want events that are over a minimum length
}
int minCol = (int)item[key_MIN_FREQBIN];
int maxCol = (int)item[key_MAX_FREQBIN];
double periodicity = item[key_PERIODICITY];
double[] subarray = DataTools.Subarray(avAbsolute, minRow, maxRow - minRow + 1);
double severity = 0.1;
int[] bounds = DataTools.Peaks_CropToFirstAndLast(subarray, severity);
minRow = minRow + bounds[0];
maxRow = minRow + bounds[1];
if (maxRow >= rowCount)
{
maxRow = rowCount - 1;
}
Oblong o = new Oblong(minRow, minCol, maxRow, maxCol);
var ae = new AcousticEvent(segmentStartOffset, o, results2.NyquistFreq, frameSize, frameDuration, frameOffset, frameCount);
ae.Name = string.Format("p={0:f0}", periodicity);
ae.Score = item[key_SCORE];
ae.ScoreNormalised = item[key_SCORE] / 0.5;
acousticEvents.Add(ae);
//display event on the hits matrix
for (int r = minRow; r < maxRow; r++)
{
for (int c = minCol; c < maxCol; c++)
{
hitsMatrix[r, c] = periodicity;
}
}
} //foreach
//set up the songogram to return. Use the existing amplitude sonogram
int bitsPerSample = recording.WavReader.BitsPerSample;
//NoiseReductionType nrt = SNR.Key2NoiseReductionType("NONE");
NoiseReductionType nrt = SNR.KeyToNoiseReductionType("STANDARD");
var sonogram = (BaseSonogram)SpectrogramStandard.GetSpectralSonogram(recording.BaseName, frameSize, windowOverlap, bitsPerSample, windowPower, sr, tsRecordingtDuration, nrt, spectrogram);
sonogram.DecibelsNormalised = new double[sonogram.FrameCount];
for (int i = 0; i < sonogram.FrameCount; i++) //foreach frame or time step
{
sonogram.DecibelsNormalised[i] = 2 * Math.Log10(avAbsolute[i]);
}
sonogram.DecibelsNormalised = DataTools.normalise(sonogram.DecibelsNormalised);
return(Tuple.Create(sonogram, hitsMatrix, amplitudeArray, acousticEvents, tsRecordingtDuration));
} //Analysis()
} //Analysis()
public static Tuple <BaseSonogram, double[, ], double[], List <AcousticEvent> > DetectHarmonics(
AudioRecording recording,
double intensityThreshold,
int minHz,
int minFormantgap,
int maxFormantgap,
double minDuration,
int windowSize,
double windowOverlap,
TimeSpan segmentStartOffset)
{
//i: MAKE SONOGRAM
int numberOfBins = 32;
double binWidth = recording.SampleRate / (double)windowSize;
int sr = recording.SampleRate;
double frameDuration = windowSize / (double)sr; // Duration of full frame or window in seconds
double frameOffset = frameDuration * (1 - windowOverlap); //seconds between starts of consecutive frames
double framesPerSecond = 1 / frameOffset;
//double framesPerSecond = sr / (double)windowSize;
//int frameOffset = (int)(windowSize * (1 - overlap));
//int frameCount = (length - windowSize + frameOffset) / frameOffset;
double epsilon = Math.Pow(0.5, recording.BitsPerSample - 1);
var results2 = DSP_Frames.ExtractEnvelopeAndAmplSpectrogram(
recording.WavReader.Samples,
sr,
epsilon,
windowSize,
windowOverlap);
double[] avAbsolute = results2.Average; //average absolute value over the minute recording
//double[] envelope = results2.Item2;
double[,]
matrix = results2
.AmplitudeSpectrogram; //amplitude spectrogram. Note that column zero is the DC or average energy value and can be ignored.
double windowPower = results2.WindowPower;
//window sr frameDuration frames/sec hz/bin 64frameDuration hz/64bins hz/128bins
// 1024 22050 46.4ms 21.5 21.5 2944ms 1376hz 2752hz
// 1024 17640 58.0ms 17.2 17.2 3715ms 1100hz 2200hz
// 2048 17640 116.1ms 8.6 8.6 7430ms 551hz 1100hz
//the Xcorrelation-FFT technique requires number of bins to scan to be power of 2.
//assuming sr=17640 and window=1024, then 64 bins span 1100 Hz above the min Hz level. i.e. 500 to 1600
//assuming sr=17640 and window=1024, then 128 bins span 2200 Hz above the min Hz level. i.e. 500 to 2700
int minBin = (int)Math.Round(minHz / binWidth);
int maxHz = (int)Math.Round(minHz + (numberOfBins * binWidth));
int rowCount = matrix.GetLength(0);
int colCount = matrix.GetLength(1);
int maxbin = minBin + numberOfBins;
double[,] subMatrix = MatrixTools.Submatrix(matrix, 0, minBin + 1, rowCount - 1, maxbin);
//ii: DETECT HARMONICS
int zeroBinCount = 5; //to remove low freq content which dominates the spectrum
var results = CrossCorrelation.DetectBarsInTheRowsOfaMatrix(subMatrix, intensityThreshold, zeroBinCount);
double[] intensity = results.Item1; //an array of periodicity scores
double[] periodicity = results.Item2;
//transfer periodicity info to a hits matrix.
//intensity = DataTools.filterMovingAverage(intensity, 3);
double[] scoreArray = new double[intensity.Length];
var hits = new double[rowCount, colCount];
for (int r = 0; r < rowCount; r++)
{
double relativePeriod = periodicity[r] / numberOfBins / 2;
if (intensity[r] > intensityThreshold)
{
for (int c = minBin; c < maxbin; c++)
{
hits[r, c] = relativePeriod;
}
}
double herzPeriod = periodicity[r] * binWidth;
if (herzPeriod > minFormantgap && herzPeriod < maxFormantgap)
{
scoreArray[r] = 2 * intensity[r] * intensity[r]; //enhance high score wrt low score.
}
}
scoreArray = DataTools.filterMovingAverage(scoreArray, 11);
//iii: CONVERT TO ACOUSTIC EVENTS
double maxDuration = 100000.0; //abitrary long number - do not want to restrict duration of machine noise
List <AcousticEvent> predictedEvents = AcousticEvent.ConvertScoreArray2Events(
scoreArray,
minHz,
maxHz,
framesPerSecond,
binWidth,
intensityThreshold,
minDuration,
maxDuration,
segmentStartOffset);
hits = null;
//set up the songogram to return. Use the existing amplitude sonogram
int bitsPerSample = recording.WavReader.BitsPerSample;
TimeSpan duration = recording.Duration;
NoiseReductionType nrt = SNR.KeyToNoiseReductionType("STANDARD");
var sonogram = (BaseSonogram)SpectrogramStandard.GetSpectralSonogram(
recording.BaseName,
windowSize,
windowOverlap,
bitsPerSample,
windowPower,
sr,
duration,
nrt,
matrix);
sonogram.DecibelsNormalised = new double[rowCount];
//foreach frame or time step
for (int i = 0; i < rowCount; i++)
{
sonogram.DecibelsNormalised[i] = 2 * Math.Log10(avAbsolute[i]);
}
sonogram.DecibelsNormalised = DataTools.normalise(sonogram.DecibelsNormalised);
return(Tuple.Create(sonogram, hits, scoreArray, predictedEvents));
} //end Execute_HDDetect
void OnPostprocessTexture(Texture2D texture)
{
this.m_csWaifu2x = Resources.Load("waifu2x-models/waifu2x") as ComputeShader;
if (this.m_csWaifu2x == null)
{
EditorUtility.CompressTexture(texture, texture.format, TextureCompressionQuality.Best);
return;
}
if ((texture.width < Model.BUCKET_WIDHT / 2) || (texture.height < Model.BUCKET_HDEIGHT / 2))
{
EditorUtility.CompressTexture(texture, texture.format, TextureCompressionQuality.Best);
return;
}
string _fileName = Path.GetFileNameWithoutExtension(assetPath);
this.m_useNoiseReductionType = NoiseReductionType.None;
if (_fileName.Contains("NoiseReduction1"))
{
this.m_useNoiseReductionType = NoiseReductionType.Level1;
}
else if (_fileName.Contains("NoiseReduction2"))
{
this.m_useNoiseReductionType = NoiseReductionType.Level2;
}
if (this.m_useNoiseReductionType == NoiseReductionType.None)
{
if (!_fileName.EndsWith("Waifu2x"))
{
return;
}
}
this.m_copyImage = new RenderTexture(texture.width, texture.height, 0, RenderTextureFormat.ARGB32);
this.m_copyImage.filterMode = FilterMode.Point;
this.m_copyImage.wrapMode = TextureWrapMode.Clamp;
this.m_copyImage.enableRandomWrite = true;
this.m_copyImage.useMipMap = false;
this.m_copyImage.Create();
this.copyTexture(texture, ref this.m_copyImage);
this.m_waifu2xImage = new RenderTexture(texture.width * 2, texture.height * 2, 0, RenderTextureFormat.ARGB32);
this.m_waifu2xImage.filterMode = FilterMode.Point;
this.m_waifu2xImage.wrapMode = TextureWrapMode.Clamp;
this.m_waifu2xImage.enableRandomWrite = true;
this.m_waifu2xImage.useMipMap = false;
this.m_waifu2xImage.Create();
if (this.m_useNoiseReductionType != NoiseReductionType.None)
{
this.clearDstImage(texture.width, texture.height);
this.loadJson("waifu2x-models/noise" + ((int)this.m_useNoiseReductionType).ToString() + "_model");
this.noiseReduction();
Graphics.Blit(this.m_dstImage, this.m_copyImage);
GC.Collect();
}
if (_fileName.EndsWith("Waifu2x"))
{
this.clearDstImage(this.m_waifu2xImage.width, this.m_waifu2xImage.height);
this.loadJson("waifu2x-models/scale2.0x_model");
this.scaling();
Graphics.Blit(this.m_dstImage, this.m_waifu2xImage);
GC.Collect();
texture.Resize(this.m_waifu2xImage.width, this.m_waifu2xImage.height);
texture.Apply();
RenderTexture.active = this.m_waifu2xImage;
texture.ReadPixels(new Rect(0, 0, this.m_waifu2xImage.width, this.m_waifu2xImage.height), 0, 0);
texture.Apply();
RenderTexture.active = null;
}
else
{
RenderTexture.active = this.m_copyImage;
texture.ReadPixels(new Rect(0, 0, this.m_copyImage.width, this.m_copyImage.height), 0, 0);
texture.Apply();
RenderTexture.active = null;
}
this.disposeTexture(ref this.m_copyImage);
this.disposeTexture(ref this.m_dstImage);
this.disposeTexture(ref this.m_waifu2xImage);
GC.Collect();
EditorUtility.CompressTexture(texture, texture.format, TextureCompressionQuality.Best);
}
