public void Construction()
{
var s = @"
# comment
key1:value1
key2 : value2 # comment
key.key3: value3
# comment
key4= value4
key5=value5
multilines = line1 # comment
line2 # comment
line3
multilines2 : line1
line2
line3 # comment
multilines3 :
line1
line2
line3 # comment
key key = value value
";
var dict = new ConfigDictionary(new StringReader(s));
Assert.IsTrue(dict.ContainsKey("key1"));
Assert.AreEqual(dict["key1"].Value, "value1");
Assert.IsTrue(dict.ContainsKey("key2"));
Assert.AreEqual(dict["key2"].Value, "value2");
Assert.IsTrue(dict.ContainsKey("key.key3"));
Assert.AreEqual(dict["key.key3"].Value, "value3");
Assert.IsTrue(dict.ContainsKey("key4"));
Assert.AreEqual(dict["key4"].Value, "value4");
Assert.IsTrue(dict.ContainsKey("key5"));
Assert.AreEqual(dict["key5"].Value, "value5");
Assert.IsTrue(dict.ContainsKey("multilines"));
Assert.AreEqual(dict["multilines"].Value, "line1");
CollectionAssert.AreEqual(dict["multilines"].Subsididary, new[] { "line2", "line3" });
Assert.IsTrue(dict.ContainsKey("multilines2"));
Assert.AreEqual(dict["multilines2"].Value, "line1");
CollectionAssert.AreEqual(dict["multilines2"].Subsididary, new[] { "line2", "line3" });
Assert.IsTrue(dict.ContainsKey("multilines3"));
Assert.AreEqual(dict["multilines3"].Value, "");
CollectionAssert.AreEqual(dict["multilines3"].Subsididary, new[] { "line1", "line2", "line3" });
Assert.IsTrue(dict.ContainsKey("key key"));
Assert.AreEqual(dict["key key"].Value, "value value");
Dump(dict);
}
public WatchSharingConfiguration()
{
DebugEnabled = false;
TraktEnabled = false;
TraktConfiguration = new ConfigDictionary();
FollwitEnabled = false;
FollwitConfiguration = new ConfigDictionary();
}
public WebMediaPortal()
{
StreamType = StreamType.DirectWhenPossible;
MASUrl = "127.0.0.1:4322";
TASUrl = "127.0.0.1:4322";
Skin = "default";
ExternalUrlScheme = UrlScheme.Default;
ExternalUrlHost = null;
SkinConfiguration = new ConfigDictionary();
MusicLayout = MusicLayoutType.Artist;
}
public void Setup()
{
_expected = new EngineConfiguration();
_expected.LockTimeout = TimeSpan.FromSeconds(30);
_expected.Kernel = Kernels.RoyalFoodTaster;
_expected.MaxBytesPerJournalSegment = 8192*1024;
_configDictionary = new ConfigDictionary();
_configDictionary.Set("kernel", _expected.Kernel);
_configDictionary.Set("engineconfiguration.locktimeout", _expected.LockTimeout);
_configDictionary.Set("EngineConfiguration.MaxbytesPerJournalSegment", _expected.MaxBytesPerJournalSegment);
}
private string[] RefreshArchitectures(string keyChain, ConfigDictionary refreshedConfigSet, string variantKey, BuildArchitecture[] architectures, BuildDistribution[] distributions, ConfigDictionary prevConfigSet)
{
List <string> childKeys = new List <string>();
for (int i = 0; i < architectures.Length; i++)
{
// Skip if architecture is disabled.
if (!architectures[i].enabled)
{
continue;
}
string key = keyChain + "/" + architectures[i].name;
if (variantKey.Length > 0)
{
key += " (" + variantKey + ")";
}
Configuration relConfig = new Configuration();
// Check for a duplicate key.
if (refreshedConfigSet.ContainsKey(key))
{
continue;
}
// Copy previous settings if they exist.
if (prevConfigSet != null && prevConfigSet.ContainsKey(key))
{
relConfig.enabled = prevConfigSet[key].enabled;
}
// Refresh distributions.
if (distributions.Length > 0)
{
relConfig.childKeys = RefreshDistributions(key, refreshedConfigSet, distributions, prevConfigSet);
}
// Save configuration.
refreshedConfigSet.Add(key, relConfig);
// Add key to list to send back to parent.
childKeys.Add(key);
}
return(childKeys.ToArray());
}
public static async Task <dynamic> GetCampaignInsights(string campaignID, HttpClient client)
{
string url = $"{ConfigDictionary.Config()["FacebookAPIURLRoot"]}/{ConfigDictionary.Config()["FacebookAPIVersion"]}/{campaignID}/insights?date_preset=this_month&access_token={ConfigDictionary.Config()["FacebookAccessToken"]}";
try
{
HttpResponseMessage response = await client.GetAsync(url);
string responseData = await response.Content.ReadAsStringAsync();
InsightsRoot insightsRoot = JsonConvert.DeserializeObject <InsightsRoot>(responseData);
return(insightsRoot);
}
catch (Exception ex)
{
return(ex);
}
}
/// <summary>
/// Builds OSLEBot configuration based on input data
/// </summary>
/// <param name="candidateFiles">Physical files to filter for checks. For global checks, all candidate files will be processed.</param>
/// <param name="checkPaths">List of paths to C# files containing source code of checks to be compiled.</param>
/// <param name="language">Language to filter the input file set on.</param>
/// <param name="project">Project to filter the input file set on.</param>
/// <returns>Configuration for OSLEBot that can be simply passed into OSLEBotEngine.Execute().</returns>
internal EngineConfig CreateAsimoConfig(IEnumerable <ConfigItem> candidateFiles, IEnumerable <string> checkPaths, string language, string project)
{
//Currently only process global checks (should be 1 for proof of concept).
var configuration = new EngineConfig {
EngineLog = "Asimo log for Cerberus.txt"
};
#region DataSourceProviderTypes
configuration.AddDataSourceProvider <LcxDataSource>();
configuration.AddDataSourceProvider <ConfigDictionaryDataSource>();
#endregion
#region PropertyAdapterTypes
configuration.AddPropertyAdapter <LocItemPropertyAdapter>();
configuration.AddPropertyAdapter <ConfigDictPropertyAdapter>();
configuration.AddPropertyAdapter <LocResourceSelfPropertyAdapter>(); //Although we don't use it explicitly, LCXLocResourceDataAdapter.LoadObjects() will crash when starting OSLEBot because it expects this property adapter to be available.???
#endregion
#region COAdatpers
configuration.AddCOAdapter <LCXLocResourceDataAdapter>();
#endregion
#region COTypes
configuration.AddClassificationObject <LocResource>();
#endregion
#region DataSourcePackages
var inputForGlobalChecks = from file in candidateFiles
where file.Language.Equals(language)
where file.Project.Equals(project)
select file;
foreach (var file in inputForGlobalChecks)
{
var package = new DataSourcePackage(); //This package will contain 2 data sources
package.SetCOType <LocResource>();
var dataSource = new DataSourceInfo();
dataSource.SetSourceType <LocDocument>();
dataSource.SetSourceLocation(file.PhysicalPath);
package.AddDataSource(dataSource);
dataSource = new DataSourceInfo();
dataSource.SetSourceType <ConfigDictionary>();
var staticProperties = new ConfigDictionary
{
{ "BuildNumber", "1" },
{ "Project", file.Project },
{ "Locgroup", file.LocGroup }
};
dataSource.SetSourceLocation(staticProperties);
package.AddDataSource(dataSource);
configuration.AddDataSourcePackage(package);
}
#endregion
#region Add a sample rule
foreach (var check in checkPaths) //For test should be 1.
{
configuration.AddRule(check, string.Empty, RuleContainerType.Source);
}
#region And some configuration for dynamic compiler
configuration.AddBinaryReference("System.Core.dll");
configuration.AddBinaryReference("mscorlib.dll");
configuration.AddBinaryReference("System.dll");
configuration.AddBinaryReference("Microsoft.Localization.dll");
configuration.AddBinaryReference("OSLEBotCore.dll");
configuration.AddBinaryReference("LocResource.dll");
configuration.AddBinaryReference("OSLEBot.LCXDataAdapter.dll");
#endregion
#endregion
#region Configure output behavior
var outputCfg = new OSLEBot.Core.Output.OutputWriterConfig();
outputCfg.SetDataSourceProvider <LocResourcePerLCLOutputWriter>();
outputCfg.Schema = "OSLEBotOutput.xsd";
outputCfg.Path = Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location);
outputCfg.AddPropertyToIncludeInOutput("LSResID");
outputCfg.AddPropertyToIncludeInOutput("SourceString");
outputCfg.AddPropertyToIncludeInOutput("TargetString");
outputCfg.AddPropertyToIncludeInOutput("Comments");
outputCfg.AddPropertyToIncludeInOutput("TargetCulture");
outputCfg.AddPropertyToIncludeInOutput("Locgroup");
outputCfg.AddPropertyToIncludeInOutput("Project");
configuration.OutputConfigs.Add(outputCfg);
#endregion
#region Because we don't store LSBuild enviornment, point compiler to that of Office14 source depot
configuration.AddAssemblyResolverPaths(Enlistment.LSBuildToolsPath);
#endregion
return(configuration);
}
public async Task <dynamic> Get(string companyName)
{
if (CheckClientSecret())
{
DateTime operationStarted = DateTime.Now;
Entities db = new Entities();
using (HttpClient client = new HttpClient())
{
var result = await BulkAction.PerformBulkAction(client, companyName, db, ConfigDictionary.Config()["ShopifyAdminAPIVersion"]);
if (Equals(result.GetType(), typeof(string)))
{
StoreCustomData store = db.StoreCustomDatas.FirstOrDefault(x => x.StoreName == companyName);
var dbResult = BulkAction.WriteSalesData(result,
store.StoreID,
db,
operationStarted);
if (!Equals(dbResult.GetType(), typeof(Exception)))
{
return(db.SalesDatas.Where(x => x.StoreID == store.StoreID).OrderBy(x => x.Date));
}
else
{
return(new ArgumentException("An error occurred while writing to the database."));
}
}
else
{
return(result);
}
}
}
else
{
return(new HttpResponseMessage(System.Net.HttpStatusCode.Forbidden));
}
}
public static Tuple <Dictionary <string, double>, TimeSpan> RainAnalyser(FileInfo fiAudioFile, AnalysisSettings analysisSettings, SourceMetadata originalFile)
{
Dictionary <string, string> config = analysisSettings.ConfigDict;
// get parameters for the analysis
int frameSize = IndexCalculateConfig.DefaultWindowSize;
double windowOverlap = 0.0;
int lowFreqBound = 1000;
int midFreqBound = 8000;
if (config.ContainsKey(AnalysisKeys.FrameLength))
{
frameSize = ConfigDictionary.GetInt(AnalysisKeys.FrameLength, config);
}
if (config.ContainsKey(key_LOW_FREQ_BOUND))
{
lowFreqBound = ConfigDictionary.GetInt(key_LOW_FREQ_BOUND, config);
}
if (config.ContainsKey(key_MID_FREQ_BOUND))
{
midFreqBound = ConfigDictionary.GetInt(key_MID_FREQ_BOUND, config);
}
if (config.ContainsKey(AnalysisKeys.FrameOverlap))
{
windowOverlap = ConfigDictionary.GetDouble(AnalysisKeys.FrameOverlap, config);
}
// get recording segment
AudioRecording recording = new AudioRecording(fiAudioFile.FullName);
// calculate duration/size of various quantities.
int signalLength = recording.WavReader.Samples.Length;
TimeSpan audioDuration = TimeSpan.FromSeconds(recording.WavReader.Time.TotalSeconds);
double duration = frameSize * (1 - windowOverlap) / (double)recording.SampleRate;
TimeSpan frameDuration = TimeSpan.FromTicks((long)(duration * TimeSpan.TicksPerSecond));
int chunkDuration = 10; //seconds
double framesPerSecond = 1 / frameDuration.TotalSeconds;
int chunkCount = (int)Math.Round(audioDuration.TotalSeconds / (double)chunkDuration);
int framesPerChunk = (int)(chunkDuration * framesPerSecond);
string[] classifications = new string[chunkCount];
//i: EXTRACT ENVELOPE and FFTs
double epsilon = Math.Pow(0.5, recording.BitsPerSample - 1);
var signalextract = DSP_Frames.ExtractEnvelopeAndAmplSpectrogram(recording.WavReader.Samples, recording.SampleRate, epsilon, frameSize, windowOverlap);
double[] envelope = signalextract.Envelope;
double[,] spectrogram = signalextract.AmplitudeSpectrogram; //amplitude spectrogram
int colCount = spectrogram.GetLength(1);
int nyquistFreq = recording.Nyquist;
int nyquistBin = spectrogram.GetLength(1) - 1;
double binWidth = nyquistFreq / (double)spectrogram.GetLength(1);
// calculate the bin id of boundary between mid and low frequency spectrum
int lowBinBound = (int)Math.Ceiling(lowFreqBound / binWidth);
// IFF there has been UP-SAMPLING, calculate bin of the original audio nyquist. this iwll be less than 17640/2.
int originalAudioNyquist = originalFile.SampleRate / 2; // original sample rate can be anything 11.0-44.1 kHz.
if (recording.Nyquist > originalAudioNyquist)
{
nyquistFreq = originalAudioNyquist;
nyquistBin = (int)Math.Floor(originalAudioNyquist / binWidth);
}
// vi: CALCULATE THE ACOUSTIC COMPLEXITY INDEX
var subBandSpectrogram = MatrixTools.Submatrix(spectrogram, 0, lowBinBound, spectrogram.GetLength(0) - 1, nyquistBin);
double[] aciArray = AcousticComplexityIndex.CalculateACI(subBandSpectrogram);
double aci1 = aciArray.Average();
// ii: FRAME ENERGIES -
// convert signal to decibels and subtract background noise.
double StandardDeviationCount = 0.1; // number of noise SDs to calculate noise threshold - determines severity of noise reduction
var results3 = SNR.SubtractBackgroundNoiseFromWaveform_dB(SNR.Signal2Decibels(signalextract.Envelope), StandardDeviationCount);
var dBarray = SNR.TruncateNegativeValues2Zero(results3.NoiseReducedSignal);
//// vii: remove background noise from the full spectrogram i.e. BIN 1 to Nyquist
//spectrogramData = MatrixTools.Submatrix(spectrogramData, 0, 1, spectrogramData.GetLength(0) - 1, nyquistBin);
//const double SpectralBgThreshold = 0.015; // SPECTRAL AMPLITUDE THRESHOLD for smoothing background
//double[] modalValues = SNR.CalculateModalValues(spectrogramData); // calculate modal value for each freq bin.
//modalValues = DataTools.filterMovingAverage(modalValues, 7); // smooth the modal profile
//spectrogramData = SNR.SubtractBgNoiseFromSpectrogramAndTruncate(spectrogramData, modalValues);
//spectrogramData = SNR.RemoveNeighbourhoodBackgroundNoise(spectrogramData, SpectralBgThreshold);
//set up the output
if (Verbose)
{
LoggedConsole.WriteLine("{0:d2}, {1}, {2}, {3}, {4}, {5}, {6}, {7}, {8}, {9}, {10}, {11}", "start", "end", "avDB", "BG", "SNR", "act", "spik", "lf", "mf", "hf", "H[t]", "H[s]", "index1", "index2");
}
StringBuilder sb = null;
if (WriteOutputFile)
{
string header = string.Format("{0:d2},{1},{2},{3},{4},{5},{6},{7},{8},{9},{10},{11}", "start", "end", "avDB", "BG", "SNR", "act", "spik", "lf", "mf", "hf", "H[t]", "H[s]", "index1", "index2");
sb = new StringBuilder(header + "\n");
}
Dictionary <string, double> dict = RainIndices.GetIndices(envelope, audioDuration, frameDuration, spectrogram, lowFreqBound, midFreqBound, binWidth);
return(Tuple.Create(dict, audioDuration));
} //Analysis()
/// <summary>
/// Gets a value of of the property specified by name from ConfigDictionary.
/// </summary>
/// <param name="propertyId">Name of the property to retrieve.</param>
/// <param name="md">Dictionary (data source) containing the data for the property.</param>
/// <exception cref="PropertyRetrievalException">Thrown when either data source is null or when the data source does not contain the string value for the specified property name.</exception>
private static Property GetConfigDictionaryProperty(LocResource.LocResourceProps propertyId, ConfigDictionary md)
{
if (md == null)
{
throw new PropertyRetrievalException(String.Format(CultureInfo.CurrentCulture, "DataSource must not be null. DataSource name: {0}", md.GetType().Name));
}
string value;
md.TryGetValue(propertyId.ToString(), out value);
switch (propertyId)
{
case LocResource.LocResourceProps.ResourceId:
return(new StringProperty((byte)propertyId, value));
case LocResource.LocResourceProps.SourceString:
return(new StringProperty((byte)propertyId, value));
case LocResource.LocResourceProps.Comments:
return(new StringProperty((byte)propertyId, value));
case LocResource.LocResourceProps.FilePath:
return(new StringProperty((byte)propertyId, value));
default: throw new PropertyRetrievalException(String.Format(CultureInfo.CurrentCulture, "Cannot provide property {0} from datasource of type {1}.", propertyId.ToString(), md.GetType().Name));
}
}
public static async Task <dynamic> GetEngagements(HttpClient client, string companyName, Entities db, string companyID)
{
string alias;
try
{
alias = AliasMethods.GetAlias(companyName, db, "HubSpot");
}
catch
{
return(new ArgumentException($"No HubSpot alias found for the company name {companyName}"));
}
string uri = $"https://api.hubapi.com/engagements/v1/engagements/associated/COMPANY/{companyID}/paged?limit=100&hapikey=" + ConfigDictionary.Config()["HubSpotAPIKey"];
try
{
HttpResponseMessage response = await client.GetAsync(uri);
string responseData = await response.Content.ReadAsStringAsync();
EngagementsRoot engagementsData = JsonConvert.DeserializeObject <EngagementsRoot>(responseData);
while (engagementsData.HasMore)
{
engagementsData = await ChainedEngagementsRequest(engagementsData, companyID, client);
}
return(engagementsData);
}
catch (HttpRequestException ex)
{
return(ex);
}
}
public void Subsidiary()
{
var s = @"
key1: value
sub1
sub2
sub3
key2:
sub1
sub2
";
var dict = new ConfigDictionary(new StringReader(s));
Assert.IsTrue(dict.ContainsKey("key1"));
{
var cv = dict["key1"];
Assert.AreEqual("value", cv.Value);
CollectionAssert.AreEqual(new []{ "sub1", "sub2", "sub3" }, cv.Subsididary);
}
Assert.IsTrue(dict.ContainsKey("key2"));
{
var cv = dict["key2"];
Assert.AreEqual("", cv.Value);
CollectionAssert.AreEqual(new []{ "sub1", "sub2" }, cv.Subsididary);
}
Dump(dict);
}
public static void Execute(Arguments arguments)
{
if (arguments == null)
{
arguments = Dev();
}
LoggedConsole.WriteLine("DATE AND TIME:" + DateTime.Now);
LoggedConsole.WriteLine("Syntactic Pattern Recognition\n");
//StringBuilder sb = new StringBuilder("DATE AND TIME:" + DateTime.Now + "\n");
//sb.Append("SCAN ALL RECORDINGS IN A DIRECTORY USING HTK-RECOGNISER\n");
Log.Verbosity = 1;
FileInfo recordingPath = arguments.Source;
FileInfo iniPath = arguments.Config;
DirectoryInfo outputDir = arguments.Output;
string opFName = "SPR-output.txt";
string opPath = outputDir + opFName;
Log.WriteIfVerbose("# Output folder =" + outputDir);
// A: READ PARAMETER VALUES FROM INI FILE
var config = new ConfigDictionary(iniPath);
Dictionary <string, string> dict = config.GetTable();
Dictionary <string, string> .KeyCollection keys = dict.Keys;
string callName = dict[key_CALL_NAME];
double frameOverlap = Convert.ToDouble(dict[key_FRAME_OVERLAP]);
//SPT PARAMETERS
double intensityThreshold = Convert.ToDouble(dict[key_SPT_INTENSITY_THRESHOLD]);
int smallLengthThreshold = Convert.ToInt32(dict[key_SPT_SMALL_LENGTH_THRESHOLD]);
//WHIPBIRD PARAMETERS
int whistle_MinHz = int.Parse(dict[key_WHISTLE_MIN_HZ]);
int whistle_MaxHz = int.Parse(dict[key_WHISTLE_MAX_HZ]);
double optimumWhistleDuration = double.Parse(dict[key_WHISTLE_DURATION]); //optimum duration of whistle in seconds
int whip_MinHz = (dict.ContainsKey(key_WHIP_MIN_HZ)) ? int.Parse(dict[key_WHIP_MIN_HZ]) : 0;
int whip_MaxHz = (dict.ContainsKey(key_WHIP_MAX_HZ)) ? int.Parse(dict[key_WHIP_MAX_HZ]) : 0;
double whipDuration = (dict.ContainsKey(key_WHIP_DURATION)) ? double.Parse(dict[key_WHIP_DURATION]) : 0.0; //duration of whip in seconds
//CURLEW PARAMETERS
double minDuration = (dict.ContainsKey(key_MIN_DURATION)) ? double.Parse(dict[key_MIN_DURATION]) : 0.0; //min duration of call in seconds
double maxDuration = (dict.ContainsKey(key_MAX_DURATION)) ? double.Parse(dict[key_MAX_DURATION]) : 0.0; //duration of call in seconds
double eventThreshold = double.Parse(dict[key_EVENT_THRESHOLD]); //min score for an acceptable event
int DRAW_SONOGRAMS = Convert.ToInt16(dict[key_DRAW_SONOGRAMS]);
// B: CHECK to see if conversion from .MP3 to .WAV is necessary
var destinationAudioFile = recordingPath;
//LOAD RECORDING AND MAKE SONOGRAM
BaseSonogram sonogram = null;
using (var recording = new AudioRecording(destinationAudioFile.FullName))
{
// if (recording.SampleRate != 22050) recording.ConvertSampleRate22kHz(); // THIS METHOD CALL IS OBSOLETE
var sonoConfig = new SonogramConfig
{
NoiseReductionType = NoiseReductionType.None,
//NoiseReductionType = NoiseReductionType.STANDARD,
WindowOverlap = frameOverlap,
};
sonogram = new SpectrogramStandard(sonoConfig, recording.WavReader);
}
List <AcousticEvent> predictedEvents = null;
double[,] hits = null;
double[] scores = null;
var audioFileName = Path.GetFileNameWithoutExtension(destinationAudioFile.FullName);
if (callName.Equals("WHIPBIRD"))
{
//SPT
var result1 = SPT.doSPT(sonogram, intensityThreshold, smallLengthThreshold);
//SPR
Log.WriteLine("SPR start: intensity threshold = " + intensityThreshold);
int slope = 0; //degrees of the circle. i.e. 90 = vertical line.
double sensitivity = 0.7; //lower value = more sensitive
var mHori = MarkLine(result1.Item1, slope, smallLengthThreshold, intensityThreshold, sensitivity);
slope = 87; //84
sensitivity = 0.8; //lower value = more sensitive
var mVert = MarkLine(result1.Item1, slope, smallLengthThreshold - 4, intensityThreshold + 1, sensitivity);
Log.WriteLine("SPR finished");
Log.WriteLine("Extract Whipbird calls - start");
int minBound_Whistle = (int)(whistle_MinHz / sonogram.FBinWidth);
int maxBound_Whistle = (int)(whistle_MaxHz / sonogram.FBinWidth);
int whistleFrames = (int)(sonogram.FramesPerSecond * optimumWhistleDuration); //86 = frames/sec.
int minBound_Whip = (int)(whip_MinHz / sonogram.FBinWidth);
int maxBound_Whip = (int)(whip_MaxHz / sonogram.FBinWidth);
int whipFrames = (int)(sonogram.FramesPerSecond * whipDuration); //86 = frames/sec.
var result3 = DetectWhipBird(mHori, mVert, minBound_Whistle, maxBound_Whistle, whistleFrames, minBound_Whip, maxBound_Whip, whipFrames, smallLengthThreshold);
scores = result3.Item1;
hits = DataTools.AddMatrices(mHori, mVert);
predictedEvents = AcousticEvent.ConvertScoreArray2Events(
scores,
whip_MinHz,
whip_MaxHz,
sonogram.FramesPerSecond,
sonogram.FBinWidth,
eventThreshold,
minDuration,
maxDuration,
TimeSpan.Zero);
foreach (AcousticEvent ev in predictedEvents)
{
ev.FileName = audioFileName;
ev.Name = callName;
}
sonogram.Data = result1.Item1;
Log.WriteLine("Extract Whipbird calls - finished");
}
else if (callName.Equals("CURLEW"))
{
//SPT
double backgroundThreshold = 4.0;
var result1 = SNR.NoiseReduce(sonogram.Data, NoiseReductionType.Standard, backgroundThreshold);
//var result1 = SPT.doSPT(sonogram, intensityThreshold, smallLengthThreshold);
//var result1 = doNoiseRemoval(sonogram, intensityThreshold, smallLengthThreshold);
//SPR
Log.WriteLine("SPR start: intensity threshold = " + intensityThreshold);
int slope = 20; //degrees of the circle. i.e. 90 = vertical line.
double sensitivity = 0.8; //lower value = more sensitive
var mHori = MarkLine(result1.Item1, slope, smallLengthThreshold, intensityThreshold, sensitivity);
slope = 160;
sensitivity = 0.8; //lower value = more sensitive
var mVert = MarkLine(result1.Item1, slope, smallLengthThreshold - 3, intensityThreshold + 1, sensitivity);
Log.WriteLine("SPR finished");
//detect curlew calls
int minBound_Whistle = (int)(whistle_MinHz / sonogram.FBinWidth);
int maxBound_Whistle = (int)(whistle_MaxHz / sonogram.FBinWidth);
int whistleFrames = (int)(sonogram.FramesPerSecond * optimumWhistleDuration);
var result3 = DetectCurlew(mHori, mVert, minBound_Whistle, maxBound_Whistle, whistleFrames, smallLengthThreshold);
//process curlew scores - look for curlew characteristic periodicity
double minPeriod = 1.2;
double maxPeriod = 1.8;
int minPeriod_frames = (int)Math.Round(sonogram.FramesPerSecond * minPeriod);
int maxPeriod_frames = (int)Math.Round(sonogram.FramesPerSecond * maxPeriod);
scores = DataTools.filterMovingAverage(result3.Item1, 21);
scores = DataTools.PeriodicityDetection(scores, minPeriod_frames, maxPeriod_frames);
//extract events
predictedEvents = AcousticEvent.ConvertScoreArray2Events(
scores,
whistle_MinHz,
whistle_MaxHz,
sonogram.FramesPerSecond,
sonogram.FBinWidth,
eventThreshold,
minDuration,
maxDuration,
TimeSpan.Zero);
foreach (AcousticEvent ev in predictedEvents)
{
ev.FileName = audioFileName;
ev.Name = callName;
}
hits = DataTools.AddMatrices(mHori, mVert);
sonogram.Data = result1.Item1;
Log.WriteLine("Extract Curlew calls - finished");
}
else if (callName.Equals("CURRAWONG"))
{
//SPT
var result1 = SPT.doSPT(sonogram, intensityThreshold, smallLengthThreshold);
//SPR
Log.WriteLine("SPR start: intensity threshold = " + intensityThreshold);
int slope = 70; //degrees of the circle. i.e. 90 = vertical line.
//slope = 210;
double sensitivity = 0.7; //lower value = more sensitive
var mHori = MarkLine(result1.Item1, slope, smallLengthThreshold, intensityThreshold, sensitivity);
slope = 110;
//slope = 340;
sensitivity = 0.7; //lower value = more sensitive
var mVert = MarkLine(result1.Item1, slope, smallLengthThreshold - 3, intensityThreshold + 1, sensitivity);
Log.WriteLine("SPR finished");
int minBound_Whistle = (int)(whistle_MinHz / sonogram.FBinWidth);
int maxBound_Whistle = (int)(whistle_MaxHz / sonogram.FBinWidth);
int whistleFrames = (int)(sonogram.FramesPerSecond * optimumWhistleDuration); //86 = frames/sec.
var result3 = DetectCurlew(mHori, mVert, minBound_Whistle, maxBound_Whistle, whistleFrames + 10, smallLengthThreshold);
scores = result3.Item1;
hits = DataTools.AddMatrices(mHori, mVert);
predictedEvents = AcousticEvent.ConvertIntensityArray2Events(
scores,
TimeSpan.Zero,
whistle_MinHz,
whistle_MaxHz,
sonogram.FramesPerSecond,
sonogram.FBinWidth,
eventThreshold,
0.5,
maxDuration);
foreach (AcousticEvent ev in predictedEvents)
{
ev.FileName = audioFileName;
//ev.Name = callName;
}
}
//write event count to results file.
double sigDuration = sonogram.Duration.TotalSeconds;
//string fname = Path.GetFileName(recordingPath);
int count = predictedEvents.Count;
Log.WriteIfVerbose("Number of Events: " + count);
string str = string.Format("{0}\t{1}\t{2}", callName, sigDuration, count);
FileTools.WriteTextFile(opPath, AcousticEvent.WriteEvents(predictedEvents, str).ToString());
// SAVE IMAGE
string imageName = outputDir + audioFileName;
string imagePath = imageName + ".png";
if (File.Exists(imagePath))
{
int suffix = 1;
while (File.Exists(imageName + "." + suffix.ToString() + ".png"))
{
suffix++;
}
//{
// suffix = (suffix == string.Empty) ? "1" : (int.Parse(suffix) + 1).ToString();
//}
//File.Delete(outputDir + audioFileName + "." + suffix.ToString() + ".png");
File.Move(imagePath, imageName + "." + suffix.ToString() + ".png");
}
//string newPath = imagePath + suffix + ".png";
if (DRAW_SONOGRAMS == 2)
{
DrawSonogram(sonogram, imagePath, hits, scores, predictedEvents, eventThreshold);
}
else
if ((DRAW_SONOGRAMS == 1) && (predictedEvents.Count > 0))
{
DrawSonogram(sonogram, imagePath, hits, scores, predictedEvents, eventThreshold);
}
Log.WriteIfVerbose("Image saved to: " + imagePath);
//string savePath = outputDir + Path.GetFileNameWithoutExtension(recordingPath);
//string suffix = string.Empty;
//Image im = sonogram.GetImage(false, false);
//string newPath = savePath + suffix + ".jpg";
//im.Save(newPath);
LoggedConsole.WriteLine("\nFINISHED RECORDING!");
Console.ReadLine();
}
public static async Task <dynamic> GetAllCampaigns(string companyName, Entities db, HttpClient client)
{
string alias;
try
{
alias = AliasMethods.GetAlias(companyName, db, "FacebookAdAccountID");
}
catch
{
return(new ArgumentException($"No HubSpot alias found for the company name {companyName}"));
}
string url = $"{ConfigDictionary.Config()["FacebookAPIURLRoot"]}/{ConfigDictionary.Config()["FacebookAPIVersion"]}/act_{alias}/campaigns?access_token={ConfigDictionary.Config()["FacebookAccessToken"]}";
try
{
HttpResponseMessage response = await client.GetAsync(url);
string responseData = await response.Content.ReadAsStringAsync();
AllCampaignsRoot allCampaignsRoot = JsonConvert.DeserializeObject <AllCampaignsRoot>(responseData);
return(allCampaignsRoot);
}
catch (HttpRequestException ex)
{
return(ex);
}
}
/// <summary>
/// 保存配置信息
/// </summary>
public void Save()
{
if (File.Exists(DefaultXmlFileName)) File.Delete(DefaultXmlFileName);
NewLife.Xml.XmlWriterX xml = new NewLife.Xml.XmlWriterX();
using (XmlWriter writer = XmlWriter.Create(DefaultXmlFileName))
{
xml.Writer = writer;
Dictionary<string, string> dic = ReadFromDgv();
ConfigDictionary temp = new ConfigDictionary();
temp.Items = dic;
xml.WriteObject(temp , typeof(ConfigDictionary ), null);
}
}
public static async Task <EngagementsRoot> ChainedEngagementsRequest(EngagementsRoot engagementsData, string companyID, HttpClient client)
{
string uri = $"https://api.hubapi.com/engagements/v1/engagements/associated/COMPANY/{companyID}/paged?limit=100&offset={engagementsData.Offset}&hapikey=" + ConfigDictionary.Config()["HubSpotAPIKey"];
HttpResponseMessage response = await client.GetAsync(uri);
string responseData = await response.Content.ReadAsStringAsync();
engagementsData = JsonConvert.DeserializeObject <EngagementsRoot>(responseData);
return(engagementsData);
}
public void MultipleKeys()
{
var s = @"
# comment
key1:value1
key1:value2
key1:value3
";
var dict = new ConfigDictionary(new StringReader(s));
Assert.IsTrue(dict.ContainsKey("key1"));
var cv = dict["key1"];
Assert.AreEqual("value1", cv.Value);
cv = cv.NextValue();
Assert.AreEqual("value2", cv.Value);
cv = cv.NextValue();
Assert.AreEqual("value3", cv.Value);
cv = cv.NextValue();
Assert.IsNull(cv);
var array = dict["key1"].Collect();
Assert.AreEqual(3, array.Count);
Assert.AreEqual("value1", array[0].Value);
Assert.AreEqual("value2", array[1].Value);
Assert.AreEqual("value3", array[2].Value);
Dump(dict);
}
private static IConfigPropertyBase GetOrAddProperty(IConfigPropertyBase property)
{
return(ConfigDictionary.GetOrAdd(property.Config, new IndexDictionary <String, IConfigPropertyBase>())
.GetOrAdd(property.Path, property));
}
public IEnumerable <IConfigPropertyBase> GetProperties()
{
return(ConfigDictionary.TryGetValue(this, out IndexDictionary <String, IConfigPropertyBase> dictionary) ? dictionary.Values : null);
}
private static string Dumps(ConfigDictionary dict)
{
StringWriter writer = new StringWriter();
dict.Dump(writer);
writer.Close();
return writer.ToString();
}
/// <summary>
/// The constructor must specify what datasources are expected to be provided to the object factory
/// For example, at this moment all LocResource properties are created from ResourceFiles but some properties (e.g. project)
/// cannot be retrieved from ResourceFiles so we need additional datasource(s) (in this case a simple struct with strings)
/// </summary>
/// <param name="resourceFile">The ResourceFile from which to load objects</param>
/// <param name="md">Extra properties</param>
/// <returns>Objects loaded from the Resource file and 'flattened', with all properties extracted.</returns>
private ICollection <ClassificationObject> LoadObjects(ResourceFile resourceFile, ConfigDictionary md)
{
myConfig = md;
objects = new List <ClassificationObject>(1024);
try
{
// Get all property adapters that deliver LocResource properties from ResourceFileEntry objects
resourceFilePropertyAdapters = this.propertyAdapters.Where(pa => pa.DataSourceType.Equals(typeof(ResourceFileEntry))).ToArray();
if (resourceFilePropertyAdapters.Length == 0)
{
throw new InvalidOperationException("Could not find property adapters for data source type: " + typeof(ResourceFileEntry));
}
if (md != null)
{
// Get all property adapters that deliver LocResource properties from a ConfigDictionary object
configDictPropertyAdapters = this.propertyAdapters.Where(pa => pa.DataSourceType.Equals(typeof(ConfigDictionary))).ToArray();
if (configDictPropertyAdapters.Length == 0)
{
throw new InvalidOperationException("Could not find property adapters for data source type: " + typeof(ConfigDictionary));
}
}
// Get all property adapters that deliver LocResource properties from existing LocResource poperties. This adapter will use the LocResource
// object as its data source
selfPropertyAdapters = this.propertyAdapters.Where(pa => pa.DataSourceType.Equals(typeof(LocResource))).ToArray();
if (selfPropertyAdapters.Length == 0)
{
throw new InvalidOperationException("Could not find property adapters for data source type: " + typeof(LocResource));
}
}
finally { }
// Code specific to the ResourceFileDataSource which is the primary data source
// iterate recursively through all ResourceFileEntry objects in ResourceFile
// each ResourceFileEntry object becomes an instance of LocResource
foreach (ResourceFileEntry entry in resourceFile.Entries)
{
ProcessResourceFileEntry(entry);
}
return(objects);
}
public static void Execute(Arguments arguments)
{
if (arguments == null)
{
arguments = Dev();
}
const string Title = "# EXTRACT AND SAVE ACOUSTIC EVENT.";
string date = "# DATE AND TIME: " + DateTime.Now;
Log.WriteLine(Title);
Log.WriteLine(date);
FileInfo recordingPath = arguments.Source;
FileInfo iniPath = arguments.Config; // path of the ini or params file
string targetName = arguments.Target; // prefix of name of created files
DirectoryInfo outputDir = iniPath.Directory;
FileInfo targetPath = outputDir.CombineFile(targetName + "_target.txt");
FileInfo targetNoNoisePath = outputDir.CombineFile(targetName + "_targetNoNoise.txt");
FileInfo noisePath = outputDir.CombineFile(targetName + "_noise.txt");
FileInfo targetImagePath = outputDir.CombineFile(targetName + "_target.png");
FileInfo paramsPath = outputDir.CombineFile(targetName + "_params.txt");
FileInfo sonogramImagePath = outputDir.CombineFile(Path.GetFileNameWithoutExtension(recordingPath.Name) + ".png");
Log.WriteIfVerbose("# Output folder =" + outputDir);
//i: GET RECORDING
AudioRecording recording = new AudioRecording(recordingPath.FullName);
//if (recording.SampleRate != 22050) recording.ConvertSampleRate22kHz(); // THIS METHOD CALL IS OBSOLETE
int sr = recording.SampleRate;
//ii: READ PARAMETER VALUES FROM INI FILE
var config = new ConfigDictionary(iniPath);
Dictionary <string, string> dict = config.GetTable();
//Dictionary<string, string>.KeyCollection keys = dict.Keys;
double frameOverlap = FeltTemplates_Use.FeltFrameOverlap; // Double.Parse(dict[key_FRAME_OVERLAP]);
double eventStart = double.Parse(dict[key_EVENT_START]);
double eventEnd = double.Parse(dict[key_EVENT_END]);
int minHz = int.Parse(dict[key_MIN_HZ]);
int maxHz = int.Parse(dict[key_MAX_HZ]);
double dBThreshold = double.Parse(dict[key_DECIBEL_THRESHOLD]); //threshold to set MIN DECIBEL BOUND
int DRAW_SONOGRAMS = int.Parse(dict[key_DRAW_SONOGRAMS]); //options to draw sonogram
// iii: Extract the event as TEMPLATE
// #############################################################################################################################################
Log.WriteLine("# Start extracting target event.");
var results = Execute_Extraction(recording, eventStart, eventEnd, minHz, maxHz, frameOverlap, dBThreshold, TimeSpan.Zero);
var sonogram = results.Item1;
var extractedEvent = results.Item2;
var template = results.Item3; // event's matrix of target values before noise removal
var noiseSubband = results.Item4; // event's array of noise values
var templateMinusNoise = results.Item5; // event's matrix of target values after noise removal
Log.WriteLine("# Finished extracting target event.");
// #############################################################################################################################################
// iv: SAVE extracted event as matrix of dB intensity values
FileTools.WriteMatrix2File(template, targetPath.FullName); // write template values to file PRIOR to noise removal.
FileTools.WriteMatrix2File(templateMinusNoise, targetNoNoisePath.FullName); // write template values to file AFTER to noise removal.
FileTools.WriteArray2File(noiseSubband, noisePath.FullName);
// v: SAVE image of extracted event in the original sonogram
DrawSonogram(sonogram, sonogramImagePath.FullName, extractedEvent);
// vi: SAVE extracted event as noise reduced image
// alter matrix dynamic range so user can determine correct dynamic range from image
// matrix = SNR.SetDynamicRange(matrix, 0.0, dynamicRange); // set event's dynamic range
var results1 = BaseSonogram.Data2ImageData(templateMinusNoise);
var targetImage = results1.Item1;
var min = results1.Item2;
var max = results1.Item3;
ImageTools.DrawMatrix(targetImage, 1, 1, targetImagePath.FullName);
// vii: SAVE parameters file
dict.Add(key_SOURCE_DIRECTORY, arguments.Source.DirectoryName);
dict.Add(key_SOURCE_RECORDING, arguments.Source.Name);
dict.Add(key_TEMPLATE_MIN_INTENSITY, min.ToString());
dict.Add(key_TEMPLATE_MAX_INTENSITY, max.ToString());
WriteParamsFile(paramsPath.FullName, dict);
Log.WriteLine("# Finished everything!");
}
/// <summary>
/// Initializes a new instance of the <see cref="SonogramConfig"/> class.
/// CONSTRUCTOR
/// Initialises sonogram config with key-value-pairs in the passed ConfigDictionary
/// </summary>
public SonogramConfig(ConfigDictionary config)
{
this.Initialize(config);
}
/// <summary>
/// 配置设置
/// </summary>
protected virtual void toolStripSetting_Click(object sender, EventArgs e)
{
if (ControlParams.SettingFileName == null)
{
string direct = Application.StartupPath + @"\Setting\";
ControlParams.SettingFileName = direct + EntityOper.Table.TableName + "-配置.xml";
if (!Directory.Exists(direct)) Directory.CreateDirectory(direct);
}
ConfigSetting.CreateForm(ControlParams.SettingFileName).ShowDialog();
//需要将配置文件的字典值进行动态的更新才行,需要设置一个静态变量
Items = ConfigDictionary.Create(ConfigSetting.LoadDic(ControlParams.SettingFileName));//重新加载一遍
}
internal static EngineConfig CreateTestConfig(TestContext testContext, string testDataFile, params string[] testRuleFileNames)
{
var configuration = new EngineConfig {
EngineLog = string.Format("Engine log for {0}.txt", testContext.TestName)
};
#region DataSourceProviderTypes
configuration.AddDataSourceProvider <LcxDataSource>();
configuration.AddDataSourceProvider <ConfigDictionaryDataSource>();
#endregion
#region PropertyAdapterTypes
configuration.AddPropertyAdapter <LocItemPropertyAdapter>();
configuration.AddPropertyAdapter <ConfigDictPropertyAdapter>();
configuration.AddPropertyAdapter <LocResourceSelfPropertyAdapter>(); //Although we don't use it explicitly, LCXLocResourceDataAdapter.LoadObjects() will crash when starting OSLEBot because it expects this property adapter to be available.???
#endregion
#region COAdatpers
configuration.AddCOAdapter <LCXLocResourceDataAdapter>();
#endregion
#region COTypes
configuration.AddClassificationObject <LocResource>();
#endregion
#region DataSourcePackages
var package = new DataSourcePackage(); //This package will contain 2 data sources
package.SetCOType <LocResource>();
var dataSource = new DataSourceInfo();
dataSource.SetSourceType <LocDocument>();
dataSource.SetSourceLocation(testDataFile);
package.AddDataSource(dataSource);
dataSource = new DataSourceInfo();
dataSource.SetSourceType <ConfigDictionary>();
var configDictionary = new ConfigDictionary
{
{ "BuildNumber", "1" },
{ "Project", "test Romka" },
{ "Locgroup", "MacBU" }
};
dataSource.SetSourceLocation(configDictionary);
package.AddDataSource(dataSource);
configuration.AddDataSourcePackage(package);
#endregion
#region Add rules to be tested
foreach (var ruleFileName in testRuleFileNames)
{
configuration.AddRule(
ruleFileName,
string.Empty,
Path.GetExtension(ruleFileName).Equals(".cs", StringComparison.OrdinalIgnoreCase) ? RuleContainerType.Source : RuleContainerType.Module);
}
#endregion
#region And some configuration for dynamic compiler
configuration.AddBinaryReference("System.Core.dll");
configuration.AddBinaryReference("mscorlib.dll");
configuration.AddBinaryReference("System.dll");
configuration.AddBinaryReference("Microsoft.Localization.dll");
configuration.AddBinaryReference("OSLEBotCore.dll");
configuration.AddBinaryReference("LocResource.dll");
configuration.AddBinaryReference("OSLEBot.LCXDataAdapter.dll");
configuration.AddBinaryReference("Logger.dll");
configuration.AddBinaryReference("Cerberus.Core.dll");
configuration.AddBinaryReference("System.Xml.dll");
configuration.AddBinaryReference("System.Xml.Linq.dll");
#endregion
#region Configure output behavior
var outputCfg = new OSLEBot.Core.Output.OutputWriterConfig();
outputCfg.SetDataSourceProvider <LocResourcePerLCLOutputWriter>();
outputCfg.Schema = "OSLEBotOutput.xsd";
outputCfg.Path = testContext.TestDeploymentDir;
outputCfg.AddPropertyToIncludeInOutput("LSResID");
outputCfg.AddPropertyToIncludeInOutput("SourceString");
outputCfg.AddPropertyToIncludeInOutput("Comments");
outputCfg.AddPropertyToIncludeInOutput("TargetString");
outputCfg.AddPropertyToIncludeInOutput("TargetCulture");
outputCfg.AddPropertyToIncludeInOutput("Locgroup");
outputCfg.AddPropertyToIncludeInOutput("Project");
configuration.OutputConfigs.Add(outputCfg);
#endregion
return(configuration);
}
public Tuple <DataTable, DataTable> ProcessCsvFile(FileInfo fiCsvFile, FileInfo fiConfigFile)
{
DataTable dt = CsvTools.ReadCSVToTable(fiCsvFile.FullName, true); //get original data table
if ((dt == null) || (dt.Rows.Count == 0))
{
return(null);
}
//get its column headers
var dtHeaders = new List <string>();
var dtTypes = new List <Type>();
foreach (DataColumn col in dt.Columns)
{
dtHeaders.Add(col.ColumnName);
dtTypes.Add(col.DataType);
}
List <string> displayHeaders = null;
//check if config file contains list of display headers
if (fiConfigFile != null)
{
var configuration = new ConfigDictionary(fiConfigFile.FullName);
Dictionary <string, string> configDict = configuration.GetTable();
if (configDict.ContainsKey(AnalysisKeys.DisplayColumns))
{
displayHeaders = configDict[AnalysisKeys.DisplayColumns].Split(',').ToList();
}
}
//if config file does not exist or does not contain display headers then use the original headers
if (displayHeaders == null)
{
displayHeaders = dtHeaders; //use existing headers if user supplies none.
}
//now determine how to display tracks in display datatable
Type[] displayTypes = new Type[displayHeaders.Count];
bool[] canDisplay = new bool[displayHeaders.Count];
for (int i = 0; i < displayTypes.Length; i++)
{
displayTypes[i] = typeof(double);
canDisplay[i] = false;
if (dtHeaders.Contains(displayHeaders[i]))
{
canDisplay[i] = true;
}
}
DataTable table2Display = DataTableTools.CreateTable(displayHeaders.ToArray(), displayTypes);
foreach (DataRow row in dt.Rows)
{
DataRow newRow = table2Display.NewRow();
for (int i = 0; i < canDisplay.Length; i++)
{
if (canDisplay[i])
{
newRow[displayHeaders[i]] = row[displayHeaders[i]];
}
else
{
newRow[displayHeaders[i]] = 0.0;
}
}
table2Display.Rows.Add(newRow);
}
//order the table if possible
if (dt.Columns.Contains(AnalysisKeys.EventStartAbs))
{
dt = DataTableTools.SortTable(dt, AnalysisKeys.EventStartAbs + " ASC");
}
else if (dt.Columns.Contains(AnalysisKeys.EventCount))
{
dt = DataTableTools.SortTable(dt, AnalysisKeys.EventCount + " ASC");
}
else if (dt.Columns.Contains(AnalysisKeys.KeyRankOrder))
{
dt = DataTableTools.SortTable(dt, AnalysisKeys.KeyRankOrder + " ASC");
}
else if (dt.Columns.Contains(AnalysisKeys.KeyStartMinute))
{
dt = DataTableTools.SortTable(dt, AnalysisKeys.KeyStartMinute + " ASC");
}
//this depracted now that use class indexProperties to do normalisation
//table2Display = NormaliseColumnsOfDataTable(table2Display);
//add in column of weighted indices
//bool addColumnOfweightedIndices = true;
//if (addColumnOfweightedIndices)
//{
// double[] comboWts = IndexCalculate.CalculateComboWeights();
// double[] weightedIndices = IndexCalculate.GetArrayOfWeightedAcousticIndices(dt, comboWts);
// string colName = "WeightedIndex";
// DataTableTools.AddColumnOfDoubles2Table(table2Display, colName, weightedIndices);
//}
return(Tuple.Create(dt, table2Display));
} // ProcessCsvFile()
public static void Execute(Arguments arguments)
{
MainEntry.WarnIfDeveloperEntryUsed();
string title = "# EVENT PATTERN RECOGNITION.";
string date = "# DATE AND TIME: " + DateTime.Now;
Log.WriteLine(title);
Log.WriteLine(date);
Log.Verbosity = 1;
string targetName = arguments.Target; // prefix of name of created files
var input = arguments.Source;
string recordingFileName = input.Name;
string recordingDirectory = input.DirectoryName;
DirectoryInfo outputDir = arguments.Config.ToFileInfo().Directory;
FileInfo targetPath = outputDir.CombineFile(targetName + "_target.txt");
FileInfo targetNoNoisePath = outputDir.CombineFile(targetName + "_targetNoNoise.txt");
FileInfo noisePath = outputDir.CombineFile(targetName + "_noise.txt");
FileInfo targetImagePath = outputDir.CombineFile(targetName + "_target.png");
FileInfo paramsPath = outputDir.CombineFile(targetName + "_params.txt");
Log.WriteIfVerbose("# Output folder =" + outputDir);
//i: GET RECORDING
AudioRecording recording = new AudioRecording(input.FullName);
//if (recording.SampleRate != 22050) recording.ConvertSampleRate22kHz(); THIS METHOD CALL IS OBSOLETE
int sr = recording.SampleRate;
//ii: READ PARAMETER VALUES FROM INI FILE
var config = new ConfigDictionary(arguments.Config);
Dictionary <string, string> dict = config.GetTable();
// framing parameters
//double frameOverlap = FeltTemplates_Use.FeltFrameOverlap; // default = 0.5
double frameOverlap = double.Parse(dict["FRAME_OVERLAP"]);
//frequency band
int minHz = int.Parse(dict["MIN_HZ"]);
int maxHz = int.Parse(dict["MAX_HZ"]);
// oscillation OD parameters
double dctDuration = double.Parse(dict[OscillationRecogniser.key_DCT_DURATION]); // 2.0; // seconds
double dctThreshold = double.Parse(dict[OscillationRecogniser.key_DCT_THRESHOLD]); // 0.5;
int minOscilFreq = int.Parse(dict[OscillationRecogniser.key_MIN_OSCIL_FREQ]); // 4;
int maxOscilFreq = int.Parse(dict[OscillationRecogniser.key_MAX_OSCIL_FREQ]); // 5;
bool normaliseDCT = false;
// iii initialize the sonogram config class.
SonogramConfig sonoConfig = new SonogramConfig(); //default values config
sonoConfig.SourceFName = recording.BaseName;
//sonoConfig.WindowSize = windowSize;
sonoConfig.WindowOverlap = frameOverlap;
// iv: generate the sonogram
BaseSonogram sonogram = new SpectrogramStandard(sonoConfig, recording.WavReader);
Log.WriteLine("Frames: Size={0}, Count={1}, Duration={2:f1}ms, Overlap={5:f2}%, Offset={3:f1}ms, Frames/s={4:f1}",
sonogram.Configuration.WindowSize, sonogram.FrameCount, sonogram.FrameDuration * 1000,
sonogram.FrameStep * 1000, sonogram.FramesPerSecond, frameOverlap);
int binCount = (int)(maxHz / sonogram.FBinWidth) - (int)(minHz / sonogram.FBinWidth) + 1;
Log.WriteIfVerbose("Freq band: {0} Hz - {1} Hz. (Freq bin count = {2})", minHz, maxHz, binCount);
// v: extract the subband energy array
Log.WriteLine("# Start extracting target event.");
double[] dBArray = SNR.DecibelsInSubband(sonogram.Data, minHz, maxHz, sonogram.FBinWidth);
for (int i = 0; i < sonogram.FrameCount; i++)
{
dBArray[i] /= binCount; // get average dB energy
}
double Q = 0.0;
double SD = 0.0;
throw new NotImplementedException("Mike changed the API here, I don't know how to fix it.");
dBArray = new[] { 0.0 }; // SNR.NoiseSubtractMode(dBArray, out Q, out SD);
double maxDB = 6.0;
double dBThreshold = 2 * SD / maxDB; //set dB threshold to 2xSD above background noise
dBArray = SNR.NormaliseDecibelArray_ZeroOne(dBArray, maxDB);
dBArray = DataTools.filterMovingAverage(dBArray, 7);
//Log.WriteLine("Q ={0}", Q);
//Log.WriteLine("SD={0}", SD);
//Log.WriteLine("Th={0}", dBThreshold); //normalised threshhold
// #############################################################################################################################################
// vi: look for oscillation at required OR for ground parrots.
double[] odScores = Oscillations2010.DetectOscillationsInScoreArray(dBArray, dctDuration, sonogram.FramesPerSecond, dctThreshold,
normaliseDCT, minOscilFreq, maxOscilFreq);
//odScores = SNR.NoiseSubtractMode(odScores, out Q, out SD);
double maxOD = 1.0;
odScores = SNR.NormaliseDecibelArray_ZeroOne(odScores, maxOD);
odScores = DataTools.filterMovingAverage(odScores, 5);
//odScores = DataTools.NormaliseMatrixValues(odScores); //NormaliseMatrixValues 0 - 1
//double odThreshold = (10 * SD) / maxOD; //set od threshold to 2xSD above background noise
//double odThreshold = dctThreshold;
double odThreshold = 0.4;
Log.WriteLine("Max={0}", odScores.Max());
//Log.WriteLine("Q ={0}", Q);
//Log.WriteLine("SD ={0}", SD);
Log.WriteLine("Th ={0}", dctThreshold); //normalised threshhold
// #############################################################################################################################################
// vii: LOOK FOR GROUND PARROTS USING TEMPLATE
var template = GroundParrotRecogniser.ReadGroundParrotTemplateAsList(sonogram);
double[] gpScores = DetectEPR(template, sonogram, odScores, odThreshold);
gpScores = DataTools.normalise(gpScores); //NormaliseMatrixValues 0 - 1
// #############################################################################################################################################
// iv: SAVE extracted event as matrix of dB intensity values
//FileTools.WriteMatrix2File(template, targetPath); // write template values to file PRIOR to noise removal.
//FileTools.WriteMatrix2File(templateMinusNoise, targetNoNoisePath); // write template values to file AFTER to noise removal.
//FileTools.WriteArray2File(noiseSubband, noisePath);
// v: SAVE image of extracted event in the original sonogram
string sonogramImagePath = outputDir + Path.GetFileNameWithoutExtension(recordingFileName) + ".png";
//DrawSonogram(sonogram, sonogramImagePath, dBArray, dBThreshold / maxDB, odScores, dctThreshold, gpScores, template);
}
public static void Execute(Arguments arguments)
{
MainEntry.WarnIfDevleoperEntryUsed();
string date = "# DATE AND TIME: " + DateTime.Now;
Log.WriteLine("# SEGMENTING A RECORDING");
Log.WriteLine(date);
Log.Verbosity = 1;
FileInfo recordingPath = arguments.Source;
FileInfo iniPath = arguments.Config.ToFileInfo();
DirectoryInfo outputDir = arguments.Output;
string opFName = "segment-output.txt";
FileInfo opPath = outputDir.CombineFile(opFName);
Log.WriteIfVerbose("# Output folder =" + outputDir);
//READ PARAMETER VALUES FROM INI FILE
var config = new ConfigDictionary(iniPath);
Dictionary <string, string> dict = config.GetTable();
Dictionary <string, string> .KeyCollection keys = dict.Keys;
int minHz = int.Parse(dict[key_MIN_HZ]);
int maxHz = int.Parse(dict[key_MAX_HZ]);
double frameOverlap = double.Parse(dict[key_FRAME_OVERLAP]);
double smoothWindow = double.Parse(dict[key_SMOOTH_WINDOW]); //smoothing window (seconds) before segmentation
double thresholdSD = double.Parse(dict[key_THRESHOLD]); //segmentation threshold in noise SD
double minDuration = double.Parse(dict[key_MIN_DURATION]); //min duration of segment & width of smoothing window in seconds
double maxDuration = double.Parse(dict[key_MAX_DURATION]); //max duration of segment in seconds
int DRAW_SONOGRAMS = int.Parse(dict[key_DRAW_SONOGRAMS]); //options to draw sonogram
Log.WriteIfVerbose("# Freq band: {0} Hz - {1} Hz.)", minHz, maxHz);
Log.WriteIfVerbose("# Smoothing Window: {0}s.", smoothWindow);
Log.WriteIfVerbose("# Duration bounds: " + minDuration + " - " + maxDuration + " seconds");
//#############################################################################################################################################
var results = Execute_Segmentation(recordingPath, minHz, maxHz, frameOverlap, smoothWindow, thresholdSD, minDuration, maxDuration);
Log.WriteLine("# Finished detecting segments.");
//#############################################################################################################################################
var sonogram = results.Item1;
var predictedEvents = results.Item2; //contain the segments detected
var Q = results.Item3;
var oneSD_dB = results.Item4;
var dBThreshold = results.Item5;
var intensity = results.Item6;
Log.WriteLine("# Signal: Duration={0}, Sample Rate={1}", sonogram.Duration, sonogram.SampleRate);
Log.WriteLine("# Frames: Size={0}, Count={1}, Duration={2:f1}ms, Overlap={5:f0}%, Offset={3:f1}ms, Frames/s={4:f1}",
sonogram.Configuration.WindowSize, sonogram.FrameCount, sonogram.FrameDuration * 1000,
sonogram.FrameStep * 1000, sonogram.FramesPerSecond, frameOverlap);
int binCount = (int)(maxHz / sonogram.FBinWidth) - (int)(minHz / sonogram.FBinWidth) + 1;
Log.WriteLine("# FreqBand: {0} Hz - {1} Hz. (Freq bin count = {2})", minHz, maxHz, binCount);
Log.WriteLine("# Intensity array - noise removal: Q={0:f1}dB. 1SD={1:f3}dB. Threshold={2:f3}dB.", Q, oneSD_dB, dBThreshold);
Log.WriteLine("# Events: Count={0}", predictedEvents.Count());
int pcHIF = 100;
if (intensity != null)
{
int hifCount = intensity.Count(p => p > dBThreshold); //count of high intensity frames
pcHIF = 100 * hifCount / sonogram.FrameCount;
}
//write event count to results file.
double sigDuration = sonogram.Duration.TotalSeconds;
string fname = recordingPath.Name;
int count = predictedEvents.Count;
//string str = String.Format("#RecordingName\tDuration(sec)\t#Ev\tCompT(ms)\t%hiFrames\n{0}\t{1}\t{2}\t{3}\t{4}\n", fname, sigDuration, count, analysisDuration.TotalMilliseconds, pcHIF);
//StringBuilder sb = new StringBuilder(str);
//StringBuilder sb = new StringBuilder();
string str = string.Format("{0}\t{1}\t{2}\t{3}", fname, sigDuration, count, pcHIF);
StringBuilder sb = AcousticEvent.WriteEvents(predictedEvents, str);
FileTools.WriteTextFile(opPath.FullName, sb.ToString());
//draw images of sonograms
string imagePath = outputDir + Path.GetFileNameWithoutExtension(recordingPath.Name) + ".png";
double min, max;
DataTools.MinMax(intensity, out min, out max);
double threshold_norm = dBThreshold / max; //min = 0.0;
intensity = DataTools.normalise(intensity);
if (DRAW_SONOGRAMS == 2)
{
DrawSonogram(sonogram, imagePath, predictedEvents, threshold_norm, intensity);
}
else
if (DRAW_SONOGRAMS == 1 && predictedEvents.Count > 0)
{
DrawSonogram(sonogram, imagePath, predictedEvents, threshold_norm, intensity);
}
Log.WriteLine("# Finished recording:- " + recordingPath.Name);
}
/// <summary>
/// Builds OSLEBot configuration based on file list and Cerberus check configuration.
/// </summary>
/// <param name="fileList">List of files to be converted into OSLEBot data sources.</param>
/// <param name="checkConfigs">List of checks that will be executed against the data sources.</param>
/// <param name="cerberusOutputPath">Full path to Cerberus output XML file.</param>
/// <returns>Configuration data that can be passed into OSLEBot for execution.</returns>
internal static EngineConfig CreateConfig(IEnumerable <InputFileItem> fileList, IEnumerable <CheckConfig> checkConfigs, string cerberusOutputPath, string oslebotEngineLogPath)
{
var configuration = new EngineConfig {
EngineLog = oslebotEngineLogPath
};
#region DataSourceProviderTypes
configuration.AddDataSourceProvider <LcxDataSource>();
configuration.AddDataSourceProvider <ConfigDictionaryDataSource>();
#endregion
#region PropertyAdapterTypes
configuration.AddPropertyAdapter <LocItemPropertyAdapter>();
configuration.AddPropertyAdapter <ConfigDictPropertyAdapter>();
configuration.AddPropertyAdapter <LocResourceSelfPropertyAdapter>(); //Although we don't use it explicitly, LCXLocResourceDataAdapter.LoadObjects() will crash when starting OSLEBot because it expects this property adapter to be available.???
#endregion
#region COAdatpers
configuration.AddCOAdapter <LCXLocResourceDataAdapter>();
#endregion
#region COTypes
configuration.AddClassificationObject <LocResource>();
#endregion
#region DataSourcePackages
foreach (var file in fileList)
{
var package = new DataSourcePackage(); //This package will contain 2 data sources
package.SetCOType <LocResource>();
var dataSource = new DataSourceInfo();
dataSource.SetSourceType <LocDocument>();
dataSource.SetSourceLocation(file.File);
package.AddDataSource(dataSource);
dataSource = new DataSourceInfo();
dataSource.SetSourceType <ConfigDictionary>();
var staticProperties = new ConfigDictionary
{
{ "BuildNumber", file.BuildNumber },
{ "Project", file.Project },
{ "Locgroup", file.LocGroup }
};
dataSource.SetSourceLocation(staticProperties);
package.AddDataSource(dataSource);
configuration.AddDataSourcePackage(package);
}
#endregion
#region Add checks
// only add checks that are not globally disabled (could never execute)
foreach (var check in checkConfigs.Where(ch => !ch.IsGloballyDisabled))
{
configuration.AddRule(check.PhysicalFile, check.GetOSLEBotFilteringExpression(), check.ContainerType);
}
#region And some configuration for dynamic compiler
var executingAssemblyPath = Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location);
// figure out the directory where LocStudio libs are located
var locstudioLibsPath = Path.GetDirectoryName(typeof(LocDocument).Assembly.Location);
// figure out the directory where OSLEBot libs are located
var oslebotLibsPath = Path.GetDirectoryName(typeof(OSLEBotEngine).Assembly.Location);
configuration.AddBinaryReference("System.Core.dll");
configuration.AddBinaryReference("mscorlib.dll");
configuration.AddBinaryReference("System.dll");
configuration.AddBinaryReference("System.Xml.dll");
configuration.AddBinaryReference("System.Xml.Linq.dll");
configuration.AddBinaryReference(Path.Combine(locstudioLibsPath, "Microsoft.Localization.dll"));
configuration.AddBinaryReference(Path.Combine(oslebotLibsPath, "OSLEBotCore.dll"));
configuration.AddBinaryReference(Path.Combine(oslebotLibsPath, "LocResource.dll"));
configuration.AddBinaryReference(Path.Combine(oslebotLibsPath, "OSLEBot.LCXDataAdapter.dll"));
configuration.AddBinaryReference(Path.Combine(executingAssemblyPath, "Cerberus.Core.dll"));
#endregion
#endregion
#region Configure output behavior
// define all properties here to be consistent for all output writers
string[] propertiesToInclude =
{
"LSResID",
"SourceString",
"TargetString",
"Comments",
"TargetCulture",
"Locgroup",
"Project",
"LcxFileName"
};
Microsoft.Localization.OSLEBot.Core.Output.OutputWriterConfig outputCfg;
// set up output writer that creates one merged output for all files
outputCfg = new OSLEBot.Core.Output.OutputWriterConfig();
outputCfg.SetDataSourceProvider <Microsoft.Localization.OSLEBot.Core.Output.Specialized.XMLDOMOutputWriter>();
outputCfg.Schema = "OSLEBotOutput.xsd";
outputCfg.Path = cerberusOutputPath;
Array.ForEach(propertiesToInclude, outputCfg.AddPropertyToIncludeInOutput);
configuration.OutputConfigs.Add(outputCfg);
// set up output writer that creates one output file per LCX processed.
outputCfg = new OSLEBot.Core.Output.OutputWriterConfig();
outputCfg.SetDataSourceProvider <LocResourcePerLCLOutputWriter>();
outputCfg.Schema = "OSLEBotOutput.xsd";
outputCfg.Path = System.IO.Path.GetDirectoryName(System.Reflection.Assembly.GetExecutingAssembly().Location);
Array.ForEach(propertiesToInclude, outputCfg.AddPropertyToIncludeInOutput);
configuration.OutputConfigs.Add(outputCfg);
#endregion
#region Because we don't store LSBuild enviornment, point compiler to that of Office14 source depot
configuration.AddAssemblyResolverPaths(locstudioLibsPath);
#endregion
return(configuration);
}
/// <summary>
/// Wraps up the resources into a template.ZIP file
/// and then runs a test on the source recording.
/// </summary>
public static void Execute(Arguments arguments)
{
if (arguments == null)
{
arguments = Dev();
}
string title = "# EDIT TEMPLATE.";
string date = "# DATE AND TIME: " + DateTime.Now;
Log.WriteLine(title);
Log.WriteLine(date);
//ZIP THE OUTPUT FILES
const bool ZipOutput = true;
FileInfo iniPath = arguments.Config;
string[] nameComponents = (Path.GetFileNameWithoutExtension(iniPath.Name)).Split('_');
string targetName = nameComponents[0] + "_" + nameComponents[1];
//i: Set up the file names
DirectoryInfo outputDir = iniPath.Directory;
FileInfo targetImagePath = outputDir.CombineFile(targetName + "_target.png"); // input image file
FileInfo binaryOpPath = outputDir.CombineFile(targetName + "_binary.bmp");
FileInfo trinaryOpPath = outputDir.CombineFile(targetName + "_trinary.bmp");
FileInfo sprOpPath = outputDir.CombineFile(targetName + "_spr.txt"); // syntactic pattern recognition file
//additional files to be zipped up with template
FileInfo targetPath = outputDir.CombineFile(targetName + "_target.txt");
FileInfo targetNoNoisePath = outputDir.CombineFile(targetName + "_targetNoNoise.txt");
FileInfo noisePath = outputDir.CombineFile(targetName + "_noise.txt");
//ii: READ PARAMETER VALUES FROM INI FILE
var config = new ConfigDictionary(iniPath);
Dictionary <string, string> dict = config.GetTable();
string sourceFile = dict[FeltTemplate_Create.key_SOURCE_RECORDING];
string sourceDir = dict[FeltTemplate_Create.key_SOURCE_DIRECTORY];
double dB_Threshold = double.Parse(dict[FeltTemplate_Create.key_DECIBEL_THRESHOLD]);
double maxTemplateIntensity = double.Parse(dict[FeltTemplate_Create.key_TEMPLATE_MAX_INTENSITY]);
int neighbourhood = int.Parse(dict[FeltTemplate_Create.key_DONT_CARE_NH]); //the do not care neighbourhood
int lineLength = int.Parse(dict[FeltTemplate_Create.key_LINE_LENGTH]);
double templateThreshold = dB_Threshold / maxTemplateIntensity;
int bitmapThreshold = (int)(255 - (templateThreshold * 255));
Log.WriteLine("#################################### WRITE THE BINARY TEMPLATE ##################################");
Bitmap bitmap = ImageTools.ReadImage2Bitmap(targetImagePath.FullName);
var binaryBmp = Image2BinaryBitmap(bitmap, bitmapThreshold);
binaryBmp.Save(binaryOpPath.FullName);
Log.WriteLine("#################################### WRITE THE TRINARY TEMPLATE ##################################");
var trinaryBmp = Image2TrinaryBitmap(binaryBmp, neighbourhood);
trinaryBmp.Save(trinaryOpPath.FullName);
Log.WriteLine("#################################### WRITE THE SPR TEMPLATE ##################################");
double[,] matrix = ImageTools.GreyScaleImage2Matrix(bitmap);
matrix = DataTools.MatrixRotate90Clockwise(matrix); //rows=time cols=freq.
//ImageTools.DrawMatrix(matrix, @"C:\SensorNetworks\Output\FELT_LewinsRail1\SPR_output1.bmp");
//int smallLengthThreshold = 10;
//var tuple = SPT.doSPT(matrix, templateThreshold, smallLengthThreshold);
//matrix = tuple.Item1;
//ImageTools.DrawMatrix(matrix, @"C:\SensorNetworks\Output\FELT_LewinsRail1\SPR_output2.bmp");
char[,] spr = SprTools.Target2SymbolicTracks(matrix, templateThreshold, lineLength);
FileTools.WriteMatrix2File(spr, sprOpPath.FullName);
var tuple1 = FindMatchingEvents.Execute_One_Spr_Match(spr, matrix, templateThreshold);
double sprScore = tuple1.Item1;
double dBScore = sprScore * maxTemplateIntensity;
Log.WriteLine("#################################### WRITE THE OSCILATION TEMPLATE ##################################");
double[,] target = FileTools.ReadDoubles2Matrix(targetPath.FullName);
// oscillations in time
double[,] rotatedtarget = DataTools.MatrixRotate90Clockwise(target);
var colSums = DataTools.GetColumnsAverages(rotatedtarget);
double[] periods = Oscillations2010.PeriodicityAnalysis(colSums); // frame periodicity
LoggedConsole.WriteLine("Periodicity (sec) = {0:f3}, {1:f3}, {2:f3}",
periods[0] * FeltTemplates_Use.FeltFrameOffset, periods[1] * FeltTemplates_Use.FeltFrameOffset, periods[2] * FeltTemplates_Use.FeltFrameOffset);
//double oscilFreq = indexOfMaxValue / dctDuration * 0.5; //Times 0.5 because index = Pi and not 2Pi
// oscillations in freq i.e. harmonics
colSums = DataTools.GetColumnsAverages(target);
periods = Oscillations2010.PeriodicityAnalysis(colSums);
LoggedConsole.WriteLine("Periodicity (Hz) = {0:f0}, {1:f0}, {2:f0}.",
periods[0] * FeltTemplates_Use.FeltFreqBinWidth, periods[1] * FeltTemplates_Use.FeltFreqBinWidth, periods[2] * FeltTemplates_Use.FeltFreqBinWidth);
//double oscilFreq = indexOfMaxValue / dctDuration * 0.5; //Times 0.5 because index = Pi and not 2Pi
//FileTools.WriteMatrix2File(spr, sprOpPath);
// ZIP THE OUTPUT
Log.WriteLine("#################################### ZIP THE TEMPLATES ##################################");
if (ZipOutput == true)
{
var filenames = new[]
{ iniPath, targetImagePath, binaryOpPath, targetPath, targetNoNoisePath, noisePath };
string biOutZipFile = outputDir + targetName + "_binaryTemplate.zip";
//FileTools.ZipFiles(filenames, biOutZipFile);
filenames = new[]
{ iniPath, targetImagePath, trinaryOpPath, targetPath, targetNoNoisePath, noisePath };
string triOutZipFile = outputDir + targetName + "_trinaryTemplate.zip";
//FileTools.ZipFiles(filenames, triOutZipFile);
filenames = new[]
{ iniPath, targetImagePath, sprOpPath, targetPath, targetNoNoisePath, noisePath };
string sprOutZipFile = outputDir + targetName + "_syntacticTemplate.zip";
//FileTools.ZipFiles(filenames, sprOutZipFile);
// Zipping files can be done by using standard .NET 4.6 System.IO.Compression
// however, this code hasn't been used in years and I've opted to just comment out the lines above
throw new NotImplementedException("Zipping template files intentionally broken");
}
Log.WriteLine("\n\n#################################### TEST THE EXTRACTED EVENT ON SOURCE FILE ##################################");
//vi: TEST THE EVENT ON ANOTHER FILE
//felt "C:\SensorNetworks\WavFiles\Canetoad\DM420010_128m_00s__130m_00s - Toads.mp3" C:\SensorNetworks\Output\FELT_CaneToad\FELT_CaneToad_Params.txt events.txt
//string testRecording = @"C:\SensorNetworks\WavFiles\Gecko\Suburban_March2010\geckos_suburban_104.mp3";
//string testRecording = @"C:\SensorNetworks\WavFiles\Gecko\Suburban_March2010\geckos_suburban_18.mp3";
//string testRecording = @"C:\SensorNetworks\WavFiles\Currawongs\Currawong_JasonTagged\West_Knoll_Bees_20091102-170000.mp3";
//string testRecording = @"C:\SensorNetworks\WavFiles\Curlew\Curlew2\Top_Knoll_-_St_Bees_20090517-210000.wav";
string listOpDir = "C:\\SensorNetworks\\Output\\FELT_templateList\\";
string templateListPath = listOpDir + "templateTestList.txt";
var list = new List <string>();
list.Add("#" + outputDir + targetName + "_binaryTemplate.zip");
list.Add("#" + outputDir + targetName + "_trinaryTemplate.zip");
list.Add("#" + outputDir + targetName + "_syntacticTemplate.zip");
FileTools.Append2TextFile(templateListPath, list); //write the template.ZIP file
//TEST THE TEMPLATE ON SOURCE FILE
//string[] arguments = new string[3];
/*
* var args = new FeltTemplates_Use.Arguments()
* {
* Source = "",
* arguments[0] = sourceDir + "\\" + sourceFile;
* arguments[1] = templateListPath;
* arguments[2] = listOpDir;
*
* }*/
//FeltTemplates_Use.Dev(arguments);
Log.WriteLine("# Finished everything!");
}
/// <summary>
///
/// </summary>
/// <returns></returns>
public static Image_MultiTrack Sonogram2MultiTrackImage(BaseSonogram sonogram, Dictionary <string, string> configDict)
{
bool doHighlightSubband = false;
//check if doing a reduced sonogram
//int timeReductionFactor = 1;
//if (configDict.ContainsKey(Keys.TIME_REDUCTION_FACTOR))
// timeReductionFactor = ConfigDictionary.GetInt(Keys.TIME_REDUCTION_FACTOR, configDict);
//int freqReductionFactor = 1;
//if (configDict.ContainsKey(Keys.FREQ_REDUCTION_FACTOR))
// freqReductionFactor = ConfigDictionary.GetInt(Keys.FREQ_REDUCTION_FACTOR, configDict);
//if (!((timeReductionFactor == 1) && (freqReductionFactor == 1)))
//{
// sonogram.Data = ReduceDimensionalityOfSpectrogram(sonogram.Data, timeReductionFactor, freqReductionFactor);
// return sonogram.GetImage(doHighlightSubband, add1kHzLines);
//}
// (iii) NOISE REDUCTION
//bool doNoiseReduction = false;
//if (configDict.ContainsKey(AnalysisKeys.NoiseDoReduction))
// doNoiseReduction = ConfigDictionary.GetBoolean(AnalysisKeys.NoiseDoReduction, configDict);
//if (doNoiseReduction)
//{
// //LoggedConsole.WriteLine("PERFORMING NOISE REDUCTION");
// double bgThreshold = 3.0;
// if (configDict.ContainsKey(AnalysisKeys.NoiseBgThreshold))
// bgThreshold = ConfigDictionary.GetDouble(AnalysisKeys.NoiseBgThreshold, configDict);
// var tuple = SNR.NoiseReduce(sonogram.Data, NoiseReductionType.STANDARD, bgThreshold);
// sonogram.Data = tuple.Item1; // store data matrix
//}
//ADD time and frequency scales
bool addScale = false;
if (configDict.ContainsKey(AnalysisKeys.AddTimeScale))
{
addScale = ConfigDictionary.GetBoolean(AnalysisKeys.AddTimeScale, configDict);
}
else
if (configDict.ContainsKey(AnalysisKeys.AddAxes))
{
addScale = ConfigDictionary.GetBoolean(AnalysisKeys.AddAxes, configDict);
}
Image img = sonogram.GetImage(doHighlightSubband, add1KHzLines: addScale, doMelScale: false);
Image_MultiTrack mti = new Image_MultiTrack(img);
if (addScale)
{
mti.AddTrack(ImageTrack.GetTimeTrack(sonogram.Duration, sonogram.FramesPerSecond)); //add time scale
}
bool addSegmentationTrack = false;
//add segmentation track
if (configDict.ContainsKey(AnalysisKeys.AddSegmentationTrack))
{
addSegmentationTrack = ConfigDictionary.GetBoolean(AnalysisKeys.AddSegmentationTrack, configDict);
}
if (addSegmentationTrack)
{
mti.AddTrack(ImageTrack.GetSegmentationTrack(sonogram)); //add segmentation track
}
return(mti);
//mti.AddTrack(ImageTrack.GetWavEnvelopeTrack(sonogram)); //add segmentation track
}//Sonogram2MultiTrackImage()
private string[] RefreshPlatforms(string keyChain, ConfigDictionary refreshedConfigSet, ConfigDictionary prevConfigSet)
{
List <string> childKeys = new List <string>();
List <BuildPlatform> platforms = BuildSettings.platformList.platforms;
for (int i = 0; i < platforms.Count; i++)
{
// Skip if platform is disabled or if it doesn't have any enabled architectures.
if (!platforms[i].enabled || !platforms[i].atLeastOneArch)
{
continue;
}
string key = keyChain + "/" + platforms[i].platformName;
Configuration relConfig = new Configuration();
// Check for duplicate key.
if (refreshedConfigSet.ContainsKey(key))
{
continue;
}
// Copy previous settings if they exist.
if (prevConfigSet != null && prevConfigSet.ContainsKey(key))
{
relConfig.enabled = prevConfigSet[key].enabled;
}
// Refresh architectures.
BuildArchitecture[] architectures = platforms[i].architectures;
if (architectures.Length > 0)
{
relConfig.childKeys = RefreshArchitectures(key, refreshedConfigSet, platforms[i].variantKey, architectures, platforms[i].distributionList.distributions, prevConfigSet);
}
// Scan ahead for other versions of this platform with different variants.
for (int j = i; j < platforms.Count; j++)
{
BuildPlatform otherPlatform = platforms[j];
if (otherPlatform.platformName == platforms[i].platformName && otherPlatform.enabled && otherPlatform.atLeastOneArch)
{
List <string> currentKeys = new List <string>(relConfig.childKeys);
string[] additionalKeys = RefreshArchitectures(key, refreshedConfigSet, otherPlatform.variantKey, otherPlatform.architectures, otherPlatform.distributionList.distributions, prevConfigSet);
for (int k = 0; k < additionalKeys.Length; k++)
{
if (!currentKeys.Contains(additionalKeys[k]))
{
currentKeys.Add(additionalKeys[k]);
}
}
relConfig.childKeys = currentKeys.ToArray();
}
}
// Save configuration.
refreshedConfigSet.Add(key, relConfig);
// Add key to list to send back to parent.
childKeys.Add(key);
}
return(childKeys.ToArray());
}
/// <summary>
///
/// </summary>
/// <param name="fiAudio"></param>
/// <param name="fiConfig"></param>
/// <param name="fiImage"></param>
/// <returns></returns>
public static Image GetImageFromAudioSegment(FileInfo fiAudio, FileInfo fiConfig, FileInfo fiImage, IAnalyser2 analyser)
{
var config = new ConfigDictionary(fiConfig.FullName); //read in config file
bool doAnnotate = config.GetBoolean(AnalysisKeys.AnnotateSonogram);
//bool doNoiseReduction = config.GetBoolean(Keys.NOISE_DO_REDUCTION);
//double bgNoiseThreshold = config.GetDouble(Keys.NOISE_BG_REDUCTION);
var diOutputDir = new DirectoryInfo(Path.GetDirectoryName(fiImage.FullName));
//Image image = null;
if (doAnnotate)
{
if (analyser == null)
{
string analyisName = config.GetString(AnalysisKeys.AnalysisName);
LoggedConsole.WriteLine("\nWARNING: Could not construct annotated image because analysis name not recognized:");
LoggedConsole.WriteLine("\t " + analyisName);
return(null);
}
throw new NotSupportedException("Code intentionally broken because it is out of date and not used");
/*
* Image image = null;
* var settings = new AnalysisSettings
* {
* ConfigDict = config.GetDictionary(),
* SegmentAudioFile = fiAudio,
* ConfigFile = fiConfig,
* SegmentImageFile = fiImage,
* SegmentOutputDirectory = diOutputDir
* };
*
* // want to pass SampleRate of the original file.
* settings.SampleRateOfOriginalAudioFile = int.Parse(settings.ConfigDict[AnalysisKeys.ResampleRate]);
*
* analyser.BeforeAnalyze(settings);
*
* var results = analyser.Analyze(settings, new SegmentSettings<FileInfo>(se));
*
* image = results.ImageFile == null ? null : Image.FromFile(results.ImageFile.FullName);
*
* analyser = null;
* return image;*/
}
else
{
analyser = null;
var configDict = config.GetDictionary();
BaseSonogram sonogram = Audio2DecibelSonogram(fiAudio, configDict);
var mti = Sonogram2MultiTrackImage(sonogram, configDict);
var image = mti.GetImage();
if (image != null)
{
if (fiImage.Exists)
{
fiImage.Delete();
}
image.Save(fiImage.FullName, ImageFormat.Png);
}
return(image);
}
}
private string[] RefreshDistributions(string keyChain, ConfigDictionary refreshedConfigSet, BuildDistribution[] distributions, ConfigDictionary prevConfigSet)
{
List <string> childKeys = new List <string>();
for (int i = 0; i < distributions.Length; i++)
{
if (!distributions[i].enabled)
{
continue;
}
string key = keyChain + "/" + distributions[i].distributionName;
Configuration relConfig = new Configuration();
if (refreshedConfigSet.ContainsKey(key))
{
continue;
}
if (prevConfigSet != null && prevConfigSet.ContainsKey(key))
{
relConfig.enabled = prevConfigSet[key].enabled;
}
refreshedConfigSet.Add(key, relConfig);
// Add key to list to send back to parent.
childKeys.Add(key);
}
return(childKeys.ToArray());
}
public static void MakeSonogramWithSox(FileInfo fiAudio, Dictionary <string, string> configDict, FileInfo output)
{
var soxPath = new FileInfo(AppConfigHelper.SoxExe);
if (!soxPath.Exists)
{
LoggedConsole.WriteLine("SOX ERROR: Path does not exist: <{0}>", soxPath.FullName);
throw new FileNotFoundException("SOX ERROR: Path for executable does not exist.", soxPath.FullName);
}
// must quote the path because has a space in it.
string soxCmd = "\"" + AppConfigHelper.SoxExe + "\"";
string title = string.Empty;
if (configDict.ContainsKey(AnalysisKeys.SonogramTitle))
{
title = " -t " + configDict[AnalysisKeys.SonogramTitle];
}
string comment = string.Empty;
if (configDict.ContainsKey(AnalysisKeys.SonogramComment))
{
comment = " -c " + configDict[AnalysisKeys.SonogramComment];
}
string axes = "-r";
if (configDict.ContainsKey(AnalysisKeys.AddAxes) && !ConfigDictionary.GetBoolean(AnalysisKeys.AddAxes, configDict))
{
axes = string.Empty;
}
string coloured = " -m "; // default
if (configDict.ContainsKey(AnalysisKeys.SonogramColored) && ConfigDictionary.GetBoolean(AnalysisKeys.SonogramColored, configDict))
{
coloured = string.Empty;
}
string quantisation = " -q 64 "; // default
if (configDict.ContainsKey(AnalysisKeys.SonogramQuantisation))
{
quantisation = " -q " + ConfigDictionary.GetInt(AnalysisKeys.SonogramQuantisation, configDict);
}
// Path\sox.exe -V "sourcefile.wav" -n rate 22050 spectrogram -m -r -l -a -q 249 -w hann -y 257 -X 43.06640625 -z 100 -o "imagefile.png"
//string soxCommandLineArguments = " -V \"{0}\" -n rate 22050 spectrogram -m -r -l -a -q 249 -w hann -y 257 -X 43.06640625 -z 100 -o \"{1}\""; //greyscale only
//string soxCommandLineArguments = " -V \"{0}\" -n spectrogram -m -l -o \"{1}\""; //greyscale with time, freq and intensity scales
//string soxCommandLineArguments = " -V \"{0}\" -n spectrogram -m -o \"{1}\""; //reverse image greyscale with time, freq and intensity scales
//string soxCommandLineArguments = " -V \"{0}\" -n spectrogram -l -o \"{1}\""; //colour with time, freq and intensity scales
//string soxCommandLineArguments = " -V \"{0}\" -n spectrogram -m -q 64 -r -l -o \"{6}\""; //64 grey scale, with time, freq and intensity scales
const string SoxCommandLineArguments = " -V \"{0}\" -n spectrogram -m {1} -q 64 -l -o \"{6}\""; //64 grey scale, with time, freq and intensity scales
//string soxCommandLineArguments = " -V \"{0}\" -n spectrogram -l {1} {2} {3} {4} {5} -o \"{6}\""; //64 grey scale, with time, freq and intensity scales
// FOR COMMAND LINE OPTIONS SEE: http://sox.sourceforge.net/sox.html
// −a Suppress display of axis lines. This is sometimes useful in helping to discern artefacts at the spectrogram edges.
// -l Print firendly monochrome spectrogram.
// −m Creates a monochrome spectrogram (the default is colour).
// -q Number of intensity quanitisation levels/colors - try -q 64
// −r Raw spectrogram: suppress the display of axes and legends.
// −t text Set the image title - text to display above the spectrogram.
// −c text Set (or clear) the image comment - text to display below and to the left of the spectrogram.
// trim 20 30 displays spectrogram of 30 seconds duratoin starting at 20 seconds.
var args = string.Format(SoxCommandLineArguments, fiAudio.FullName, title, comment, axes, coloured, quantisation, output.FullName);
using (var process = new ProcessRunner(soxCmd))
{
process.Run(args, output.DirectoryName);
}
}
public void MultipleKeysAndSubsidiary()
{
var s = @"
# comment
key1:value11
value12
key1:value21
value22
value23
key1:value31
value32
value33
value34
";
var dict = new ConfigDictionary(new StringReader(s));
Assert.IsTrue(dict.ContainsKey("key1"));
var cv = dict["key1"];
Assert.AreEqual("value11", cv.Value);
CollectionAssert.AreEqual(new[] { "value12" }, cv.Subsididary);
cv = cv.NextValue();
Assert.AreEqual("value21", cv.Value);
CollectionAssert.AreEqual(new[] { "value22", "value23" }, cv.Subsididary);
cv = cv.NextValue();
Assert.AreEqual("value31", cv.Value);
CollectionAssert.AreEqual(new[] { "value32", "value33", "value34" }, cv.Subsididary);
cv = cv.NextValue();
Assert.IsNull(cv);
var array = dict["key1"].Collect();
Assert.AreEqual(3, array.Count);
Assert.AreEqual("value11", array[0].Value);
Assert.AreEqual("value21", array[1].Value);
Assert.AreEqual("value31", array[2].Value);
CollectionAssert.AreEqual(new[] { "value12" }, array[0].Subsididary);
CollectionAssert.AreEqual(new[] { "value22", "value23" }, array[1].Subsididary);
CollectionAssert.AreEqual(new[] { "value32", "value33", "value34" }, array[2].Subsididary);
Dump(dict);
}
public static void Execute(Arguments arguments)
{
string date = "# DATE AND TIME: " + DateTime.Now;
Log.WriteLine("# DETECTING LOW FREQUENCY AMPLITUDE OSCILLATIONS");
Log.WriteLine(date);
Log.Verbosity = 1;
FileInfo recordingFile = arguments.Source;
FileInfo iniPath = arguments.Config.ToFileInfo();
DirectoryInfo outputDir = arguments.Output;
string opFName = "OcillationReconiserResults.cav";
string opPath = outputDir + opFName;
Log.WriteIfVerbose("# Output folder =" + outputDir);
//READ PARAMETER VALUES FROM INI FILE
var config = new ConfigDictionary(iniPath);
Dictionary <string, string> dict = config.GetTable();
Dictionary <string, string> .KeyCollection keys = dict.Keys;
bool doSegmentation = bool.Parse(dict[Key_DO_SEGMENTATION]);
int minHz = int.Parse(dict[Key_MIN_HZ]);
int maxHz = int.Parse(dict[Key_MAX_HZ]);
double frameOverlap = double.Parse(dict[Key_FRAME_OVERLAP]);
double dctDuration = double.Parse(dict[Key_DCT_DURATION]); //duration of DCT in seconds
double dctThreshold = double.Parse(dict[Key_DCT_THRESHOLD]); //minimum acceptable value of a DCT coefficient
int minOscilFreq = int.Parse(dict[Key_MIN_OSCIL_FREQ]); //ignore oscillations below this threshold freq
int maxOscilFreq = int.Parse(dict[Key_MAX_OSCIL_FREQ]); //ignore oscillations above this threshold freq
double minDuration = double.Parse(dict[Key_MIN_DURATION]); //min duration of event in seconds
double maxDuration = double.Parse(dict[Key_MAX_DURATION]); //max duration of event in seconds
double eventThreshold = double.Parse(dict[Key_EVENT_THRESHOLD]); //min score for an acceptable event
int DRAW_SONOGRAMS = int.Parse(dict[Key_DRAW_SONOGRAMS]); //options to draw sonogram
Log.WriteIfVerbose("Freq band: {0} Hz - {1} Hz.)", minHz, maxHz);
Log.WriteIfVerbose("Oscill bounds: " + minOscilFreq + " - " + maxOscilFreq + " Hz");
Log.WriteIfVerbose("minAmplitude = " + dctThreshold);
Log.WriteIfVerbose("Duration bounds: " + minDuration + " - " + maxDuration + " seconds");
//#############################################################################################################################################
var results = Execute_ODDetect(
recordingFile,
doSegmentation,
minHz,
maxHz,
frameOverlap,
dctDuration,
dctThreshold,
minOscilFreq,
maxOscilFreq,
eventThreshold,
minDuration,
maxDuration);
Log.WriteLine("# Finished detecting oscillation events.");
//#############################################################################################################################################
var sonogram = results.Item1;
var hits = results.Item2;
var scores = results.Item3;
var predictedEvents = results.Item4;
var intensity = results.Item5;
var analysisDuration = results.Item6;
Log.WriteLine("# Event Count = " + predictedEvents.Count());
int pcHIF = 100;
if (intensity != null)
{
int hifCount = intensity.Count(p => p >= 0.001); //count of high intensity frames
pcHIF = 100 * hifCount / sonogram.FrameCount;
}
// write event count to results file.
string fname = recordingFile.BaseName();
Csv.WriteToCsv(opPath.ToFileInfo(), predictedEvents);
//draw images of sonograms
string imagePath = outputDir + fname + ".png";
if (DRAW_SONOGRAMS == 2)
{
DrawSonogram(sonogram, imagePath, hits, scores, predictedEvents, eventThreshold, intensity);
}
else
if (DRAW_SONOGRAMS == 1 && predictedEvents.Count > 0)
{
DrawSonogram(sonogram, imagePath, hits, scores, predictedEvents, eventThreshold, intensity);
}
Log.WriteLine("# Finished recording:- " + recordingFile.Name);
}
public void Serializable()
{
var s = @"
# comment
key:value
key1:value11
key1:value12
key1:value13
multilines = line1
line2
line3
";
var dict = new ConfigDictionary(new StringReader(s));
using (var tmpfile = new TemporaryFile())
{
PersistenceHelper.SaveObj(dict, tmpfile.FullName);
var deserialized = PersistenceHelper.RestoreObj<ConfigDictionary>(tmpfile.FullName);
Assert.AreEqual(Dumps(dict), Dumps(deserialized));
Dump(deserialized);
}
}
public async Task <dynamic> Get()
{
if (CheckClientSecret())
{
Entities db = new Entities();
Dictionary <string, string> emailConfig = ConfigDictionary.EmailConfig();
var storesRequiringUpdate = new List <StoreCustomData>();
foreach (StoreCustomData store in db.StoreCustomDatas)
{
DateTime lastUpdate = db.SalesTargetHistories
.Where(x => x.StoreID == store.StoreID)
.OrderByDescending(x => x.DateUpdated)
.FirstOrDefault()
.DateUpdated;
if ((DateTime.Now - lastUpdate).TotalDays > 30)
{
storesRequiringUpdate.Add(store);
}
}
if (storesRequiringUpdate.Any())
{
string storeNames = string.Empty;
foreach (StoreCustomData store in storesRequiringUpdate)
{
storeNames += $"<li><b>{ store.StoreName }</b> - Current Sales Target: £{ store.SalesTarget }</li>";
}
string messageBody = "<html><body>" +
$"<span style='display:inline-block;'><img src='https://cdn.shopify.com/s/files/applications/75204543d0fec5bbf9e7f1db609e8804_512x512.png?1594118407' style='width:50%;'/><h2>Sales Target Update Alert</h2></span>" +
$"<p>It has been more than 30 days since Sales Targets were updated for the following stores:</p><br/>" +
$"<ul>{ storeNames }</ul>" +
"</body></html>";
try
{
await EmailAlert.SendEmailAlert(new EmailAddress(emailConfig["SalesTargetRecipientAddress"]), "Sales Target Update Alert", messageBody);
return("Email sent!");
}
catch (Exception exception)
{
Console.WriteLine(exception.ToString());
return("Email failed to send.");
}
}
else
{
return("No updates required.");
}
}
else
{
return(new HttpResponseMessage(System.Net.HttpStatusCode.Forbidden));
}
}
private static void Dump(ConfigDictionary dict)
{
Console.WriteLine(Dumps(dict));
}
public override AnalysisResult2 Analyze <T>(AnalysisSettings analysisSettings, SegmentSettings <T> segmentSettings)
{
var fiAudioF = segmentSettings.SegmentAudioFile;
var diOutputDir = segmentSettings.SegmentOutputDirectory;
//######################################################################
var results = Analysis(fiAudioF, analysisSettings, segmentSettings.Segment.SourceMetadata.SampleRate, segmentSettings.SegmentStartOffset);
//######################################################################
if (results == null)
{
return(null); //nothing to process (broken)
}
var sonogram = results.Item1;
var hits = results.Item2;
var scores = results.Item3;
var predictedEvents = results.Item4;
var recordingTimeSpan = results.Item5;
var result = new AnalysisResult2(analysisSettings, segmentSettings, recordingTimeSpan);
result.AnalysisIdentifier = this.Identifier;
result.MiscellaneousResults["dataTable"] = null;
DataTable dataTable = null;
if (predictedEvents != null)
{
string analysisName = analysisSettings.ConfigDict[AnalysisKeys.AnalysisName];
string fName = Path.GetFileNameWithoutExtension(fiAudioF.Name);
foreach (AcousticEvent ev in predictedEvents)
{
ev.FileName = fName;
//ev.Name = analysisName; //TEMPORARY DISABLE
ev.SegmentDurationSeconds = recordingTimeSpan.TotalSeconds;
}
//write events to a data table to return.
dataTable = WriteEvents2DataTable(predictedEvents);
string sortString = AnalysisKeys.EventStartAbs + " ASC";
dataTable = DataTableTools.SortTable(dataTable, sortString); //sort by start time before returning
}
if (analysisSettings.AnalysisDataSaveBehavior)
{
CsvTools.DataTable2CSV(dataTable, segmentSettings.SegmentEventsFile.FullName);
}
else
{
result.EventsFile = null;
}
if (analysisSettings.AnalysisDataSaveBehavior)
{
double scoreThreshold = 0.01;
if (analysisSettings.ConfigDict.ContainsKey(AnalysisKeys.IntensityThreshold))
{
scoreThreshold = ConfigDictionary.GetDouble(AnalysisKeys.IntensityThreshold, analysisSettings.ConfigDict);
}
TimeSpan unitTime = TimeSpan.FromSeconds(60); //index for each time span of i minute
var indicesDT = this.ConvertEvents2Indices(dataTable, unitTime, recordingTimeSpan, scoreThreshold);
CsvTools.DataTable2CSV(indicesDT, segmentSettings.SegmentSummaryIndicesFile.FullName);
}
else
{
result.SummaryIndices = null;
}
//save image of sonograms
if (analysisSettings.AnalysisImageSaveBehavior.ShouldSave(predictedEvents.Count))
{
string imagePath = segmentSettings.SegmentImageFile.FullName;
Image image = DrawSonogram(sonogram, hits, scores, predictedEvents);
image.Save(imagePath, ImageFormat.Png);
}
result.MiscellaneousResults["dataTable"] = dataTable;
result.ImageFile = segmentSettings.SegmentImageFile;
//result.DisplayItems = { { 0, "example" }, { 1, "example 2" }, }
//result.OutputFiles = { { "exmaple file key", new FileInfo("Where's that file?") } }
return(result);
}
public void KeyValueFormat()
{
var s = @"
key1=value1
key2 = value2
key3:value3
key4 : value4
space key: space value
period.key: period.value
";
var dict = new ConfigDictionary(new StringReader(s));
Assert.AreEqual("value1", dict["key1"].Value);
Assert.AreEqual("value2", dict["key2"].Value);
Assert.AreEqual("value3", dict["key3"].Value);
Assert.AreEqual("value4", dict["key4"].Value);
Assert.AreEqual("space value", dict["space key"].Value);
Assert.AreEqual("period.value", dict["period.key"].Value);
Dump(dict);
}
public void ClearProperties()
{
ForEachProperty(ClearProperty);
ConfigDictionary.TryGetValue(this)?.Clear();
}