public NeteaseRequest(AnalyzerConfig config) : base(config)
{
ReqClient = new RestClient(config.Netease);
_config = config;
SongAudio = "/song/url?id=";
SongLyric = "/lyric?br=128000&id=";
}
public MusicInfoCollector(AnalyzerConfig config)
{
_config = config;
netease = new NeteaseRequest(_config);
qqmusic = new QQMusicRequest(_config);
kugou = new KugouRequest(_config);
}
public static Dictionary <int, IAnalyzerRuleHandler> CreateRulesHandlers(AnalyzerConfig analyzerConfig)
{
string anaylzerAssemblyFile = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, _analyzerAssemblyFileName);
Dictionary <int, IAnalyzerRuleHandler> keyValuePairs = new Dictionary <int, IAnalyzerRuleHandler>();
analyzerConfig.Rules.ToList().ForEach(r =>
{
if (r.IsEnabled)
{
IAnalyzerRuleHandler analyzerRuleHandler = null;
if (File.Exists(anaylzerAssemblyFile))
{
var loggerAssembly = Assembly.LoadFrom(anaylzerAssemblyFile);
Type loggerType = loggerAssembly.GetType(_namespace + "." + r.AnalysisType.ToString() + "RuleHandler");
analyzerRuleHandler = (IAnalyzerRuleHandler)Activator.CreateInstance(loggerType, r.ID, r, analyzerConfig);
}
if (analyzerRuleHandler != null)
{
keyValuePairs.Add(r.ID, analyzerRuleHandler);
}
}
});
return(keyValuePairs);
}
public static Dictionary <int, IAnalyzerOutputHandler> GetOutputsHandlers(AnalyzerConfig analyzerConfig)
{
Dictionary <int, IAnalyzerOutputHandler> keyValuePairs = new Dictionary <int, IAnalyzerOutputHandler>();
analyzerConfig.Outputs.ToList().ForEach(o =>
{
IAnalyzerOutputHandler analyzerOutputHandler = null;
string assemblyFileName = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, _analyzerAssemblyFileName);
if (o.OutputType == Common.Enums.OutputTypes.WindowsNotification)
{
assemblyFileName = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "AnalyzerNotifications.dll");
}
if (File.Exists(assemblyFileName))
{
var loggerAssembly = Assembly.LoadFrom(assemblyFileName);
Type outputHandlerType = loggerAssembly.GetType(_namespace + "." + o.OutputType.ToString() + "OutputHandler");
analyzerOutputHandler = (IAnalyzerOutputHandler)Activator.CreateInstance(outputHandlerType, o);
}
if (analyzerOutputHandler != null)
{
keyValuePairs.Add(o.ID, analyzerOutputHandler);
}
});
return(keyValuePairs);
}
public ImmutableArray <AnalyzerConfigOptionsResult> BuildAnalyzerConfigs()
{
var cmdLineArgs = Helpers.GetReproCommandLineArgs();
var analyzerConfigs = cmdLineArgs.AnalyzerConfigPaths
.SelectAsArray(p => AnalyzerConfig.Parse(File.ReadAllText(p), p));
var set = AnalyzerConfigSet.Create(analyzerConfigs);
return(cmdLineArgs.SourceFiles
.SelectAsArray(s => set.GetOptionsForSourcePath(s.Path)));
}
public MongoContext(AnalyzerConfig config)
{
_config = config;
client = new MongoClient(_config.MongoDBServer);
var db = client.GetDatabase(_config.DatabaseName);
MusicLibrary = db.GetCollection <SystemMusicLibrary>("SystemMusicLibrary");
Behaviour = db.GetCollection <UserBehaviour>("UserBehaviour");
UserTimeSpan = db.GetCollection <UserListeningTimeSpan>("UserListeningTimeSpan");
UserLikeBpm = db.GetCollection <UserLikeBpm>("UserLikeBpm");
UserLikeLanguage = db.GetCollection <UserLikeLanguage>("UserLikeLanguage");
UserSavedSingerList = db.GetCollection <UserSavedSingerList <ObjectId> >("UserSavedSingerList");
Singers = db.GetCollection <Singers>("Singers");
}
public LogFrequencyRuleHandler(int id, LogFrequencyRule logFrequencyRule, AnalyzerConfig analyzerConfig)
{
RuleID = id;
_logFrequencyRule = logFrequencyRule;
_logMessage = analyzerConfig.GetLogMessageConfig(_logFrequencyRule.LogMessageID);
_frequencyCheckTimer = new Timer(_logFrequencyRule.TimeInSeconds * 1000);
_frequencyCheckTimer.Elapsed += FrequencyCheckTimer_Elapsed;
_frequencyCheckTimer.Start();
}
public void Start()
{
AnalyzerConfig analyzerConfig = ConfigurationsFactory.GetAnalyzerConfig();
if (analyzerConfig?.IsEnabled ?? false)
{
_isRunning = true;
Dictionary <int, IAnalyzerRuleHandler> analyzerRuleHandlers = RuleHandlerFactory.CreateRulesHandlers(analyzerConfig);
Dictionary <int, IAnalyzerOutputHandler> analyzerOutputHandlers = OutputsHandlersFactory.GetOutputsHandlers(analyzerConfig);
Dictionary <int, IAnalyzerScheduleHandler> analyzerScheduleHandlers = SchedulesHandlersFactory.CreateSchedulesHandlers(analyzerConfig);
_logsAnalyzer = new LogsAnalyzer(analyzerConfig, null, analyzerRuleHandlers, analyzerOutputHandlers, analyzerScheduleHandlers);
_logsAnalyzer.Start();
}
}
public void FindReturnsNullOnGivenNull()
{
var nullConfig = new IndexCalculateConfig()
{
IndexPropertiesConfig = null,
};
Assert.IsNull(IndexProperties.Find(nullConfig));
var genericConfig = new AnalyzerConfig()
{
// property does not exist
//IndexPropertiesConfig = string.Empty,
};
Assert.IsNull(IndexProperties.Find(genericConfig as IIndexPropertyReferenceConfiguration));
// and test direct
Assert.IsNull(IndexProperties.Find((string)null, null));
}
public static AnalyzerConfig GetAnalyzerConfig()
{
AnalyzerConfig analyzerConfig = null;
if (File.Exists(_configFilePath))
{
JsonSerializerSettings settings = new JsonSerializerSettings
{
TypeNameHandling = TypeNameHandling.Auto
};
JsonConvert.DefaultSettings = () => settings;
string settingss = File.ReadAllText(_configFilePath);
analyzerConfig = JsonConvert.DeserializeObject <AnalyzerConfig>(settingss, new Newtonsoft.Json.Converters.StringEnumConverter());
}
return(analyzerConfig);
}
static void LoadConfig(bool prod)
{
string ConfigJson = string.Empty;
try
{
var prodFlag = prod ? "-prod" : "";
var configFile = "config" + prodFlag + ".json";
using (var sr = new StreamReader(configFile, Encoding.UTF8))
{
ConfigJson = sr.ReadToEnd();
}
System.Console.WriteLine($"Configuration {configFile} loaded.");
}
catch (FileNotFoundException fileNotFoundEx)
{
System.Console.WriteLine("Configuration file not found! " + fileNotFoundEx.Message);
Environment.Exit(1);
}
config = JsonConvert.DeserializeObject(ConfigJson, typeof(AnalyzerConfig)) as AnalyzerConfig;
}
private async Task <SyntaxTreeAnalysisContext> CreateAnalysisContextFromEditorConfigAsync(string editorConfig)
{
var projectId = ProjectId.CreateNewId();
var documentId = DocumentId.CreateNewId(projectId);
var analyzerConfigDocumentId = DocumentId.CreateNewId(projectId);
var solution = GenericAnalyzerTest.CreateWorkspace()
.CurrentSolution
.AddProject(projectId, TestProjectName, TestProjectName, LanguageNames.CSharp)
.AddDocument(documentId, "/0/Test0.cs", SourceText.From(string.Empty))
.AddAnalyzerConfigDocument(analyzerConfigDocumentId, "/.editorconfig", SourceText.From(editorConfig), filePath: "/.editorconfig");
var document = solution.GetDocument(documentId);
var syntaxTree = await document.GetSyntaxTreeAsync(CancellationToken.None).ConfigureAwait(false);
var analyzerConfigSet = AnalyzerConfigSet.Create(new[] { AnalyzerConfig.Parse(SourceText.From(editorConfig), "/.editorconfig") });
var additionalFiles = ImmutableArray <AdditionalText> .Empty;
var optionsProvider = this.CreateAnalyzerConfigOptionsProvider(analyzerConfigSet);
var analyzerOptions = new AnalyzerOptions(additionalFiles, optionsProvider);
return(new SyntaxTreeAnalysisContext(syntaxTree, analyzerOptions, reportDiagnostic: _ => { }, isSupportedDiagnostic: _ => true, CancellationToken.None));
}
public LogsAnalyzer(AnalyzerConfig analyzerConfig, ILogsReceiver logsReceiver,
Dictionary <int, IAnalyzerRuleHandler> ruleHandlers,
Dictionary <int, IAnalyzerOutputHandler> outputHandlers,
Dictionary <int, IAnalyzerScheduleHandler> scheduleHandlers)
{
_analyzerConfig = analyzerConfig;
_logsReceiver = logsReceiver;
_ruleHandlers = ruleHandlers;
_outputHandlers = outputHandlers;
_scheduleHandlers = scheduleHandlers;
_ruleHandlers.Values.ToList().ForEach(arh => arh.OnAnalyzerResult += OnAnalyzerResult);
if (logsReceiver != null)
{
logsReceiver.OnNewLogMessage += LogsReceiver_OnNewLogMessage;
}
}
public static ILogsReceiver Create(string[] applicationArgs, AnalyzerConfig analyzerConfig)
{
ILogsReceiver logsReceiver = null;
switch (analyzerConfig.LogsReceiverConfig.LogReceiverType)
{
case Common.Enums.LogReceiverTypes.AnonymousPipes:
if (applicationArgs?.Length > 0)
{
logsReceiver = new AnonymousPipesLogsReceiver(applicationArgs[0]);
}
break;
case Common.Enums.LogReceiverTypes.NamedPipes:
logsReceiver = new NamedPipesLogsReceiver((NamedPipesReceiverConfig)analyzerConfig.LogsReceiverConfig);
break;
case Common.Enums.LogReceiverTypes.WindowsEventsViewer:
string mircosoftEventViewerReceiverAssemblyFileName = "LogsManager.MircosoftEventViewerReceiverReceiver.dll";
IAnalyzerOutputHandler analyzerOutputHandler = null;
string assemblyFileName = Path.Combine(AppDomain.CurrentDomain.BaseDirectory, mircosoftEventViewerReceiverAssemblyFileName);
if (File.Exists(assemblyFileName))
{
var loggerAssembly = Assembly.LoadFrom(assemblyFileName);
Type outputHandlerType = loggerAssembly.GetType("LogsManager.MircosoftEventViewerReceiver.EventLogsReceiver");
analyzerOutputHandler = (IAnalyzerOutputHandler)Activator.CreateInstance(outputHandlerType, analyzerConfig.LogsReceiverConfig);
}
break;
}
return(logsReceiver);
}
private static AnalyzerConfig GetAnalyzerConfig()
{
JsonSerializerSettings settings = new JsonSerializerSettings
{
TypeNameHandling = TypeNameHandling.Auto
};
JsonConvert.DefaultSettings = () => settings;
AnalyzerConfig analyzerConfig = new AnalyzerConfig
{
LogsReceiverConfig = new LogsReceiverConfig {
LogReceiverType = LogReceiverTypes.None
},
LogMessages = GetLogMessageConfigs(),
IsEnabled = true,
Outputs = GetAnalyzerOutputConfigs(),
Rules = GetAnalyzerRuleConfigs(),
Schedules = GetAnalyzerScheduleConfigs(),
Conditions = GetAnalyzerConditionConfigs(),
RulesOutputs = new Dictionary <int, int> {
{ 1, 1 }, { 2, 2 }, { 3, 3 }, { 4, 4 }, { 5, 4 }
},
RulesSchedules = new Dictionary <int, int> {
{ 1, 2 }, { 2, 2 }, { 3, 1 }, { 4, 1 }, { 5, 1 }
}
};
return(analyzerConfig);
}
public static void Main(string[] args)
{
try
{
AnalyzerConfig analyzerConfig = ConfigurationsFactory.GetAnalyzerConfig();
if (analyzerConfig?.IsEnabled ?? false)
{
ILogsReceiver logsReceiver = LogsReceiverFactory.Create(args, analyzerConfig);
Dictionary <int, IAnalyzerRuleHandler> analyzerRuleHandlers = RuleHandlerFactory.CreateRulesHandlers(analyzerConfig);
Dictionary <int, IAnalyzerOutputHandler> analyzerOutputHandlers = OutputsHandlersFactory.GetOutputsHandlers(analyzerConfig);
Dictionary <int, IAnalyzerScheduleHandler> analyzerScheduleHandlers = SchedulesHandlersFactory.CreateSchedulesHandlers(analyzerConfig);
_logsAnalyzer = new LogsAnalyzer(analyzerConfig, logsReceiver, analyzerRuleHandlers, analyzerOutputHandlers, analyzerScheduleHandlers);
_logsAnalyzer.Start();
}
Console.Write("Press enter to exit...");
Console.ReadLine();
_logsAnalyzer.Dispose();
System.Threading.Thread.Sleep(2000);
}
catch (Exception exception)
{
Console.ForegroundColor = ConsoleColor.Red;
Console.WriteLine("Error:");
Console.WriteLine(exception?.Message);
}
}
public CommonRequest(AnalyzerConfig config)
{
ReqClient = new RestClient();
_config = config;
}
public KugouRequest(AnalyzerConfig config) : base(config)
{
ReqClient = new RestClient(config.Kugou);
_config = config;
SongAudio = "/songurl?hash=";
}
public static Dictionary <int, IAnalyzerScheduleHandler> CreateSchedulesHandlers(AnalyzerConfig analyzerConfig)
{
Dictionary <int, IAnalyzerScheduleHandler> keyValuePairs = new Dictionary <int, IAnalyzerScheduleHandler>();
analyzerConfig.Schedules.ToList().ForEach(s =>
{
IAnalyzerScheduleHandler analyzerScheduleHandler = new AnalyzerScheduleHandler(s);
keyValuePairs.Add(s.ID, analyzerScheduleHandler);
});
return(keyValuePairs);
}
/// <summary>
/// 2. Analyses long audio recording (mp3 or wav) as per passed config file. Outputs an events.csv file AND an
/// indices.csv file
/// Signed off: Michael Towsey 4th December 2012
/// </summary>
public static void Execute(Arguments arguments)
{
if (arguments == null)
{
throw new NoDeveloperMethodException();
}
LoggedConsole.WriteLine("# PROCESS LONG RECORDING");
LoggedConsole.WriteLine("# DATE AND TIME: " + DateTime.Now);
// 1. set up the necessary files
var sourceAudio = arguments.Source;
var configFile = arguments.Config.ToFileInfo();
var outputDirectory = arguments.Output;
var tempFilesDirectory = arguments.TempDir;
// if a temp dir is not given, use output dir as temp dir
if (tempFilesDirectory == null)
{
Log.Warn("No temporary directory provided, using output directory");
tempFilesDirectory = outputDirectory;
}
// try an automatically find the config file
if (configFile == null)
{
throw new FileNotFoundException("No config file argument provided");
}
else if (!configFile.Exists)
{
Log.Warn($"Config file {configFile.FullName} not found... attempting to resolve config file");
// we use .ToString() here to get the original input string - Using fullname always produces an absolute path wrt to pwd... we don't want to prematurely make asusmptions:
// e.g. We require a missing absolute path to fail... that wouldn't work with .Name
// e.g. We require a relative path to try and resolve, using .FullName would fail the first absolute check inside ResolveConfigFile
configFile = ConfigFile.Resolve(configFile.ToString(), Directory.GetCurrentDirectory().ToDirectoryInfo());
}
if (arguments.StartOffset.HasValue ^ arguments.EndOffset.HasValue)
{
throw new InvalidStartOrEndException("If StartOffset or EndOffset is specified, then both must be specified");
}
if (arguments.StartOffset.HasValue && arguments.EndOffset.HasValue && arguments.EndOffset.Value <= arguments.StartOffset.Value)
{
throw new InvalidStartOrEndException("Start offset must be less than end offset.");
}
LoggedConsole.WriteLine("# Recording file: " + sourceAudio.FullName);
LoggedConsole.WriteLine("# Configuration file: " + configFile);
LoggedConsole.WriteLine("# Output folder: " + outputDirectory);
LoggedConsole.WriteLine("# Temp File Directory: " + tempFilesDirectory);
// optionally copy logs / config to make results easier to understand
// TODO: remove, see https://github.com/QutEcoacoustics/audio-analysis/issues/133
if (arguments.WhenExitCopyConfig || arguments.WhenExitCopyLog)
{
AppDomain.CurrentDomain.ProcessExit += (sender, args) => { Cleanup(arguments, configFile); };
}
// 2. initialize the analyzer
// we're changing the way resolving config files works. Ideally, we'd like to use statically typed config files
// but we can't do that unless we know which type we have to load first! Currently analyzer to load is in
// the config file so we can't know which analyzer we can use. Thus we will change to using the file name,
// or an argument to resolve the analyzer to load.
// Get analysis name:
IAnalyser2 analyzer = FindAndCheckAnalyzer <IAnalyser2>(arguments.AnalysisIdentifier, configFile.Name);
// 2. get the analysis config
AnalyzerConfig configuration = analyzer.ParseConfig(configFile);
SaveBehavior saveIntermediateWavFiles = configuration.SaveIntermediateWavFiles;
bool saveIntermediateDataFiles = configuration.SaveIntermediateCsvFiles;
SaveBehavior saveSonogramsImages = configuration.SaveSonogramImages;
bool filenameDate = configuration.RequireDateInFilename;
if (configuration[AnalysisKeys.AnalysisName].IsNotWhitespace())
{
Log.Warn("Your config file has `AnalysisName` set - this property is deprecated and ignored");
}
// AT 2018-02: changed logic so default index properties loaded if not provided
FileInfo indicesPropertiesConfig = IndexProperties.Find(configuration, configFile);
if (indicesPropertiesConfig == null || !indicesPropertiesConfig.Exists)
{
Log.Warn("IndexProperties config can not be found! Loading a default");
indicesPropertiesConfig = ConfigFile.Default <Dictionary <string, IndexProperties> >();
}
LoggedConsole.WriteLine("# IndexProperties Cfg: " + indicesPropertiesConfig.FullName);
// min score for an acceptable event
Log.Info("Minimum event threshold has been set to " + configuration.EventThreshold);
FileSegment.FileDateBehavior defaultBehavior = FileSegment.FileDateBehavior.Try;
if (filenameDate)
{
if (!FileDateHelpers.FileNameContainsDateTime(sourceAudio.Name))
{
throw new InvalidFileDateException(
"When RequireDateInFilename option is set, the filename of the source audio file must contain "
+ "a valid AND UNAMBIGUOUS date. Such a date was not able to be parsed.");
}
defaultBehavior = FileSegment.FileDateBehavior.Required;
}
// 3. initilize AnalysisCoordinator class that will do the analysis
var analysisCoordinator = new AnalysisCoordinator(
new LocalSourcePreparer(),
saveIntermediateWavFiles,
false,
arguments.Parallel);
// 4. get the segment of audio to be analysed
// if tiling output, specify that FileSegment needs to be able to read the date
var fileSegment = new FileSegment(sourceAudio, arguments.AlignToMinute, null, defaultBehavior);
var bothOffsetsProvided = arguments.StartOffset.HasValue && arguments.EndOffset.HasValue;
if (bothOffsetsProvided)
{
fileSegment.SegmentStartOffset = TimeSpan.FromSeconds(arguments.StartOffset.Value);
fileSegment.SegmentEndOffset = TimeSpan.FromSeconds(arguments.EndOffset.Value);
}
else
{
Log.Debug("Neither start nor end segment offsets provided. Therefore both were ignored.");
}
// 6. initialize the analysis settings object
var analysisSettings = analyzer.DefaultSettings;
analysisSettings.ConfigFile = configFile;
analysisSettings.Configuration = configuration;
analysisSettings.AnalysisOutputDirectory = outputDirectory;
analysisSettings.AnalysisTempDirectory = tempFilesDirectory;
analysisSettings.AnalysisDataSaveBehavior = saveIntermediateDataFiles;
analysisSettings.AnalysisImageSaveBehavior = saveSonogramsImages;
analysisSettings.AnalysisChannelSelection = arguments.Channels;
analysisSettings.AnalysisMixDownToMono = arguments.MixDownToMono;
var segmentDuration = configuration.SegmentDuration?.Seconds();
if (!segmentDuration.HasValue)
{
segmentDuration = analysisSettings.AnalysisMaxSegmentDuration ?? TimeSpan.FromMinutes(1);
Log.Warn(
$"Can't read `{nameof(AnalyzerConfig.SegmentDuration)}` from config file. "
+ $"Default value of {segmentDuration} used)");
}
analysisSettings.AnalysisMaxSegmentDuration = segmentDuration.Value;
var segmentOverlap = configuration.SegmentOverlap?.Seconds();
if (!segmentOverlap.HasValue)
{
segmentOverlap = analysisSettings.SegmentOverlapDuration;
Log.Warn(
$"Can't read `{nameof(AnalyzerConfig.SegmentOverlap)}` from config file. "
+ $"Default value of {segmentOverlap} used)");
}
analysisSettings.SegmentOverlapDuration = segmentOverlap.Value;
// set target sample rate
var resampleRate = configuration.ResampleRate;
if (!resampleRate.HasValue)
{
resampleRate = analysisSettings.AnalysisTargetSampleRate ?? AppConfigHelper.DefaultTargetSampleRate;
Log.Warn(
$"Can't read {nameof(configuration.ResampleRate)} from config file. "
+ $"Default value of {resampleRate} used)");
}
analysisSettings.AnalysisTargetSampleRate = resampleRate;
Log.Info(
$"{nameof(configuration.SegmentDuration)}={segmentDuration}, "
+ $"{nameof(configuration.SegmentOverlap)}={segmentOverlap}, "
+ $"{nameof(configuration.ResampleRate)}={resampleRate}");
// 7. ####################################### DO THE ANALYSIS ###################################
LoggedConsole.WriteLine("START ANALYSIS ...");
var analyserResults = analysisCoordinator.Run(fileSegment, analyzer, analysisSettings);
// ##############################################################################################
// 8. PROCESS THE RESULTS
LoggedConsole.WriteLine(string.Empty);
LoggedConsole.WriteLine("START PROCESSING RESULTS ...");
if (analyserResults == null)
{
LoggedConsole.WriteErrorLine("###################################################\n");
LoggedConsole.WriteErrorLine("The Analysis Run Coordinator has returned a null result.");
LoggedConsole.WriteErrorLine("###################################################\n");
throw new AnalysisOptionDevilException();
}
// Merge and correct main result types
EventBase[] mergedEventResults = ResultsTools.MergeResults(analyserResults, ar => ar.Events, ResultsTools.CorrectEvent);
SummaryIndexBase[] mergedIndicesResults = ResultsTools.MergeResults(analyserResults, ar => ar.SummaryIndices, ResultsTools.CorrectSummaryIndex);
SpectralIndexBase[] mergedSpectralIndexResults = ResultsTools.MergeResults(analyserResults, ar => ar.SpectralIndices, ResultsTools.CorrectSpectrumIndex);
// not an exceptional state, do not throw exception
if (mergedEventResults != null && mergedEventResults.Length == 0)
{
LoggedConsole.WriteWarnLine("The analysis produced no EVENTS (mergedResults had zero count)");
}
if (mergedIndicesResults != null && mergedIndicesResults.Length == 0)
{
LoggedConsole.WriteWarnLine("The analysis produced no Summary INDICES (mergedResults had zero count)");
}
if (mergedSpectralIndexResults != null && mergedSpectralIndexResults.Length == 0)
{
LoggedConsole.WriteWarnLine("The analysis produced no Spectral INDICES (merged results had zero count)");
}
// 9. CREATE SUMMARY INDICES IF NECESSARY (FROM EVENTS)
#if DEBUG
// get the duration of the original source audio file - need this to convert Events datatable to Indices Datatable
var audioUtility = new MasterAudioUtility(tempFilesDirectory);
var mimeType = MediaTypes.GetMediaType(sourceAudio.Extension);
var sourceInfo = audioUtility.Info(sourceAudio);
// updated by reference all the way down in LocalSourcePreparer
Debug.Assert(fileSegment.TargetFileDuration == sourceInfo.Duration);
#endif
var duration = fileSegment.TargetFileDuration.Value;
ResultsTools.ConvertEventsToIndices(
analyzer,
mergedEventResults,
ref mergedIndicesResults,
duration,
configuration.EventThreshold);
int eventsCount = mergedEventResults?.Length ?? 0;
int numberOfRowsOfIndices = mergedIndicesResults?.Length ?? 0;
// 10. Allow analysers to post-process
// TODO: remove results directory if possible
var instanceOutputDirectory =
AnalysisCoordinator.GetNamedDirectory(analysisSettings.AnalysisOutputDirectory, analyzer);
// 11. IMPORTANT - this is where IAnalyser2's post processor gets called.
// Produces all spectrograms and images of SPECTRAL INDICES.
// Long duration spectrograms are drawn IFF analysis type is Towsey.Acoustic
analyzer.SummariseResults(analysisSettings, fileSegment, mergedEventResults, mergedIndicesResults, mergedSpectralIndexResults, analyserResults);
// 12. SAVE THE RESULTS
string fileNameBase = Path.GetFileNameWithoutExtension(sourceAudio.Name);
var eventsFile = ResultsTools.SaveEvents(analyzer, fileNameBase, instanceOutputDirectory, mergedEventResults);
var indicesFile = ResultsTools.SaveSummaryIndices(analyzer, fileNameBase, instanceOutputDirectory, mergedIndicesResults);
var spectraFile = ResultsTools.SaveSpectralIndices(analyzer, fileNameBase, instanceOutputDirectory, mergedSpectralIndexResults);
// 13. THIS IS WHERE SUMMARY INDICES ARE PROCESSED
// Convert summary indices to black and white tracks image
if (mergedIndicesResults == null)
{
Log.Info("No summary indices produced");
}
else
{
if (indicesPropertiesConfig == null || !indicesPropertiesConfig.Exists)
{
throw new InvalidOperationException("Cannot process indices without an index configuration file, the file could not be found!");
}
// this arbitrary amount of data.
if (mergedIndicesResults.Length > 5000)
{
Log.Warn("Summary Indices Image not able to be drawn - there are too many indices to render");
}
else
{
var basename = Path.GetFileNameWithoutExtension(fileNameBase);
string imageTitle = $"SOURCE:{basename}, {Meta.OrganizationTag}; ";
// Draw Tracks-Image of Summary indices
// set time scale resolution for drawing of summary index tracks
TimeSpan timeScale = TimeSpan.FromSeconds(0.1);
Bitmap tracksImage =
IndexDisplay.DrawImageOfSummaryIndices(
IndexProperties.GetIndexProperties(indicesPropertiesConfig),
indicesFile,
imageTitle,
timeScale,
fileSegment.TargetFileStartDate);
var imagePath = FilenameHelpers.AnalysisResultPath(instanceOutputDirectory, basename, "SummaryIndices", ImageFileExt);
tracksImage.Save(imagePath);
}
}
// 14. wrap up, write stats
LoggedConsole.WriteLine("INDICES CSV file(s) = " + (indicesFile?.Name ?? "<<No indices result, no file!>>"));
LoggedConsole.WriteLine("\tNumber of rows (i.e. minutes) in CSV file of indices = " + numberOfRowsOfIndices);
LoggedConsole.WriteLine(string.Empty);
if (eventsFile == null)
{
LoggedConsole.WriteLine("An Events CSV file was NOT returned.");
}
else
{
LoggedConsole.WriteLine("EVENTS CSV file(s) = " + eventsFile.Name);
LoggedConsole.WriteLine("\tNumber of events = " + eventsCount);
}
Log.Success($"Analysis Complete.\nSource={sourceAudio.Name}\nOutput={instanceOutputDirectory.FullName}");
}
public SequenceDetectionRuleHandler(int id, ISequenceDetectionRule sequenceDetectionRule, AnalyzerConfig analyzerConfig)
{
RuleID = id;
sortedLogsMessages = sequenceDetectionRule?.SortedLogMessagesIDs?
.Select(logId => analyzerConfig.GetLogMessageConfig(logId)).ToArray();
_nextExpectedLogIndex = 0;
}
public QQMusicRequest(AnalyzerConfig config) : base(config)
{
_config = config;
client = new RestClient(_config.QQMusic);
}
/// <summary>
/// Calculates the following spectrograms as per content of config.yml file:
/// Waveform: true.
/// DifferenceSpectrogram: true.
/// DecibelSpectrogram: true.
/// DecibelSpectrogram_NoiseReduced: true.
/// DecibelSpectrogram_Ridges: true.
/// AmplitudeSpectrogram_LocalContrastNormalization: true.
/// SoxSpectrogram: false.
/// Experimental: true.
/// </summary>
/// <param name="sourceRecording">The name of the original recording.</param>
/// <param name="configInfo">Contains parameter info to make spectrograms.</param>
/// <param name="sourceRecordingName">.Name of source recording. Required only spectrogram labels.</param>
public static AudioToSonogramResult GenerateSpectrogramImages(
FileInfo sourceRecording,
AnalyzerConfig configInfo,
string sourceRecordingName)
{
//int signalLength = recordingSegment.WavReader.GetChannel(0).Length;
var recordingSegment = new AudioRecording(sourceRecording.FullName);
int sampleRate = recordingSegment.WavReader.SampleRate;
var result = new AudioToSonogramResult();
// init the image stack
var list = new List <Image>();
bool doWaveForm = configInfo.GetBoolOrNull("Waveform") ?? false;
bool doDecibelSpectrogram = configInfo.GetBoolOrNull("DecibelSpectrogram") ?? false;
bool doNoiseReducedSpectrogram = configInfo.GetBoolOrNull("DecibelSpectrogram_NoiseReduced") ?? true;
bool doDifferenceSpectrogram = configInfo.GetBoolOrNull("DifferenceSpectrogram") ?? false;
bool doLcnSpectrogram = configInfo.GetBoolOrNull("AmplitudeSpectrogram_LocalContrastNormalization") ?? false;
bool doCepstralSpectrogram = configInfo.GetBoolOrNull("CepstralSpectrogram") ?? false;
bool doExperimentalSpectrogram = configInfo.GetBoolOrNull("Experimental") ?? false;
//Don't do SOX spectrogram.
//bool doSoxSpectrogram = configInfo.GetBool("SoxSpectrogram");
int frameSize = configInfo.GetIntOrNull("FrameLength") ?? 512;
int frameStep = configInfo.GetIntOrNull("FrameStep") ?? 0;
// must calculate this because used later on.
double frameOverlap = (frameSize - frameStep) / (double)frameSize;
// Default noiseReductionType = Standard
var bgNoiseThreshold = configInfo.GetDoubleOrNull("BgNoiseThreshold") ?? 3.0;
// EXTRACT ENVELOPE and SPECTROGRAM FROM RECORDING SEGMENT
var dspOutput1 = DSP_Frames.ExtractEnvelopeAndFfts(recordingSegment, frameSize, frameStep);
var sonoConfig = new SonogramConfig()
{
epsilon = recordingSegment.Epsilon,
SampleRate = sampleRate,
WindowSize = frameSize,
WindowStep = frameStep,
WindowOverlap = frameOverlap,
WindowPower = dspOutput1.WindowPower,
Duration = recordingSegment.Duration,
NoiseReductionType = NoiseReductionType.Standard,
NoiseReductionParameter = bgNoiseThreshold,
};
// IMAGE 1) draw the WAVEFORM
if (doWaveForm)
{
var minValues = dspOutput1.MinFrameValues;
var maxValues = dspOutput1.MaxFrameValues;
int height = configInfo.GetIntOrNull("WaveformHeight") ?? 180;
var waveformImage = GetWaveformImage(minValues, maxValues, height);
// add in the title bar and time scales.
string title = $"WAVEFORM - {sourceRecordingName} (min value={dspOutput1.MinSignalValue:f3}, max value={dspOutput1.MaxSignalValue:f3})";
var titleBar = BaseSonogram.DrawTitleBarOfGrayScaleSpectrogram(title, waveformImage.Width);
var startTime = TimeSpan.Zero;
var xAxisTicInterval = TimeSpan.FromSeconds(1);
TimeSpan xAxisPixelDuration = TimeSpan.FromSeconds(frameStep / (double)sampleRate);
var labelInterval = TimeSpan.FromSeconds(5);
waveformImage = BaseSonogram.FrameSonogram(waveformImage, titleBar, startTime, xAxisTicInterval, xAxisPixelDuration, labelInterval);
list.Add(waveformImage);
}
// Draw various decibel spectrograms
if (doDecibelSpectrogram || doNoiseReducedSpectrogram || doDifferenceSpectrogram || doExperimentalSpectrogram)
{
// disable noise removal for first spectrogram
var disabledNoiseReductionType = sonoConfig.NoiseReductionType;
sonoConfig.NoiseReductionType = NoiseReductionType.None;
//Get the decibel spectrogram
var decibelSpectrogram = new SpectrogramStandard(sonoConfig, dspOutput1.AmplitudeSpectrogram);
result.DecibelSpectrogram = decibelSpectrogram;
double[,] dbSpectrogramData = (double[, ])decibelSpectrogram.Data.Clone();
// IMAGE 2) DecibelSpectrogram
if (doDecibelSpectrogram)
{
var image3 = decibelSpectrogram.GetImageFullyAnnotated($"DECIBEL SPECTROGRAM ({sourceRecordingName})");
list.Add(image3);
}
if (doNoiseReducedSpectrogram || doExperimentalSpectrogram || doDifferenceSpectrogram)
{
sonoConfig.NoiseReductionType = disabledNoiseReductionType;
sonoConfig.NoiseReductionParameter = bgNoiseThreshold;
double[] spectralDecibelBgn = NoiseProfile.CalculateBackgroundNoise(decibelSpectrogram.Data);
decibelSpectrogram.Data = SNR.TruncateBgNoiseFromSpectrogram(decibelSpectrogram.Data, spectralDecibelBgn);
decibelSpectrogram.Data = SNR.RemoveNeighbourhoodBackgroundNoise(decibelSpectrogram.Data, nhThreshold: bgNoiseThreshold);
// IMAGE 3) DecibelSpectrogram - noise reduced
if (doNoiseReducedSpectrogram)
{
var image4 = decibelSpectrogram.GetImageFullyAnnotated($"DECIBEL SPECTROGRAM + Lamel noise subtraction. ({sourceRecordingName})");
list.Add(image4);
}
// IMAGE 4) EXPERIMENTAL Spectrogram
if (doExperimentalSpectrogram)
{
sonoConfig.NoiseReductionType = disabledNoiseReductionType;
var image5 = GetDecibelSpectrogram_Ridges(dbSpectrogramData, decibelSpectrogram, sourceRecordingName);
list.Add(image5);
}
// IMAGE 5) draw difference spectrogram
if (doDifferenceSpectrogram)
{
var differenceThreshold = configInfo.GetDoubleOrNull("DifferenceThreshold") ?? 3.0;
var image6 = GetDifferenceSpectrogram(dbSpectrogramData, differenceThreshold);
image6 = BaseSonogram.GetImageAnnotatedWithLinearHertzScale(image6, sampleRate, frameStep, $"DECIBEL DIFFERENCE SPECTROGRAM ({sourceRecordingName})");
list.Add(image6);
}
}
}
// IMAGE 6) Cepstral Spectrogram
if (doCepstralSpectrogram)
{
var image6 = GetCepstralSpectrogram(sonoConfig, recordingSegment, sourceRecordingName);
list.Add(image6);
}
// 7) AmplitudeSpectrogram_LocalContrastNormalization
if (doLcnSpectrogram)
{
var neighbourhoodSeconds = configInfo.GetDoubleOrNull("NeighbourhoodSeconds") ?? 0.5;
var lcnContrastParameter = configInfo.GetDoubleOrNull("LcnContrastLevel") ?? 0.4;
var image8 = GetLcnSpectrogram(sonoConfig, recordingSegment, sourceRecordingName, neighbourhoodSeconds, lcnContrastParameter);
list.Add(image8);
}
// 8) SOX SPECTROGRAM
//if (doSoxSpectrogram)
//{
//Log.Warn("SoX spectrogram set to true but is ignored when running as an IAnalyzer");
// The following parameters were once used to implement a sox spectrogram.
//bool makeSoxSonogram = configuration.GetBoolOrNull(AnalysisKeys.MakeSoxSonogram) ?? false;
//configDict[AnalysisKeys.SonogramTitle] = configuration[AnalysisKeys.SonogramTitle] ?? "Sonogram";
//configDict[AnalysisKeys.SonogramComment] = configuration[AnalysisKeys.SonogramComment] ?? "Sonogram produced using SOX";
//configDict[AnalysisKeys.SonogramColored] = configuration[AnalysisKeys.SonogramColored] ?? "false";
//configDict[AnalysisKeys.SonogramQuantisation] = configuration[AnalysisKeys.SonogramQuantisation] ?? "128";
//configDict[AnalysisKeys.AddTimeScale] = configuration[AnalysisKeys.AddTimeScale] ?? "true";
//configDict[AnalysisKeys.AddAxes] = configuration[AnalysisKeys.AddAxes] ?? "true";
//configDict[AnalysisKeys.AddSegmentationTrack] = configuration[AnalysisKeys.AddSegmentationTrack] ?? "true";
// var soxFile = new FileInfo(Path.Combine(output.FullName, sourceName + "SOX.png"));
// SpectrogramTools.MakeSonogramWithSox(sourceRecording, configDict, path2SoxSpectrogram);
// list.Add(image7);
//}
// COMBINE THE SPECTROGRAM IMAGES
result.CompositeImage = ImageTools.CombineImagesVertically(list);
return(result);
}
public AggregateFunctionRuleHandler(int id, AggregateFunctionRule aggregateFunctionRule, AnalyzerConfig analyzerConfig)
{
RuleID = id;
_aggregateFunctionRule = aggregateFunctionRule;
_logMessage = analyzerConfig.LogMessages.FirstOrDefault(lm => lm != null && _aggregateFunctionRule.LogMessageID == lm.ID);
_conditions = aggregateFunctionRule.ConditionsIDs?.Select(cID => analyzerConfig.Conditions?.FirstOrDefault(c => c.ID == cID)).ToArray();
}
public MessageAbsenceDetectionRuleHandler(int id, MessageAbsenceDetectionRule messageAbsenceDetectionRule, AnalyzerConfig analyzerConfig)
{
RuleID = id;
if (messageAbsenceDetectionRule.AbsenceIntervalInSeconds > 0)
{
_monitoringTimer = new System.Timers.Timer(messageAbsenceDetectionRule.AbsenceIntervalInSeconds * 1000);
_monitoringTimer.Elapsed += MonitoringTimer_Elapsed;
_monitoringTimer.Start();
_messageAbsenceDetectionRule = messageAbsenceDetectionRule;
_logMessageConfig = analyzerConfig.LogMessages.FirstOrDefault(lm => lm != null && messageAbsenceDetectionRule.LogMessageID == lm.ID);
}
}
public AntiSequenceRuleHandler(int id, IAntiSequenceDetectionRule antiSequenceDetectionRule, AnalyzerConfig analyzerConfig)
{
RuleID = id;
sortedLogsMessages = analyzerConfig.LogMessages.Where(lm => lm != null && antiSequenceDetectionRule
.SortedLogMessagesIDs.Contains(lm.ID)).ToArray();
_nextExpectedLogIndex = 0;
}
public MusicAnalyzer(AnalyzerConfig config)
{
_config = config;
context = new MongoContext(config);
}
public AnalyzerServices(AnalyzerConfig config)
{
_config = config;
_analyzer = new MusicAnalyzer(_config);
_collector = new MusicInfoCollector(_config);
}
public LogFilterRuleHandler(int id, Common.Analyzer.Rules.LogFilterRule logFilterRule, AnalyzerConfig analyzerConfig)
{
RuleID = id;
_logMessages = logFilterRule?.LogMessagesIDs?.Select(logId => analyzerConfig.GetLogMessageConfig(logId)).ToArray();
}
public DuplicateDetectionRuleHandler(int id, DuplicateDetectionRule duplicateDetectionRule, AnalyzerConfig analyzerConfig)
{
RuleID = id;
_duplicateDetectionRule = duplicateDetectionRule;
_logMessage = analyzerConfig.LogMessages.FirstOrDefault(lm => lm != null && duplicateDetectionRule.LogMessageID == lm.ID);
_monitoredLogMessages = new List <LogMessage>();
}
