/// <summary>
/// Stops the engine safely. Letting all current operations complete but will not allow the engine to start any now tasks.
/// </summary>
public void StopEngine()
{
Task.Run(() =>
{
AddMessage("Initiating engine shutdown.");
ChangeState(State.ShuttingDown);
AddMessage("Waiting for all analyzers to finish.");
var timer = new Stopwatch();
timer.Start();
while (Analyzers.Any(a => a.Value.State != State.Stopped) && timer.ElapsedMilliseconds < 30000)
{
AddMessage($"{Analyzers.Count(a => a.Value.State != State.Stopped)} analyzers not stopped. Waited {timer.ElapsedMilliseconds} ms.");
Task.Delay(1000).Wait();
}
AddMessage("Shutdown complete.");
ChangeState(State.Stopped);
});
}
void IUtf8JsonSerializable.Write(Utf8JsonWriter writer)
{
writer.WriteStartObject();
writer.WritePropertyName("name");
writer.WriteStringValue(Name);
if (Fields.Any())
{
writer.WritePropertyName("fields");
writer.WriteStartArray();
foreach (var item in Fields)
{
writer.WriteObjectValue(item);
}
writer.WriteEndArray();
}
else
{
writer.WriteNull("fields");
}
if (ScoringProfiles != null && ScoringProfiles.Any())
{
writer.WritePropertyName("scoringProfiles");
writer.WriteStartArray();
foreach (var item in ScoringProfiles)
{
writer.WriteObjectValue(item);
}
writer.WriteEndArray();
}
if (DefaultScoringProfile != null)
{
writer.WritePropertyName("defaultScoringProfile");
writer.WriteStringValue(DefaultScoringProfile);
}
if (CorsOptions != null)
{
writer.WritePropertyName("corsOptions");
writer.WriteObjectValue(CorsOptions);
}
if (Suggesters != null && Suggesters.Any())
{
writer.WritePropertyName("suggesters");
writer.WriteStartArray();
foreach (var item in Suggesters)
{
writer.WriteObjectValue(item);
}
writer.WriteEndArray();
}
if (Analyzers != null && Analyzers.Any())
{
writer.WritePropertyName("analyzers");
writer.WriteStartArray();
foreach (var item in Analyzers)
{
writer.WriteObjectValue(item);
}
writer.WriteEndArray();
}
if (Tokenizers != null && Tokenizers.Any())
{
writer.WritePropertyName("tokenizers");
writer.WriteStartArray();
foreach (var item in Tokenizers)
{
writer.WriteObjectValue(item);
}
writer.WriteEndArray();
}
if (TokenFilters != null && TokenFilters.Any())
{
writer.WritePropertyName("tokenFilters");
writer.WriteStartArray();
foreach (var item in TokenFilters)
{
writer.WriteObjectValue(item);
}
writer.WriteEndArray();
}
if (CharFilters != null && CharFilters.Any())
{
writer.WritePropertyName("charFilters");
writer.WriteStartArray();
foreach (var item in CharFilters)
{
writer.WriteObjectValue(item);
}
writer.WriteEndArray();
}
if (EncryptionKey != null)
{
writer.WritePropertyName("encryptionKey");
writer.WriteObjectValue(EncryptionKey);
}
if (Similarity != null)
{
writer.WritePropertyName("similarity");
writer.WriteObjectValue(Similarity);
}
if (_etag != null)
{
writer.WritePropertyName("@odata.etag");
writer.WriteStringValue(_etag);
}
writer.WriteEndObject();
}
/// <summary>
/// Runs the analyzers.
/// </summary>
/// <returns><c>true</c>, if analyzers was run, <c>false</c> otherwise.</returns>
public bool RunAnalyzers()
{
if (!Analyzers.Any())
{
LogManager.Info("No analysis operations configured.", this);
return(true);
}
bool success = false;
if (Analyzers?.Count > 0 && InputData?.Keys?.Count >= 1)
{
var analyzedData = new Dictionary <string, IEnumerable <ICsvWritable> >();
foreach (var analyzerCommand in Analyzers)
{
// ---------------------------------------------------------
// Retrieve analysis operation details
// ---------------------------------------------------------
var analyzerName = analyzerCommand.Name;
var analyzer = AnalyzerManager.GetAnalyzer(analyzerName);
if (analyzer == null)
{
continue;
}
// Sanity checking ...
LogManager.Info($"Running analysis operation:\n\t{analyzer}", this);
// ---------------------------------------------------------
// Check to see if user requested a summary
// ---------------------------------------------------------
bool summaryRequested = Summarizers.Contains(analyzerName);
// ---------------------------------------------------------
// Group input data by original file name
// ---------------------------------------------------------
var consolidatedInputFiles =
Inputs.SelectMany(x => x.InputFiles.Select(y => y.Key)).ToList();
foreach (var origInputFile in consolidatedInputFiles)
{
var analysisDataByInputFile = new Dictionary <string, IEnumerable <ICsvWritable> >();
var analysisKeysByInputFile = InputData.Keys.Where(x => x.Contains(origInputFile));
foreach (var key in analysisKeysByInputFile)
{
// -------------------------------------------------
// Perform the actual analysis operation
// -------------------------------------------------
if (analyzerCommand.HasParameters &&
FilterManager.ContainsFilter(Filters, typeof(ThresholdCalibrationFilter)))
{
var calibData = CalibrationData[origInputFile];
analyzerCommand.Parameters =
ThresholdCalibrationFilter.CalibrateParameters
(analyzerCommand.Parameters, calibData);
}
// -------------------------------------------------
// Any required analyzer-specific prep
// -------------------------------------------------
if (analyzer is Analyzers.SciKitPrepAnalysis skpAnalyzer)
{
skpAnalyzer.CurrentInput = key;
}
else if (analyzer is Analyzers.SciKitEvalAnalysis skeAnalyzer)
{
skeAnalyzer.CurrentInput = key;
}
// -------------------------------------------------
var analysisResult =
analyzer.Analyze(InputData[key], analyzerCommand.Parameters);
analysisDataByInputFile[key] = analysisResult;
// -------------------------------------------------
// Dump output to file if necessary
// -------------------------------------------------
if (WriteOutputFile)
{
CsvFileWriter.WriteResultsToFile
(new string[] { OutputDirs.Analyzers, analyzerName },
key, analyzer.HeaderCsv, analysisResult);
}
// -------------------------------------------------
// If requested, summarize file-specific results
// -------------------------------------------------
if (summaryRequested)
{
ISummarizer summarizer =
SummarizerManager.GetSummarizer(analyzerName);
var summarizedValues =
summarizer.Summarize(analysisDataByInputFile);
if (WriteOutputFile)
{
CsvFileWriter.WriteSummaryToFile
(new string[] { OutputDirs.Summarizers, analyzerName },
$"{origInputFile}{Constants.BAT.DEFAULT_INPUT_FILE_EXT}",
summarizer.HeaderCsv, summarizedValues,
summarizer.FooterCsv, summarizer.FooterValues);
}
}
}
// consolidate the results by input file
var inputFileResults = analyzer.ConsolidateData(analysisDataByInputFile);
if (analyzedData.ContainsKey(origInputFile))
{
analyzedData[origInputFile] = inputFileResults;
}
else
{
analyzedData.Add(origInputFile, inputFileResults);
}
}
// -----------------------------------------------------
// If requested, aggregate all file-specific summaries
// into one high level file in root summarizer directory
// -----------------------------------------------------
if (summaryRequested)
{
ISummarizer summarizer = SummarizerManager.GetSummarizer(analyzerName);
var summarizedValues = summarizer.Summarize(analyzedData);
if (WriteOutputFile)
{
CsvFileWriter.WriteSummaryToFile
(new string[] { OutputDirs.Summarizers },
$"{analyzerName}Aggregate{Constants.BAT.DEFAULT_INPUT_FILE_EXT}",
summarizer.HeaderCsv, summarizedValues,
summarizer.FooterCsv, summarizer.FooterValues);
}
}
success = true;
}
// use different collection to maintain integrity of original input data
AnalysisData = analyzedData;
}
else
{
LogManager.Error("No input data to run analyzers on.", this);
}
return(success);
}