private void RunTest(string[] parameters)
{
string action = parameters[0];
switch (action)
{
case "Insert":
string folderWithSongs = parameters[1];
var stride = utils.ToStride(parameters[2], parameters[3], parameters[4]);
DeleteAll();
var sb = TestRunnerWriter.StartInsert();
Insert(folderWithSongs, stride, sb);
TestRunnerWriter.SaveInsertDataToFolder(sb, pathToResultsFolder, stride);
lastInsertStride = stride;
break;
case "Run":
string folderWithPositives = parameters[1];
string folderWithNegatives = parameters[2];
var queryStride = utils.ToStride(parameters[3], parameters[4], parameters[5]);
int seconds = int.Parse(parameters[6]);
var startAts = ToStartAts(parameters[7]);
RunTestScenario(folderWithPositives, folderWithNegatives, queryStride, seconds, startAts);
break;
}
}
private void AppendLine(StringBuilder sb, object[] objects)
{
lock (this)
{
TestRunnerWriter.AppendLine(sb, objects);
}
}
public void Run()
{
suite = TestRunnerWriter.StartSuite();
foreach (var scenario in scenarious)
{
string[] parameters = scenario.Split(',');
RunTest(parameters);
}
TestRunnerWriter.SaveSuiteResultsToFolder(suite, pathToResultsFolder);
OnTestRunnerEvent(OngoingActionEvent, new TestRunnerOngoingEventArgs { Message = "All test scenarious finished!" });
}
private void RunTestScenario(string folderWithPositives, string folderWithNegatives, IStride queryStride, int seconds, List <int> startAts)
{
int iterations = startAts.Count;
var positives = AllFiles(folderWithPositives);
var negatives = AllFiles(folderWithNegatives);
for (int iteration = 0; iteration < iterations; ++iteration)
{
OnTestRunnerEvent(OngoingActionEvent, new TestRunnerOngoingEventArgs
{
Message = $"Iteration {iteration + 1} out of {iterations} with {queryStride}, query seconds {seconds}"
});
var stopwatch = new Stopwatch();
stopwatch.Start();
int trueNegatives = 0, truePositives = 0, falseNegatives = 0, falsePositives = 0, verified = 0;
var truePositiveHammingDistance = new ConcurrentBag <int>();
var falseNegativesHammingDistance = new ConcurrentBag <int>();
var falsePositivesHammingDistance = new ConcurrentBag <int>();
var sb = TestRunnerWriter.StartTestIteration();
int currentIteration = iteration;
int startAt = startAts[currentIteration];
Parallel.ForEach(
positives,
positive =>
{
Interlocked.Increment(ref verified);
var tags = GetTagsFromFile(positive);
var actualTrack = GetActualTrack(tags);
var queryResult = BuildQuery(queryStride, seconds, positive, startAt).Result;
if (!queryResult.ContainsMatches)
{
Interlocked.Increment(ref falseNegatives);
var notFoundLine = GetNotFoundLine(tags);
AppendLine(sb, notFoundLine);
OnTestRunnerEvent(
PositiveNotFoundEvent,
GetTestRunnerEventArgs(
truePositives,
trueNegatives,
falsePositives,
falseNegatives,
notFoundLine,
verified));
return;
}
var recognizedTrack = queryResult.BestMatch.Track;
bool isSuccessful = recognizedTrack.Id.Equals(actualTrack.Id);
if (isSuccessful)
{
Interlocked.Increment(ref truePositives);
truePositiveHammingDistance.Add((int)queryResult.BestMatch.Score);
}
else
{
Interlocked.Increment(ref falsePositives);
falseNegativesHammingDistance.Add((int)queryResult.BestMatch.Score);
}
var foundLine = GetFoundLine(ToTrackString(actualTrack.Artist, actualTrack.Title), recognizedTrack, isSuccessful, queryResult);
AppendLine(sb, foundLine);
OnTestRunnerEvent(PositiveFoundEvent, GetTestRunnerEventArgs(truePositives, trueNegatives, falsePositives, falseNegatives, foundLine, verified));
});
Parallel.ForEach(
negatives,
negative =>
{
Interlocked.Increment(ref verified);
var tags = GetTagsFromFile(negative);
var queryResult = BuildQuery(queryStride, seconds, negative, startAt).Result;
if (!queryResult.ContainsMatches)
{
Interlocked.Increment(ref trueNegatives);
var notFoundLine = GetNotFoundLine(tags);
AppendLine(sb, notFoundLine);
OnTestRunnerEvent(
NegativeNotFoundEvent,
GetTestRunnerEventArgs(
truePositives,
trueNegatives,
falsePositives,
falseNegatives,
notFoundLine,
verified));
return;
}
var recognizedTrack = queryResult.BestMatch.Track;
falsePositivesHammingDistance.Add((int)queryResult.BestMatch.Score);
Interlocked.Increment(ref falsePositives);
var foundLine = GetFoundLine(ToTrackString(tags), recognizedTrack, false, queryResult);
AppendLine(sb, foundLine);
OnTestRunnerEvent(
NegativeFoundEvent,
GetTestRunnerEventArgs(truePositives, trueNegatives, falsePositives, falseNegatives, foundLine, verified));
});
stopwatch.Stop();
var fscore = new FScore(truePositives, trueNegatives, falsePositives, falseNegatives);
var stats = HammingDistanceResultStatistics.From(
truePositiveHammingDistance,
falseNegativesHammingDistance,
falsePositivesHammingDistance,
testRunnerConfig.Percentiles);
TestRunnerWriter.FinishTestIteration(sb, fscore, stats, stopwatch.ElapsedMilliseconds);
TestRunnerWriter.SaveTestIterationToFolder(sb, pathToResultsFolder, queryStride, GetInsertMetadata(), seconds, startAt);
var finishedTestIteration = GetTestRunnerEventArgsForFinishedTestIteration(queryStride, seconds, startAts, fscore, stats, iteration, stopwatch, verified);
OnTestRunnerEvent(TestIterationFinishedEvent, finishedTestIteration);
TestRunnerWriter.AppendLine(suite, finishedTestIteration.RowWithDetails);
}
}
