public static void FinishTestIteration(StringBuilder sb, FScore score, HammingDistanceResultStatistics statistics, long elapsedMiliseconds)
{
sb.AppendLine();
sb.AppendLine(string.Format("Results: {0}. Elapsed Seconds: {1}", score, (double)elapsedMiliseconds / 1000));
sb.AppendLine();
sb.AppendLine(string.Format("True Positives: {0}", statistics.TruePositiveInfo));
sb.AppendLine(string.Format("True Positives Percentile: {0}", statistics.TruePositivePercentileInfo));
sb.AppendLine(string.Format("False Negatives: {0}", statistics.FalseNegativesInfo));
sb.AppendLine(string.Format("False Negatives Percentile: {0}", statistics.FalseNegativesPercentileInfo));
sb.AppendLine(string.Format("False Positives: {0}", statistics.FalsePositivesInfo));
sb.AppendLine(string.Format("False Positives Percentile: {0}", statistics.FalsePositivesPercentileInfo));
}
public void ShouldCalculatePercentilesCorrenctly()
{
var stats = HammingDistanceResultStatistics.From(
new ConcurrentBag <int> {
5, 4, 3, 2, 1
},
new ConcurrentBag <int>(),
new ConcurrentBag <int>(),
new[] { 0.9, 0.95, 0.98 });
var percentiles = stats.TruePositivePercentile;
Assert.AreEqual(4.6, percentiles[0], 0.001);
Assert.AreEqual(4.8, percentiles[1], 0.001);
Assert.AreEqual(4.92, percentiles[2], 0.001);
}
public static void FinishTestIteration(StringBuilder sb, FScore score, HammingDistanceResultStatistics statistics, long elapsedMiliseconds)
{
sb.AppendLine();
sb.AppendLine(string.Format("Results: {0}. Elapsed Seconds: {1}", score, (double)elapsedMiliseconds / 1000));
sb.AppendLine();
sb.AppendLine(string.Format("True Positives: {0}", statistics.TruePositiveInfo));
sb.AppendLine(string.Format("True Positives Percentile: {0}", statistics.TruePositivePercentileInfo));
sb.AppendLine(string.Format("False Negatives: {0}", statistics.FalseNegativesInfo));
sb.AppendLine(string.Format("False Negatives Percentile: {0}", statistics.FalseNegativesPercentileInfo));
sb.AppendLine(string.Format("False Positives: {0}", statistics.FalsePositivesInfo));
sb.AppendLine(string.Format("False Positives Percentile: {0}", statistics.FalsePositivesPercentileInfo));
}
private TestRunnerEventArgs GetTestRunnerEventArgsForFinishedTestIteration(IStride queryStride, int seconds, List <int> startAts, FScore fscore, HammingDistanceResultStatistics statistics, int iteration, Stopwatch stopwatch, int verified)
{
return(new TestRunnerEventArgs
{
FScore = fscore,
RowWithDetails =
new object[]
{
GetInsertMetadata(), queryStride.ToString(), seconds, startAts[iteration],
fscore.Precision, fscore.Recall, fscore.F1,
statistics.TruePositiveInfo,
statistics.TruePositivePercentileInfo,
statistics.FalseNegativesInfo,
statistics.FalseNegativesPercentileInfo,
statistics.FalsePositivesInfo,
statistics.FalsePositivesPercentileInfo,
(double)stopwatch.ElapsedMilliseconds / 1000
},
Verified = verified
});
}
private void RunTestScenario(string folderWithPositives, string folderWithNegatives, IStride queryStride, int seconds, List <int> startAts)
{
int iterations = startAts.Count;
var positives = AllFiles(folderWithPositives);
var negatives = AllFiles(folderWithNegatives);
for (int iteration = 0; iteration < iterations; ++iteration)
{
OnTestRunnerEvent(OngoingActionEvent, new TestRunnerOngoingEventArgs
{
Message = $"Iteration {iteration + 1} out of {iterations} with {queryStride}, query seconds {seconds}"
});
var stopwatch = new Stopwatch();
stopwatch.Start();
int trueNegatives = 0, truePositives = 0, falseNegatives = 0, falsePositives = 0, verified = 0;
var truePositiveHammingDistance = new ConcurrentBag <int>();
var falseNegativesHammingDistance = new ConcurrentBag <int>();
var falsePositivesHammingDistance = new ConcurrentBag <int>();
var sb = TestRunnerWriter.StartTestIteration();
int currentIteration = iteration;
int startAt = startAts[currentIteration];
Parallel.ForEach(
positives,
positive =>
{
Interlocked.Increment(ref verified);
var tags = GetTagsFromFile(positive);
var actualTrack = GetActualTrack(tags);
var queryResult = BuildQuery(queryStride, seconds, positive, startAt).Result;
if (!queryResult.ContainsMatches)
{
Interlocked.Increment(ref falseNegatives);
var notFoundLine = GetNotFoundLine(tags);
AppendLine(sb, notFoundLine);
OnTestRunnerEvent(
PositiveNotFoundEvent,
GetTestRunnerEventArgs(
truePositives,
trueNegatives,
falsePositives,
falseNegatives,
notFoundLine,
verified));
return;
}
var recognizedTrack = queryResult.BestMatch.Track;
bool isSuccessful = recognizedTrack.Id.Equals(actualTrack.Id);
if (isSuccessful)
{
Interlocked.Increment(ref truePositives);
truePositiveHammingDistance.Add((int)queryResult.BestMatch.Score);
}
else
{
Interlocked.Increment(ref falsePositives);
falseNegativesHammingDistance.Add((int)queryResult.BestMatch.Score);
}
var foundLine = GetFoundLine(ToTrackString(actualTrack.Artist, actualTrack.Title), recognizedTrack, isSuccessful, queryResult);
AppendLine(sb, foundLine);
OnTestRunnerEvent(PositiveFoundEvent, GetTestRunnerEventArgs(truePositives, trueNegatives, falsePositives, falseNegatives, foundLine, verified));
});
Parallel.ForEach(
negatives,
negative =>
{
Interlocked.Increment(ref verified);
var tags = GetTagsFromFile(negative);
var queryResult = BuildQuery(queryStride, seconds, negative, startAt).Result;
if (!queryResult.ContainsMatches)
{
Interlocked.Increment(ref trueNegatives);
var notFoundLine = GetNotFoundLine(tags);
AppendLine(sb, notFoundLine);
OnTestRunnerEvent(
NegativeNotFoundEvent,
GetTestRunnerEventArgs(
truePositives,
trueNegatives,
falsePositives,
falseNegatives,
notFoundLine,
verified));
return;
}
var recognizedTrack = queryResult.BestMatch.Track;
falsePositivesHammingDistance.Add((int)queryResult.BestMatch.Score);
Interlocked.Increment(ref falsePositives);
var foundLine = GetFoundLine(ToTrackString(tags), recognizedTrack, false, queryResult);
AppendLine(sb, foundLine);
OnTestRunnerEvent(
NegativeFoundEvent,
GetTestRunnerEventArgs(truePositives, trueNegatives, falsePositives, falseNegatives, foundLine, verified));
});
stopwatch.Stop();
var fscore = new FScore(truePositives, trueNegatives, falsePositives, falseNegatives);
var stats = HammingDistanceResultStatistics.From(
truePositiveHammingDistance,
falseNegativesHammingDistance,
falsePositivesHammingDistance,
testRunnerConfig.Percentiles);
TestRunnerWriter.FinishTestIteration(sb, fscore, stats, stopwatch.ElapsedMilliseconds);
TestRunnerWriter.SaveTestIterationToFolder(sb, pathToResultsFolder, queryStride, GetInsertMetadata(), seconds, startAt);
var finishedTestIteration = GetTestRunnerEventArgsForFinishedTestIteration(queryStride, seconds, startAts, fscore, stats, iteration, stopwatch, verified);
OnTestRunnerEvent(TestIterationFinishedEvent, finishedTestIteration);
TestRunnerWriter.AppendLine(suite, finishedTestIteration.RowWithDetails);
}
}
