public void ShouldCalculateFScore() { var score = new FScore(90, 90, 10, 10); Assert.AreEqual(0.9d, score.Precision, 0.001); Assert.AreEqual(0.9d, score.Recall, 0.001); Assert.AreEqual(0.9d, score.F1, 0.001); }
private void RunTestScenario(string folderWithPositives, string folderWithNegatives, IStride queryStride, int seconds, List<int> startAts) { int iterations = startAts.Count; var positives = AllFiles(folderWithPositives); var negatives = AllFiles(folderWithNegatives); for (int iteration = 0; iteration < iterations; ++iteration) { OnTestRunnerEvent( OngoingActionEvent, new TestRunnerOngoingEventArgs { Message = string.Format( "Iteration {0} out of {1} with {2}, query seconds {3}", iteration + 1, iterations, queryStride, seconds) }); var stopwatch = new Stopwatch(); stopwatch.Start(); int trueNegatives = 0, truePositives = 0, falseNegatives = 0, falsePositives = 0, verified = 0; var truePositiveHammingDistance = new ConcurrentBag<int>(); var falseNegativesHammingDistance = new ConcurrentBag<int>(); var falsePositivesHammingDistance = new ConcurrentBag<int>(); var sb = TestRunnerWriter.StartTestIteration(); int currentIteration = iteration; int startAt = startAts[currentIteration]; Parallel.ForEach( positives, positive => { Interlocked.Increment(ref verified); var tags = GetTagsFromFile(positive); var actualTrack = GetActualTrack(tags); var queryResult = BuildQuery(queryStride, seconds, positive, startAt).Result; if (!queryResult.ContainsMatches) { Interlocked.Increment(ref falseNegatives); var notFoundLine = GetNotFoundLine(tags); AppendLine(sb, notFoundLine); OnTestRunnerEvent( PositiveNotFoundEvent, GetTestRunnerEventArgs( truePositives, trueNegatives, falsePositives, falseNegatives, notFoundLine, verified)); return; } var recognizedTrack = queryResult.BestMatch.Track; bool isSuccessful = recognizedTrack.TrackReference.Equals(actualTrack.TrackReference); if (isSuccessful) { Interlocked.Increment(ref truePositives); truePositiveHammingDistance.Add(queryResult.BestMatch.HammingSimilaritySum); } else { Interlocked.Increment(ref falsePositives); falseNegativesHammingDistance.Add(queryResult.BestMatch.HammingSimilaritySum); } var foundLine = GetFoundLine(ToTrackString(actualTrack), recognizedTrack, isSuccessful, queryResult); AppendLine(sb, foundLine); OnTestRunnerEvent(PositiveFoundEvent, GetTestRunnerEventArgs(truePositives, trueNegatives, falsePositives, falseNegatives, foundLine, verified)); }); Parallel.ForEach( negatives, negative => { Interlocked.Increment(ref verified); var tags = GetTagsFromFile(negative); var queryResult = BuildQuery(queryStride, seconds, negative, startAt).Result; if (!queryResult.ContainsMatches) { Interlocked.Increment(ref trueNegatives); var notFoundLine = GetNotFoundLine(tags); AppendLine(sb, notFoundLine); OnTestRunnerEvent( NegativeNotFoundEvent, GetTestRunnerEventArgs( truePositives, trueNegatives, falsePositives, falseNegatives, notFoundLine, verified)); return; } var recognizedTrack = queryResult.BestMatch.Track; falsePositivesHammingDistance.Add(queryResult.BestMatch.HammingSimilaritySum); Interlocked.Increment(ref falsePositives); var foundLine = GetFoundLine(ToTrackString(tags), recognizedTrack, false, queryResult); AppendLine(sb, foundLine); OnTestRunnerEvent( NegativeFoundEvent, GetTestRunnerEventArgs(truePositives, trueNegatives, falsePositives, falseNegatives, foundLine, verified)); }); stopwatch.Stop(); var fscore = new FScore(truePositives, trueNegatives, falsePositives, falseNegatives); var stats = HammingDistanceResultStatistics.From( truePositiveHammingDistance, falseNegativesHammingDistance, falsePositivesHammingDistance, testRunnerConfig.Percentiles); TestRunnerWriter.FinishTestIteration(sb, fscore, stats, stopwatch.ElapsedMilliseconds); TestRunnerWriter.SaveTestIterationToFolder(sb, pathToResultsFolder, queryStride, GetInsertMetadata(), seconds, startAt); var finishedTestIteration = GetTestRunnerEventArgsForFinishedTestIteration(queryStride, seconds, startAts, fscore, stats, iteration, stopwatch, verified); OnTestRunnerEvent(TestIterationFinishedEvent, finishedTestIteration); TestRunnerWriter.AppendLine(suite, finishedTestIteration.RowWithDetails); } }
private TestRunnerEventArgs GetTestRunnerEventArgsForFinishedTestIteration(IStride queryStride, int seconds, List<int> startAts, FScore fscore, HammingDistanceResultStatistics statistics, int iteration, Stopwatch stopwatch, int verified) { return new TestRunnerEventArgs { FScore = fscore, RowWithDetails = new object[] { this.GetInsertMetadata(), queryStride.ToString(), seconds, startAts[iteration], fscore.Precision, fscore.Recall, fscore.F1, statistics.TruePositiveInfo, statistics.TruePositivePercentileInfo, statistics.FalseNegativesInfo, statistics.FalseNegativesPercentileInfo, statistics.FalsePositivesInfo, statistics.FalsePositivesPercentileInfo, (double)stopwatch.ElapsedMilliseconds / 1000 }, Verified = verified }; }