/// <summary>
/// Initializes a new instance of the <see cref="MetricsSet" /> class.
/// </summary>
/// <param name="inputsSet">The inputs set.</param>
/// <param name="resultsSet">The results set.</param>
/// <param name="userName">Name of the user.</param>
/// <param name="mode">The mode.</param>
public MetricsSet(Inputs.DataSet inputsSet, Results.ResultsSet resultsSet, string userName, ExperimentMode mode)
{
this.UserName = userName;
this.DataSetSize = inputsSet.Instances.Count;
var instances = inputsSet.Instances;
var instanceScores =
instances.Zip(resultsSet.IsRepliedProbTrue, (x, y) => new KeyValuePair <Inputs.Instance, double>(x, y)).ToArray();
var positiveInstances = inputsSet.PositiveInstances;
this.FractionPositive = (double)positiveInstances.Count / instances.Count;
var calibrationCurve =
Evaluator.CalibrationCurve(true, inputsSet, resultsSet.PredictionDicts, Bins, MinBinInstanceCount)
.Select(ia => new Point(ia.First, ia.Second));
var precisionRecallCurve = InferMetrics.PrecisionRecallCurve(positiveInstances, instanceScores).ToList();
this.AreaUnderCurve = InferMetrics.AreaUnderRocCurve(positiveInstances, instanceScores);
this.AveragePrecision = ComputeAveragePrecision(precisionRecallCurve, 0.1, 0.9);
this.CalibrationError = Math.Sqrt(calibrationCurve.Select(ia => Math.Pow(ia.X - ia.Y, 2)).Average());
if (mode != ExperimentMode.Online && mode != ExperimentMode.Incremental)
{
this.CalibrationCurve = calibrationCurve.ToArray();
// Save some memory by not calculating these in online mode
this.PrecisionRecallCurve = precisionRecallCurve.Select(ia => new Point(ia.First, ia.Second)).ToArray();
this.PrecisionRecallRandomCurve = new[] { new Point(0.0, this.FractionPositive), new Point(1.0, this.FractionPositive) };
this.RocCurve =
InferMetrics.ReceiverOperatingCharacteristicCurve(instances.Where(ia => ia.Label), instanceScores)
.Select(ia => new Point(ia.First, ia.Second))
.ToArray();
}
this.ReplyCount = inputsSet.PositiveInstances.Count;
this.ReplyFraction = (double)this.ReplyCount / inputsSet.Count;
}
/// <summary>
/// The instance source.
/// </summary>
/// <param name="dataSet">The data set.</param>
/// <returns>
/// The <see cref="IEnumerable{Instance}" />.
/// </returns>
private static IEnumerable <Inputs.Instance> InstanceSource(Inputs.DataSet dataSet)
{
return(dataSet.Instances);
}
