/// <summary> /// Initializes a new instance of the <see cref="MetricsSet" /> class. /// </summary> /// <param name="inputsSet">The inputs set.</param> /// <param name="resultsSet">The results set.</param> /// <param name="userName">Name of the user.</param> /// <param name="mode">The mode.</param> public MetricsSet(Inputs.DataSet inputsSet, Results.ResultsSet resultsSet, string userName, ExperimentMode mode) { this.UserName = userName; this.DataSetSize = inputsSet.Instances.Count; var instances = inputsSet.Instances; var instanceScores = instances.Zip(resultsSet.IsRepliedProbTrue, (x, y) => new KeyValuePair <Inputs.Instance, double>(x, y)).ToArray(); var positiveInstances = inputsSet.PositiveInstances; this.FractionPositive = (double)positiveInstances.Count / instances.Count; var calibrationCurve = Evaluator.CalibrationCurve(true, inputsSet, resultsSet.PredictionDicts, Bins, MinBinInstanceCount) .Select(ia => new Point(ia.First, ia.Second)); var precisionRecallCurve = InferMetrics.PrecisionRecallCurve(positiveInstances, instanceScores).ToList(); this.AreaUnderCurve = InferMetrics.AreaUnderRocCurve(positiveInstances, instanceScores); this.AveragePrecision = ComputeAveragePrecision(precisionRecallCurve, 0.1, 0.9); this.CalibrationError = Math.Sqrt(calibrationCurve.Select(ia => Math.Pow(ia.X - ia.Y, 2)).Average()); if (mode != ExperimentMode.Online && mode != ExperimentMode.Incremental) { this.CalibrationCurve = calibrationCurve.ToArray(); // Save some memory by not calculating these in online mode this.PrecisionRecallCurve = precisionRecallCurve.Select(ia => new Point(ia.First, ia.Second)).ToArray(); this.PrecisionRecallRandomCurve = new[] { new Point(0.0, this.FractionPositive), new Point(1.0, this.FractionPositive) }; this.RocCurve = InferMetrics.ReceiverOperatingCharacteristicCurve(instances.Where(ia => ia.Label), instanceScores) .Select(ia => new Point(ia.First, ia.Second)) .ToArray(); } this.ReplyCount = inputsSet.PositiveInstances.Count; this.ReplyFraction = (double)this.ReplyCount / inputsSet.Count; }
/// <summary> /// The instance source. /// </summary> /// <param name="dataSet">The data set.</param> /// <returns> /// The <see cref="IEnumerable{Instance}" />. /// </returns> private static IEnumerable <Inputs.Instance> InstanceSource(Inputs.DataSet dataSet) { return(dataSet.Instances); }