public double GetAvgStdErr(string expName, string algoName, OrdinalPerfMetric metric, IEnumerable <LblT> orderedLabels, out double stderr, double confidenceLevel = 0.95)
{
double zval = StatsUtils.GetZScore(confidenceLevel);
double stddev;
double avg = GetAvg(expName, algoName, metric, orderedLabels, out stddev);
stderr = zval * stddev / Math.Sqrt(GetFoldCount(expName, algoName));
return(avg);
}
public double GetScore(OrdinalPerfMetric metric, IEnumerable <LblT> orderedLabels = null)
{
LblT[] labels = null;
if (orderedLabels == null)
{
if (typeof(LblT).IsEnum) // take label order from the enum type definition
{
labels = Enum.GetValues(typeof(LblT)).Cast <LblT>().ToArray();
}
}
else
{
labels = orderedLabels as LblT[] ?? orderedLabels.ToArray();
}
switch (metric)
{
case OrdinalPerfMetric.MeanAbsoluteError:
return(GetError(labels, GetLinearWeights(labels)));
case OrdinalPerfMetric.MeanSquaredError:
return(GetError(labels, GetSquareWeights(labels)));
case OrdinalPerfMetric.ErrorTolerance1:
return(GetError(labels, GetErrorXWeights(labels, 1)));
case OrdinalPerfMetric.AccuracyTolerance1:
return(1 - GetError(labels, GetErrorXWeights(labels, 1)));
case OrdinalPerfMetric.MeanAbsoluteErrorNormalized1:
return(GetError(labels, GetLinearWeights(labels, true)));
case OrdinalPerfMetric.MeanSquaredErrorNormalized1:
return(GetError(labels, GetSquareWeights(labels, true)));
case OrdinalPerfMetric.WeightedKappaLinear:
return(GetWeightedKappa(labels, GetLinearWeights(labels, true)));
case OrdinalPerfMetric.WeightedKappaSquared:
return(GetWeightedKappa(labels, GetSquareWeights(labels, true)));
case OrdinalPerfMetric.KAlphaLinear:
return(GetKrippendorffsAlpha(labels, GetLinearWeights(labels, true)));
case OrdinalPerfMetric.KAlphaInterval:
return(GetKrippendorffsAlpha(labels, GetSquareWeights(labels, true)));
case OrdinalPerfMetric.F1AvgExtremeClasses:
return(GetF1AvgExtremeClasses(labels));
default:
throw new ArgumentValueException("invalid ordered metric");
}
}
public double GetVal(int foldNum, string expName, string algoName, OrdinalPerfMetric metric, IEnumerable <LblT> orderedLabels)
{
Utils.ThrowException(foldNum < 1 ? new ArgumentOutOfRangeException("foldNum") : null);
Utils.ThrowException(expName == null ? new ArgumentNullException("expName") : null);
Utils.ThrowException(algoName == null ? new ArgumentNullException("algoName") : null);
ConcurrentDictionary <string, FoldData> algoData;
if (mData.TryGetValue(expName, out algoData))
{
FoldData foldData;
if (algoData.TryGetValue(algoName, out foldData))
{
if (foldNum <= foldData.Count && foldData[foldNum - 1] != null)
{
return(foldData[foldNum - 1].GetScore(metric, orderedLabels));
}
}
}
return(Double.NaN);
}
public double GetAvg(string expName, string algoName, OrdinalPerfMetric metric, IEnumerable <LblT> orderedLabels)
{
double stdev;
return(GetAvg(expName, algoName, metric, orderedLabels, out stdev));
}
public double GetAvg(string expName, string algoName, OrdinalPerfMetric metric, IEnumerable <LblT> orderedLabels, out double stdev)
{
return(GetAvg(expName, algoName, mtx => mtx.GetScore(metric, orderedLabels), out stdev));
}