public double GetScore(PerfMetric metric, LblT lbl)
{
switch (metric)
{
case PerfMetric.Precision:
return(GetPrecision(lbl));
case PerfMetric.Accuracy:
case PerfMetric.Recall:
return(GetRecall(lbl));
case PerfMetric.F1:
return(GetF1(lbl));
case PerfMetric.MicroPrecision:
case PerfMetric.MicroRecall:
case PerfMetric.MicroF1:
case PerfMetric.MicroAccuracy:
return(GetMicroAverage());
case PerfMetric.MacroPrecision:
return(GetMacroPrecision());
case PerfMetric.MacroAccuracy:
case PerfMetric.MacroRecall:
return(GetMacroRecall());
case PerfMetric.MacroF1:
return(GetMacroF1());
default:
throw new ArgumentValueException("metric");
}
}
public double GetAvg(string expName, string algoName, PerfMetric metric, LblT lbl, out double stdev) // TODO: test if stdev is correct
{
Utils.ThrowException(expName == null ? new ArgumentNullException("expName") : null);
Utils.ThrowException(algoName == null ? new ArgumentNullException("algoName") : null);
stdev = double.NaN;
Dictionary <string, FoldData> algoData;
if (mData.TryGetValue(expName, out algoData))
{
FoldData foldData;
if (algoData.TryGetValue(algoName, out foldData))
{
double sum = 0;
foreach (PerfMatrix <LblT> mtx in foldData)
{
if (mtx == null)
{
throw new InvalidOperationException();
}
sum += mtx.GetScore(metric, lbl);
}
double avg = sum / (double)foldData.Count;
sum = 0;
foreach (PerfMatrix <LblT> mtx in foldData)
{
double val = mtx.GetScore(metric, lbl) - avg;
sum += val * val;
}
stdev = Math.Sqrt(sum / (double)foldData.Count);
return(avg);
}
}
return(double.NaN);
}
public double GetAvgStdErr(string expName, string algoName, PerfMetric metric, out double stderr, double confidenceLevel = 0.95)
{
double zval = StatsUtils.GetZScore(confidenceLevel);
double stddev;
double avg = GetAvg(expName, algoName, metric, out stddev);
stderr = zval * stddev / Math.Sqrt(GetFoldCount(expName, algoName));
return(avg);
}
public string ToString(string expName, PerfMetric metric, LblT lbl, string fmt) // set expName to null to get stats for all experiments
{
Utils.ThrowException(fmt == null ? new ArgumentNullException("fmt") : null);
ArrayList <string> expList = new ArrayList <string>(mData.Keys);
expList.Sort();
Set <string> tmp = new Set <string>();
foreach (Dictionary <string, FoldData> item in mData.Values)
{
tmp.AddRange(item.Keys);
}
ArrayList <string> algoList = new ArrayList <string>(tmp);
algoList.Sort();
StringBuilder sb = new StringBuilder();
// header
if (expName == null)
{
foreach (string exp in expList)
{
sb.Append("\t" + exp);
}
sb.AppendLine();
}
// rows
foreach (string algoName in algoList)
{
sb.Append(algoName);
if (expName == null)
{
foreach (string exp in expList)
{
double stdev;
double val = GetAvg(exp, algoName, metric, lbl, out stdev); // throws InvalidOperationException
sb.Append("\t" + val.ToString(fmt));
}
}
else
{
double stdev;
sb.Append("\t" + GetAvg(expName, algoName, metric, lbl, out stdev).ToString(fmt)); // throws InvalidOperationException
}
sb.AppendLine();
}
return(sb.ToString().TrimEnd());
}
public double GetScore(PerfMetric metric)
{
switch (metric)
{
case PerfMetric.Accuracy:
return(GetAccuracy());
case PerfMetric.MicroPrecision:
return(GetMicroPrecision());
case PerfMetric.MicroRecall:
return(GetMicroRecall());
case PerfMetric.MicroF1:
return(GetMicroF1());
case PerfMetric.MacroPrecision:
return(GetMacroPrecision());
case PerfMetric.MacroRecall:
return(GetMacroRecall());
case PerfMetric.MacroF1:
return(GetMacroF1());
case PerfMetric.Error:
return(GetError());
case PerfMetric.Kappa:
return(GetKappa());
case PerfMetric.KAlphaNominal:
return(GetKAlpha());
case PerfMetric.AccStdErrorConf90:
return(GetAccStdError(0.9));
case PerfMetric.AccStdErrorConf95:
return(GetAccStdError());
case PerfMetric.AccStdErrorConf99:
return(GetAccStdError(0.99));
default:
throw new ArgumentValueException("metric");
}
}
public string ToString(int foldNum, string expName, PerfMetric metric, LblT lbl, string fmt) // set expName to null to get stats for all experiments
{
Utils.ThrowException(foldNum < 1 ? new ArgumentOutOfRangeException("foldNum") : null);
Utils.ThrowException(fmt == null ? new ArgumentNullException("fmt") : null);
var expList = new ArrayList <string>(mData.Keys);
expList.Sort();
var tmp = new Set <string>();
foreach (ConcurrentDictionary <string, FoldData> item in mData.Values)
{
tmp.AddRange(item.Keys);
}
var algoList = new ArrayList <string>(tmp);
algoList.Sort();
var sb = new StringBuilder();
// header
if (expName == null)
{
foreach (string exp in expList)
{
sb.Append("\t" + exp);
}
sb.AppendLine();
}
// rows
foreach (string algoName in algoList)
{
sb.Append(algoName);
if (expName != null)
{
double val = GetVal(foldNum, expName, algoName, metric);
sb.Append("\t" + val.ToString(fmt));
}
else
{
foreach (string exp in expList)
{
sb.Append("\t" + GetVal(foldNum, exp, algoName, metric).ToString(fmt));
}
}
sb.AppendLine();
}
return(sb.ToString().TrimEnd());
}
public double GetVal(int foldNum, string expName, string algoName, PerfMetric metric, LblT lbl)
{
Utils.ThrowException(foldNum < 1 ? new ArgumentOutOfRangeException("foldNum") : null);
Utils.ThrowException(expName == null ? new ArgumentNullException("expName") : null);
Utils.ThrowException(algoName == null ? new ArgumentNullException("algoName") : null);
Dictionary <string, FoldData> algoData;
if (mData.TryGetValue(expName, out algoData))
{
FoldData foldData;
if (algoData.TryGetValue(algoName, out foldData))
{
if (foldNum <= foldData.Count && foldData[foldNum - 1] != null)
{
return(foldData[foldNum - 1].GetScore(metric, lbl));
}
}
}
return(double.NaN);
}
private void Start()
{
DevCommandManager.Instance.RegisterDevCommand(this);
PerfMetric = FindObjectOfType <PerfMetric>();
}
public static async Task SetPerfMetric(IConfiguration configuration, int reportId, PerfMetric perfMetric)
{
using (var connection = new SqlConnection(configuration.GetConnectionString("DefaultConnection")))
{
connection.Open();
perfMetric.ReportId = reportId;
string insertFCPSql = "INSERT INTO PerfMetric(reportId, score, numericValue, displayValue, title) VALUES (@ReportId, @Score, @NumericValue, @DisplayValue, @Title)";
await connection.ExecuteAsync(insertFCPSql, perfMetric);
}
}
public string ToString(string expName, PerfMetric metric)
{
return(ToString(expName, metric, default(LblT))); // throws InvalidOperationException
}
public string ToString(string expName, PerfMetric metric, LblT lbl)
{
return(ToString(expName, metric, lbl, "0.0000")); // throws InvalidOperationException
}
public string ToString(int foldNum, string expName, PerfMetric metric)
{
return(ToString(foldNum, expName, metric, default(LblT))); // throws ArgumentOutOfRangeException
}
public string ToString(int foldNum, string expName, PerfMetric metric, LblT lbl)
{
return(ToString(foldNum, expName, metric, lbl, "0.0000")); // throws ArgumentOutOfRangeException
}
public double GetAvg(string expName, string algoName, PerfMetric metric, LblT lbl)
{
double stdev;
return(GetAvg(expName, algoName, metric, lbl, out stdev)); // throws ArgumentNullException, InvalidOperationException
}
public double GetAvg(string expName, string algoName, PerfMetric metric)
{
double stdev;
return(GetAvg(expName, algoName, metric, out stdev));
}
public double GetAvg(string expName, string algoName, PerfMetric metric, out double stdev)
{
return(GetAvg(expName, algoName, mtx => mtx.GetScore(metric), out stdev));
}
