protected void RunModel(int foldN, IModel <LblT, ModelExT> model,
ILabeledDataset <LblT, InputExT> trainSet, ILabeledDataset <LblT, ModelExT> mappedTrainSet,
ILabeledDataset <LblT, InputExT> testSet, ILabeledDataset <LblT, ModelExT> mappedTestSet, CrossValidationTimeProfile modelProfile)
{
// train
ILabeledDataset <LblT, ModelExT> usedTrainSet = BeforeTrain(foldN, model, trainSet, mappedTrainSet);
Train(foldN, model, usedTrainSet);
AfterTrain(foldN, model, usedTrainSet);
modelProfile.TrainEndTime = DateTime.Now;
// test
modelProfile.TestStartTime = DateTime.Now;
ILabeledDataset <LblT, ModelExT> usedTestSet = BeforeTest(foldN, model, testSet, mappedTestSet);
PerfMatrix <LblT> foldMatrix = GetPerfMatrix(GetModelName(model), foldN);
for (int i = 0; i < usedTestSet.Count; i++)
{
LabeledExample <LblT, ModelExT> le = usedTestSet[i];
Prediction <LblT> prediction = Predict(foldN, model, le);
if (AfterPrediction(foldN, model, testSet[i].Example, le, prediction) && prediction.Any())
{
foldMatrix.AddCount(le.Label, prediction.BestClassLabel);
}
}
modelProfile.TestEndTime = DateTime.Now;
AfterTest(foldN, model, usedTestSet);
}
public static PerfMatrix <LblT> GetCoincidenceMatrix <LblT, T, U>(PerfMatrix <LblT> mtx, IEnumerable <ILabeledExample <LblT, IAgreementExample <T, U> > > examples,
AgreementKind kind = AgreementKind.Inter, U observerId = default(U), bool resolveMultiplePerObserver = false)
{
int count;
return(GetCoincidenceMatrix(mtx, examples, out count, kind, observerId, resolveMultiplePerObserver));
}
public PerfMatrix <LblT> GetSumPerfMatrix(string expName, string algoName)
{
Utils.ThrowException(expName == null ? new ArgumentNullException("expName") : null);
Utils.ThrowException(algoName == null ? new ArgumentNullException("algoName") : null);
ConcurrentDictionary <string, FoldData> algoData;
if (mData.TryGetValue(expName, out algoData))
{
FoldData foldData;
if (algoData.TryGetValue(algoName, out foldData))
{
PerfMatrix <LblT> sumMtx = new PerfMatrix <LblT>(mLblEqCmp);
foreach (PerfMatrix <LblT> foldMtx in foldData.Where(m => m != null))
{
Set <LblT> labels = foldMtx.GetLabels();
foreach (LblT actual in labels)
{
foreach (LblT predicted in labels)
{
sumMtx.AddCount(actual, predicted, foldMtx.Get(actual, predicted));
}
}
}
return(sumMtx);
}
}
return(null);
}
public void SetPerfMatrix(string expName, string algoName, int foldNum, PerfMatrix <LblT> matrix)
{
Utils.ThrowException(expName == null ? new ArgumentNullException("expName") : null);
Utils.ThrowException(algoName == null ? new ArgumentNullException("algoName") : null);
Utils.ThrowException(foldNum < 1 ? new ArgumentOutOfRangeException("foldNum") : null);
ConcurrentDictionary <string, FoldData> algoData;
if (mData.TryGetValue(expName, out algoData))
{
FoldData foldData;
if (algoData.TryGetValue(algoName, out foldData))
{
foldData.Resize(foldNum);
foldData[foldNum - 1] = matrix;
}
else
{
if (!algoData.TryAdd(algoName, foldData = new FoldData()))
{
foldData = algoData[algoName];
}
foldData.Put(foldNum, matrix);
}
}
else
{
FoldData foldData = new FoldData();
if (!mData.TryAdd(expName, algoData = new ConcurrentDictionary <string, FoldData>()))
{
algoData = mData[expName];
}
algoData.TryAdd(algoName, foldData);
foldData.Put(foldNum, matrix);
}
}
public static PerfMatrix <LblT> GetCoincidenceMatrix <LblT>(PerfMatrix <LblT> mtx, IEnumerable <LblT> excludeValues = null)
{
Preconditions.CheckNotNull(mtx);
LblT[] values = Enum.GetValues(typeof(LblT)).Cast <LblT>().Where(v => excludeValues == null || !excludeValues.Contains(v)).ToArray();
var result = new PerfMatrix <LblT>();
foreach (LblT first in values)
{
foreach (LblT second in values)
{
result.AddCount(first, second, mtx.Get(first, second) + mtx.Get(second, first));
}
}
return(result);
}
public static bool IsMatrixSymetrical <LblT>(PerfMatrix <LblT> mtx, IEnumerable <LblT> excludeValues = null)
{
Preconditions.CheckNotNull(mtx);
LblT[] values = Enum.GetValues(typeof(LblT)).Cast <LblT>().Where(v => excludeValues == null || !excludeValues.Contains(v)).ToArray();
foreach (LblT first in values)
{
foreach (LblT second in values)
{
if (EqualityComparer <LblT> .Default.Equals(first, second))
{
continue;
}
if (mtx.Get(first, second) != mtx.Get(second, first))
{
return(false);
}
}
}
return(true);
}
public PerfMatrix <LblT> GetPerfMatrix(string expName, string algoName, int foldNum)
{
Utils.ThrowException(expName == null ? new ArgumentNullException("expName") : null);
Utils.ThrowException(algoName == null ? new ArgumentNullException("algoName") : null);
Utils.ThrowException(foldNum < 1 ? new ArgumentOutOfRangeException("foldNum") : null);
ConcurrentDictionary <string, FoldData> algoData;
if (mData.TryGetValue(expName, out algoData))
{
FoldData foldData;
if (algoData.TryGetValue(algoName, out foldData))
{
foldData.Resize(foldNum);
return(foldData[foldNum - 1] ?? (foldData[foldNum - 1] = new PerfMatrix <LblT>(mLblEqCmp)));
}
if (!algoData.TryAdd(algoName, foldData = new FoldData()))
{
foldData = algoData[algoName];
}
var mtx = new PerfMatrix <LblT>(mLblEqCmp);
foldData.Put(foldNum, mtx);
return(mtx);
}
else
{
if (!mData.TryAdd(expName, algoData = new ConcurrentDictionary <string, FoldData>()))
{
algoData = mData[expName];
}
var foldData = new FoldData();
if (!algoData.TryAdd(algoName, foldData))
{
foldData = algoData[algoName];
}
var mtx = new PerfMatrix <LblT>(mLblEqCmp);
foldData.Put(foldNum, mtx);
return(mtx);
}
}
public PerfMatrix <LblT> GetPerfMatrix(string expName, string algoName, int foldNum)
{
Utils.ThrowException(expName == null ? new ArgumentNullException("expName") : null);
Utils.ThrowException(algoName == null ? new ArgumentNullException("algoName") : null);
Utils.ThrowException(foldNum < 1 ? new ArgumentOutOfRangeException("foldNum") : null);
Dictionary <string, FoldData> algoData;
if (mData.TryGetValue(expName, out algoData))
{
FoldData foldData;
if (algoData.TryGetValue(algoName, out foldData))
{
foldData.Resize(foldNum);
if (foldData[foldNum - 1] == null)
{
foldData[foldNum - 1] = new PerfMatrix <LblT>(mLblEqCmp);
}
return(foldData[foldNum - 1]);
}
else
{
algoData.Add(algoName, foldData = new FoldData());
PerfMatrix <LblT> mtx = new PerfMatrix <LblT>(mLblEqCmp);
foldData.Put(foldNum, mtx);
return(mtx);
}
}
else
{
FoldData foldData = new FoldData();
mData.Add(expName, algoData = new Dictionary <string, FoldData>());
algoData.Add(algoName, foldData);
PerfMatrix <LblT> mtx = new PerfMatrix <LblT>(mLblEqCmp);
foldData.Put(foldNum, mtx);
return(mtx);
}
}
public void Put(int foldNum, PerfMatrix <LblT> mtx)
{
Resize(foldNum);
this[foldNum - 1] = mtx;
}
public static PerfMatrix <LblT> GetCoincidenceMatrix <LblT, T, U>(PerfMatrix <LblT> mtx, IEnumerable <ILabeledExample <LblT, IAgreementExample <T, U> > > examples,
out int unitCount, AgreementKind kind = AgreementKind.Inter, U observerId = default(U), bool resolveMultiplePerObserver = false)
{
Preconditions.CheckNotNull(mtx);
ILabeledExample <LblT, IAgreementExample <T, U> >[][] units = Preconditions.CheckNotNull(examples)
.GroupBy(e => e.Example.UnitId())
.Where(g => g.Count() > 1 && !EqualityComparer <T> .Default.Equals(g.Key, default(T))) // skip nulls or zeros
.Select(g => g.ToArray()).ToArray();
switch (kind)
{
case AgreementKind.Self:
units = EqualityComparer <U> .Default.Equals(observerId, default(U))
? units // observer not specified
.Select(u => u.GroupBy(e => e.Example.ObserverId())
.Where(g => g.Count() > 1 && !EqualityComparer <U> .Default.Equals(g.Key, default(U)))
.SelectMany(g => g).ToArray())
.Where(u => u.Count() > 1).ToArray()
: units
.Select(u => u.Where(e => EqualityComparer <U> .Default.Equals(e.Example.ObserverId(), observerId)).ToArray())
.Where(u => u.Count() > 1).ToArray();
break;
case AgreementKind.InterExcludingObserver:
units = units
.Select(u => u.Where(e => !EqualityComparer <U> .Default.Equals(e.Example.ObserverId(), observerId)).ToArray())
.Where(u => u.Count() > 1).ToArray();
break;
case AgreementKind.InterIncludingUnits:
units = units
.Where(u => u.Any(e => EqualityComparer <U> .Default.Equals(e.Example.ObserverId(), observerId))).ToArray();
break;
case AgreementKind.InterExcludingUnits:
units = units
.Where(u => u.All(e => !EqualityComparer <U> .Default.Equals(e.Example.ObserverId(), observerId))).ToArray();
break;
}
unitCount = 0;
foreach (ILabeledExample <LblT, IAgreementExample <T, U> >[] unit_ in units)
{
ILabeledExample <LblT, IAgreementExample <T, U> >[] unit = unit_;
// resolve multiple labels by the same user
if (resolveMultiplePerObserver && kind != AgreementKind.Self)
{
unit = unit.GroupBy(e => e.Example.ObserverId())
.Select(ug => ug.GroupBy(ue => ue.Label)
.OrderByDescending(ulg => ulg.Count())
.First().First()).ToArray();
if (unit.Length <= 1)
{
continue;
}
}
for (int i = 0; i < unit.Length; i++)
{
for (int j = i + 1; j < unit.Length; j++)
{
mtx.AddCount(unit[i].Label, unit[j].Label, 1.0 / (unit.Length - 1));
mtx.AddCount(unit[j].Label, unit[i].Label, 1.0 / (unit.Length - 1));
}
}
unitCount++;
}
return(mtx);
}
