protected void RunModel(int foldN, IModel <LblT, ModelExT> model, ILabeledDataset <LblT, InputExT> trainSet, ILabeledDataset <LblT, ModelExT> mappedTrainSet, ILabeledDataset <LblT, InputExT> testSet, ILabeledDataset <LblT, ModelExT> mappedTestSet, CrossValidationTimeProfile modelProfile) { // train ILabeledDataset <LblT, ModelExT> usedTrainSet = BeforeTrain(foldN, model, trainSet, mappedTrainSet); Train(foldN, model, usedTrainSet); AfterTrain(foldN, model, usedTrainSet); modelProfile.TrainEndTime = DateTime.Now; // test modelProfile.TestStartTime = DateTime.Now; ILabeledDataset <LblT, ModelExT> usedTestSet = BeforeTest(foldN, model, testSet, mappedTestSet); PerfMatrix <LblT> foldMatrix = GetPerfMatrix(GetModelName(model), foldN); for (int i = 0; i < usedTestSet.Count; i++) { LabeledExample <LblT, ModelExT> le = usedTestSet[i]; Prediction <LblT> prediction = Predict(foldN, model, le); if (AfterPrediction(foldN, model, testSet[i].Example, le, prediction) && prediction.Any()) { foldMatrix.AddCount(le.Label, prediction.BestClassLabel); } } modelProfile.TestEndTime = DateTime.Now; AfterTest(foldN, model, usedTestSet); }
public PerfMatrix <LblT> GetSumPerfMatrix(string expName, string algoName) { Utils.ThrowException(expName == null ? new ArgumentNullException("expName") : null); Utils.ThrowException(algoName == null ? new ArgumentNullException("algoName") : null); ConcurrentDictionary <string, FoldData> algoData; if (mData.TryGetValue(expName, out algoData)) { FoldData foldData; if (algoData.TryGetValue(algoName, out foldData)) { PerfMatrix <LblT> sumMtx = new PerfMatrix <LblT>(mLblEqCmp); foreach (PerfMatrix <LblT> foldMtx in foldData.Where(m => m != null)) { Set <LblT> labels = foldMtx.GetLabels(); foreach (LblT actual in labels) { foreach (LblT predicted in labels) { sumMtx.AddCount(actual, predicted, foldMtx.Get(actual, predicted)); } } } return(sumMtx); } } return(null); }
public static PerfMatrix <LblT> GetCoincidenceMatrix <LblT>(PerfMatrix <LblT> mtx, IEnumerable <LblT> excludeValues = null) { Preconditions.CheckNotNull(mtx); LblT[] values = Enum.GetValues(typeof(LblT)).Cast <LblT>().Where(v => excludeValues == null || !excludeValues.Contains(v)).ToArray(); var result = new PerfMatrix <LblT>(); foreach (LblT first in values) { foreach (LblT second in values) { result.AddCount(first, second, mtx.Get(first, second) + mtx.Get(second, first)); } } return(result); }
public static PerfMatrix <LblT> GetCoincidenceMatrix <LblT, T, U>(PerfMatrix <LblT> mtx, IEnumerable <ILabeledExample <LblT, IAgreementExample <T, U> > > examples, out int unitCount, AgreementKind kind = AgreementKind.Inter, U observerId = default(U), bool resolveMultiplePerObserver = false) { Preconditions.CheckNotNull(mtx); ILabeledExample <LblT, IAgreementExample <T, U> >[][] units = Preconditions.CheckNotNull(examples) .GroupBy(e => e.Example.UnitId()) .Where(g => g.Count() > 1 && !EqualityComparer <T> .Default.Equals(g.Key, default(T))) // skip nulls or zeros .Select(g => g.ToArray()).ToArray(); switch (kind) { case AgreementKind.Self: units = EqualityComparer <U> .Default.Equals(observerId, default(U)) ? units // observer not specified .Select(u => u.GroupBy(e => e.Example.ObserverId()) .Where(g => g.Count() > 1 && !EqualityComparer <U> .Default.Equals(g.Key, default(U))) .SelectMany(g => g).ToArray()) .Where(u => u.Count() > 1).ToArray() : units .Select(u => u.Where(e => EqualityComparer <U> .Default.Equals(e.Example.ObserverId(), observerId)).ToArray()) .Where(u => u.Count() > 1).ToArray(); break; case AgreementKind.InterExcludingObserver: units = units .Select(u => u.Where(e => !EqualityComparer <U> .Default.Equals(e.Example.ObserverId(), observerId)).ToArray()) .Where(u => u.Count() > 1).ToArray(); break; case AgreementKind.InterIncludingUnits: units = units .Where(u => u.Any(e => EqualityComparer <U> .Default.Equals(e.Example.ObserverId(), observerId))).ToArray(); break; case AgreementKind.InterExcludingUnits: units = units .Where(u => u.All(e => !EqualityComparer <U> .Default.Equals(e.Example.ObserverId(), observerId))).ToArray(); break; } unitCount = 0; foreach (ILabeledExample <LblT, IAgreementExample <T, U> >[] unit_ in units) { ILabeledExample <LblT, IAgreementExample <T, U> >[] unit = unit_; // resolve multiple labels by the same user if (resolveMultiplePerObserver && kind != AgreementKind.Self) { unit = unit.GroupBy(e => e.Example.ObserverId()) .Select(ug => ug.GroupBy(ue => ue.Label) .OrderByDescending(ulg => ulg.Count()) .First().First()).ToArray(); if (unit.Length <= 1) { continue; } } for (int i = 0; i < unit.Length; i++) { for (int j = i + 1; j < unit.Length; j++) { mtx.AddCount(unit[i].Label, unit[j].Label, 1.0 / (unit.Length - 1)); mtx.AddCount(unit[j].Label, unit[i].Label, 1.0 / (unit.Length - 1)); } } unitCount++; } return(mtx); }