public static string[][] CreateFolds(ICaseFrame cases, int foldCount, bool shuffle)
{
if (cases == null)
{
throw new ArgumentNullException(nameof(cases));
}
if (foldCount <= 0)
{
throw new ArgumentOutOfRangeException(nameof(foldCount), "Fold count should be greater than 0.");
}
var ids = GetIds(cases, shuffle).ToArray();
var foldSize = Math.DivRem(ids.Length, foldCount, out var upperFoldCount);
var folds = new string[foldCount][];
var idIndex = 0;
for (var i = 0; i < foldCount; i++)
{
folds[i] = i < upperFoldCount ? new string[foldSize + 1] : new string[foldSize];
for (var j = 0; j < folds[i].Length; j++)
{
folds[i][j] = ids[idIndex];
idIndex += 1;
}
}
return(folds);
}
public DenseMatrix GetCovarianceMatrix(ICaseFrame cases)
{
var data = new List <double[]>();
var cursor = cases.GetCaseCursor();
while (cursor.MoveNext())
{
var attributes = new double[cases.Schema.Count];
for (var i = 0; i < cases.Schema.Count; i++)
{
attributes[i] = cursor.GetAttribute <double>(cases.Schema[i]);
}
data.Add(attributes);
}
var dataMat = DenseMatrix.Build.DenseOfRowArrays(data);
var covMat = new DenseMatrix(dataMat.ColumnCount, dataMat.ColumnCount);
for (var i = 0; i < dataMat.ColumnCount; i++)
{
for (var j = 0; j < dataMat.ColumnCount; j++)
{
covMat[i, j] = dataMat.Column(i).Covariance(dataMat.Column(j));
}
}
return(covMat);
}
public void Prepare(ICaseFrame cases)
{
foreach (var transformer in transformers)
{
transformer.Prepare(cases);
}
}
public MahalanobisDistance(ICaseFrame cases)
{
if (cases == null)
{
throw new ArgumentNullException(nameof(cases));
}
covarianceMatrix = GetCovarianceMatrix(cases);
}
public override ICaseFrame Transform(ICaseFrame cases)
{
if (!isPrepared)
{
throw new InvalidOperationException("Normalizer should be prepared before the transform.");
}
if (cases == null)
{
throw new ArgumentNullException(nameof(cases));
}
return(new AttributeTransformingCaseFrame(cases, this));
}
public WriteCursor(ICaseFrame caseFrame)
{
if (caseFrame == null)
{
throw new ArgumentNullException(nameof(caseFrame));
}
cursor = caseFrame.GetCaseCursor();
definition = new CaseDefinitionBuilder(typeof(TCase), cursor.Schema).Build();
idSetter = GenerateIdSetter();
solutionSetter = GenerateSolutionSetter();
setters = GenerateSetters();
}
private static IEnumerable <string> GetIds(ICaseFrame cases, bool shuffle)
{
var caseIds = new List <string>();
using (var cursor = cases.GetCaseCursor())
{
while (cursor.MoveNext())
{
caseIds.Add(cursor.GetId());
}
}
return(shuffle
? caseIds.OrderBy(a => Guid.NewGuid())
: (IEnumerable <string>)caseIds);
}
public static IEnumerable <TCase> ToEnumerable <TCase>(this ICaseFrame caseFrame) where TCase : class, new()
{
if (caseFrame == null)
{
throw new ArgumentNullException();
}
using (var cursor = new WriteCursor <TCase>(caseFrame))
{
while (cursor.MoveNext())
{
var targetCase = new TCase();
cursor.CreateCase(targetCase);
yield return(targetCase);
}
}
}
public static ICaseFrame FilterCases(this ICaseFrame caseFrame, IEnumerable <string> caseIds)
{
if (caseFrame == null)
{
throw new ArgumentNullException(nameof(caseFrame));
}
if (caseIds == null)
{
throw new ArgumentNullException(nameof(caseIds));
}
var ids = new HashSet <string>(caseIds);
var filterTransformer = new FilterTransformer(c => ids.Contains(c.GetId()));
filterTransformer.Prepare(caseFrame);
return(filterTransformer.Transform(caseFrame));
}
public override void Prepare(ICaseFrame cases)
{
if (cases == null)
{
throw new ArgumentNullException(nameof(cases));
}
BindSelectedAttributes(cases.Schema);
using (var cursor = cases.GetCaseCursor())
{
while (cursor.MoveNext())
{
foreach (var binding in AttributeBinding)
{
var getter = cursor.GetAttributeGetter <double>(cases.Schema[binding.Attribute]);
UpdateMinMax(binding.Binding, getter);
}
}
}
isPrepared = true;
}
public AttributeTransformingCaseFrame(ICaseFrame frame, AttributeTransformer transformer) : base(frame)
{
this.transformer = transformer ?? throw new ArgumentNullException(nameof(transformer));
}
public ICaseFrame Transform(ICaseFrame cases)
{
return(transformers.Aggregate(cases, (current, transformer) => transformer.Transform(current)));
}
public ReasonerContext(ICaseFrame cases, ICase targetCase)
{
Cases = cases ?? throw new ArgumentNullException(nameof(cases));
TargetCase = targetCase ?? throw new ArgumentNullException(nameof(targetCase));;
}
private static FoldValidationResult <TSolution> ProcessFold <TSolution>(string[][] folds, int testFoldIndex, ICaseFrame cases, ICaseTransformer pipeline, Reasoner reasoner)
{
var watch = Stopwatch.StartNew();
var result = new FoldValidationResult <TSolution>(testFoldIndex, folds[testFoldIndex].Length);
var testFold = cases.FilterCases(folds[testFoldIndex]);
var trainFold = cases.FilterCases(folds.Where((f, i) => i != testFoldIndex).SelectMany(f => f));
pipeline.Prepare(trainFold);
testFold = pipeline.Transform(testFold);
trainFold = pipeline.Transform(trainFold);
reasoner.SetData(trainFold);
using (var cursor = testFold.GetCaseCursor())
{
while (cursor.MoveNext())
{
var trueSolution = cursor.GetSolution <TSolution>();
var predictedSolution = reasoner.GetSolution <TSolution>(cursor);
result.AddPrediction(trueSolution, predictedSolution);
}
}
watch.Stop();
result.ValidationTime = watch.ElapsedMilliseconds;
return(result);
}
public static CrossValidationResult <TSolution> Validate <TSolution>(string[][] folds, ICaseFrame cases, Reasoner reasoner, ICaseTransformer pipeline = null)
{
if (folds == null)
{
throw new ArgumentNullException(nameof(folds));
}
if (cases == null)
{
throw new ArgumentNullException(nameof(cases));
}
if (reasoner == null)
{
throw new ArgumentNullException(nameof(reasoner));
}
var results = new FoldValidationResult <TSolution> [folds.Length];
for (var i = 0; i < folds.Length; i++)
{
results[i] = ProcessFold <TSolution>(folds, i, cases, pipeline, reasoner);
}
return(new CrossValidationResult <TSolution>(results));
}
public abstract ICaseFrame Transform(ICaseFrame cases);
public FilteringCaseFrame(ICaseFrame frame, FilterTransformer transformer) : base(frame)
{
this.transformer = transformer ?? throw new ArgumentNullException(nameof(transformer));
}
public void SetData(ICaseFrame cases)
{
data = cases ?? throw new ArgumentNullException(nameof(cases));
}
protected TransformingCaseFrame(ICaseFrame frame)
{
OriginalFrame = frame ?? throw new ArgumentNullException(nameof(frame));
}
public ICaseFrame Transform(ICaseFrame cases)
{
return(new FilteringCaseFrame(cases, this));
}
public void Prepare(ICaseFrame cases)
{
}
public abstract void Prepare(ICaseFrame cases);
public static CrossValidationResult <TSolution> ValidateInParallel <TSolution>(string[][] folds, ICaseFrame cases, Func <Reasoner> reasonerBuilder, Func <ICaseTransformer> pipelineBuilder = null, ParallelOptions options = null)
{
if (folds == null)
{
throw new ArgumentNullException(nameof(folds));
}
if (cases == null)
{
throw new ArgumentNullException(nameof(cases));
}
if (reasonerBuilder == null)
{
throw new ArgumentNullException(nameof(reasonerBuilder));
}
var results = new FoldValidationResult <TSolution> [folds.Length];
Parallel.For(0, folds.Length, options ?? new ParallelOptions(), i =>
{
results[i] = ProcessFold <TSolution>(folds, i, cases, pipelineBuilder?.Invoke(), reasonerBuilder());
});
return(new CrossValidationResult <TSolution>(results));
}