public CVResultVoter(List <CrossValidationResult> cvResults)
{
this._cvResults = cvResults;
_finalConfusionMatrix = CrossValidationResult.getAggregatedConfusionMatrix(_cvResults);
_totalClassification = CrossValidationResult.mergeNodeClassificationsSum(_cvResults);
}
internal List <CrossValidationResult> performCrossValidation()
{
Console.WriteLine("Starting Cross Validation - Number of Runs: " + _configuration.CrossValidationNumRuns);
ConcurrentBag <CrossValidationResult> cvResults = new ConcurrentBag <CrossValidationResult>();
for (int i = 1; i <= _configuration.CrossValidationNumRuns; i++)
{
Console.WriteLine("Performing Run " + i + " of " + _configuration.CrossValidationNumRuns);
CrossValidationResult result = _performCrossValidation();
cvResults.Add(result);
}
Console.WriteLine("Done. CV Results: " + cvResults.Count);
return(cvResults.ToList <CrossValidationResult>());
}
private CrossValidationResult _performCrossValidation()
{
// Create a new Cross-validation algorithm passing the data set size and the number of folds
var crossvalidation = new CrossValidation(size: _inputs.Length, folds: _configuration.CrossValidationNumFolds);
Console.WriteLine("Num Samples: " + crossvalidation.Samples);
// ConcurrentDictionary<string, NodeClassification> nodeClassifications = new ConcurrentDictionary<string, NodeClassification>();
CrossValidationResult cvResult = new CrossValidationResult();
// Define a fitting function using Support Vector Machines. The objective of this
// function is to learn a SVM in the subset of the data indicated by cross-validation.
crossvalidation.Fitting = delegate(int k, int[] indicesTrain, int[] indicesValidation)
{
// The fitting function is passing the indices of the original set which
// should be considered training data and the indices of the original set
// which should be considered validation data.
// Lets now grab the training data:
var trainingInputs = _inputs.Submatrix(indicesTrain);
var trainingOutputs = _outputs.Submatrix(indicesTrain);
var trainingNames = _names.Submatrix(indicesTrain);
// And now the validation data:
var validationInputs = _inputs.Submatrix(indicesValidation);
var validationOutputs = _outputs.Submatrix(indicesValidation);
var validationNames = _names.Submatrix(indicesValidation);
if (validationNames.Intersect <String>(trainingNames).Count() > 0)
{
Console.WriteLine("Warning, Training and Validation Set not disjunct.");
Utility.writeToConsole <String>(validationNames.Intersect(trainingNames).ToArray());
}
//Console.WriteLine("=== TRAINING ===");
//Utility.writeToConsole<String>(trainingNames);
//Utility.writeToConsole<int>(trainingInputs);
//Utility.writeToConsole<int>(trainingOutputs);
//Console.WriteLine("=== VALIDATION ===");
//Utility.writeToConsole<String>(validationNames);
//Utility.writeToConsole<double>(validationInputs);
//Utility.writeToConsole<int>(validationOutputs);
//Console.WriteLine("=== INTERSECTION ===");
//String[] intersection = Utility.Intersection(validationNames, trainingNames);
//Utility.writeToConsole<String>(intersection);
// Create a Kernel Support Vector Machine to operate on the set
var svm = new KernelSupportVectorMachine(_kernel, _inputs[0].Length);
//Accord.MachineLearning.Boosting.Boost<KernelSupportVectorMachine> b;
// Create a training algorithm and learn the training data
var smo = new SequentialMinimalOptimization(svm, trainingInputs, trainingOutputs)
{
// Set learning parameters
Tolerance = _configuration.Tolerance,
PositiveWeight = _configuration.WeightPositiveClass,
NegativeWeight = _configuration.WeightNegativeClass,
UseClassProportions = _configuration.UseComputedWeights,
UseComplexityHeuristic = true
};
//if (!_configuration.UseHeuristicalComplexity)
_smo.Complexity = _configuration.Complexity;
double trainingError = smo.Run();
// Now we can compute the validation error on the validation data:
double validationError = smo.ComputeError(validationInputs, validationOutputs);
// Predictions & Confusion Matrix
List <int> predictions = new List <int>();
List <double> rawPredictions = new List <double>();
int index = 0;
foreach (double[] inputVector in validationInputs)
{
// Compute the decision output for vector
// Console.WriteLine(validationNames[index]);
// Utility.writeToConsole<double>(inputVector);
double rawPrediction = svm.Compute(inputVector);
rawPredictions.Add(rawPrediction);
int prediction = rawPrediction > 0.0d ? +1 : -1;
predictions.Add(prediction);
try
{
// Update Node Classifications
cvResult.AddOrUpdateClassification(validationNames[index], rawPrediction, validationOutputs[index]);
}
catch (IndexOutOfRangeException)
{
Console.WriteLine("Failed to update CV Result: Input Vector larger than Name Vector.");
}
index++;
}
ConfusionMatrix confusionMatrix = new ConfusionMatrix(predictions.ToArray(), validationOutputs, 1, -1);
cvResult.ConfusionMatrices.Add(confusionMatrix);
// Return a new information structure containing the model and the errors achieved.
return(new CrossValidationValues(svm, trainingError, validationError));
};
// Compute the cross-validation
var result = crossvalidation.Compute();
// Finally, access the measured performance.
double trainingErrors = result.Training.Mean;
double validationErrors = result.Validation.Mean;
Console.WriteLine("Training Errors: " + result.Training.Mean);
Console.WriteLine("Validation Errors: " + result.Validation.Mean);
ConfusionMatrix aggregatedConfusionMatrix = ConfusionMatrix.Combine(cvResult.ConfusionMatrices.ToArray());
return(cvResult);
}
/* Basic Merge of all Predictions */
/*
* internal static Dictionary<string, NodeClassification> mergeNodeClassifications(List<CrossValidationResult> cvResults)
* {
* Dictionary<string, NodeClassification> nodeClassifications = new Dictionary<string, NodeClassification>();
*
* foreach (var cvResult in cvResults)
* {
* foreach (var key in cvResult.Classifications.Keys)
* {
* if (!nodeClassifications.ContainsKey(key))
* {
* // Add Key
* nodeClassifications.Add(key, cvResult.Classifications[key]);
* }
*
* else
* {
* // Key exists
* foreach (double prediction in cvResult.Classifications[key].RawPredictions)
* {
* nodeClassifications[key].addPrediction(prediction);
* }
* }
* }
* }
* return nodeClassifications;
* }*/
internal void merge(CrossValidationResult otherResult)
{
}