public CVResultVoter(List <CrossValidationResult> cvResults) { this._cvResults = cvResults; _finalConfusionMatrix = CrossValidationResult.getAggregatedConfusionMatrix(_cvResults); _totalClassification = CrossValidationResult.mergeNodeClassificationsSum(_cvResults); }
internal List <CrossValidationResult> performCrossValidation() { Console.WriteLine("Starting Cross Validation - Number of Runs: " + _configuration.CrossValidationNumRuns); ConcurrentBag <CrossValidationResult> cvResults = new ConcurrentBag <CrossValidationResult>(); for (int i = 1; i <= _configuration.CrossValidationNumRuns; i++) { Console.WriteLine("Performing Run " + i + " of " + _configuration.CrossValidationNumRuns); CrossValidationResult result = _performCrossValidation(); cvResults.Add(result); } Console.WriteLine("Done. CV Results: " + cvResults.Count); return(cvResults.ToList <CrossValidationResult>()); }
private CrossValidationResult _performCrossValidation() { // Create a new Cross-validation algorithm passing the data set size and the number of folds var crossvalidation = new CrossValidation(size: _inputs.Length, folds: _configuration.CrossValidationNumFolds); Console.WriteLine("Num Samples: " + crossvalidation.Samples); // ConcurrentDictionary<string, NodeClassification> nodeClassifications = new ConcurrentDictionary<string, NodeClassification>(); CrossValidationResult cvResult = new CrossValidationResult(); // Define a fitting function using Support Vector Machines. The objective of this // function is to learn a SVM in the subset of the data indicated by cross-validation. crossvalidation.Fitting = delegate(int k, int[] indicesTrain, int[] indicesValidation) { // The fitting function is passing the indices of the original set which // should be considered training data and the indices of the original set // which should be considered validation data. // Lets now grab the training data: var trainingInputs = _inputs.Submatrix(indicesTrain); var trainingOutputs = _outputs.Submatrix(indicesTrain); var trainingNames = _names.Submatrix(indicesTrain); // And now the validation data: var validationInputs = _inputs.Submatrix(indicesValidation); var validationOutputs = _outputs.Submatrix(indicesValidation); var validationNames = _names.Submatrix(indicesValidation); if (validationNames.Intersect <String>(trainingNames).Count() > 0) { Console.WriteLine("Warning, Training and Validation Set not disjunct."); Utility.writeToConsole <String>(validationNames.Intersect(trainingNames).ToArray()); } //Console.WriteLine("=== TRAINING ==="); //Utility.writeToConsole<String>(trainingNames); //Utility.writeToConsole<int>(trainingInputs); //Utility.writeToConsole<int>(trainingOutputs); //Console.WriteLine("=== VALIDATION ==="); //Utility.writeToConsole<String>(validationNames); //Utility.writeToConsole<double>(validationInputs); //Utility.writeToConsole<int>(validationOutputs); //Console.WriteLine("=== INTERSECTION ==="); //String[] intersection = Utility.Intersection(validationNames, trainingNames); //Utility.writeToConsole<String>(intersection); // Create a Kernel Support Vector Machine to operate on the set var svm = new KernelSupportVectorMachine(_kernel, _inputs[0].Length); //Accord.MachineLearning.Boosting.Boost<KernelSupportVectorMachine> b; // Create a training algorithm and learn the training data var smo = new SequentialMinimalOptimization(svm, trainingInputs, trainingOutputs) { // Set learning parameters Tolerance = _configuration.Tolerance, PositiveWeight = _configuration.WeightPositiveClass, NegativeWeight = _configuration.WeightNegativeClass, UseClassProportions = _configuration.UseComputedWeights, UseComplexityHeuristic = true }; //if (!_configuration.UseHeuristicalComplexity) _smo.Complexity = _configuration.Complexity; double trainingError = smo.Run(); // Now we can compute the validation error on the validation data: double validationError = smo.ComputeError(validationInputs, validationOutputs); // Predictions & Confusion Matrix List <int> predictions = new List <int>(); List <double> rawPredictions = new List <double>(); int index = 0; foreach (double[] inputVector in validationInputs) { // Compute the decision output for vector // Console.WriteLine(validationNames[index]); // Utility.writeToConsole<double>(inputVector); double rawPrediction = svm.Compute(inputVector); rawPredictions.Add(rawPrediction); int prediction = rawPrediction > 0.0d ? +1 : -1; predictions.Add(prediction); try { // Update Node Classifications cvResult.AddOrUpdateClassification(validationNames[index], rawPrediction, validationOutputs[index]); } catch (IndexOutOfRangeException) { Console.WriteLine("Failed to update CV Result: Input Vector larger than Name Vector."); } index++; } ConfusionMatrix confusionMatrix = new ConfusionMatrix(predictions.ToArray(), validationOutputs, 1, -1); cvResult.ConfusionMatrices.Add(confusionMatrix); // Return a new information structure containing the model and the errors achieved. return(new CrossValidationValues(svm, trainingError, validationError)); }; // Compute the cross-validation var result = crossvalidation.Compute(); // Finally, access the measured performance. double trainingErrors = result.Training.Mean; double validationErrors = result.Validation.Mean; Console.WriteLine("Training Errors: " + result.Training.Mean); Console.WriteLine("Validation Errors: " + result.Validation.Mean); ConfusionMatrix aggregatedConfusionMatrix = ConfusionMatrix.Combine(cvResult.ConfusionMatrices.ToArray()); return(cvResult); }
/* Basic Merge of all Predictions */ /* * internal static Dictionary<string, NodeClassification> mergeNodeClassifications(List<CrossValidationResult> cvResults) * { * Dictionary<string, NodeClassification> nodeClassifications = new Dictionary<string, NodeClassification>(); * * foreach (var cvResult in cvResults) * { * foreach (var key in cvResult.Classifications.Keys) * { * if (!nodeClassifications.ContainsKey(key)) * { * // Add Key * nodeClassifications.Add(key, cvResult.Classifications[key]); * } * * else * { * // Key exists * foreach (double prediction in cvResult.Classifications[key].RawPredictions) * { * nodeClassifications[key].addPrediction(prediction); * } * } * } * } * return nodeClassifications; * }*/ internal void merge(CrossValidationResult otherResult) { }