/// <summary>
/// Estimates a suitable value for the Gaussian's kernel sigma
/// </summary>
///
private void btnEstimateSigma_Click(object sender, EventArgs e)
{
// Extract inputs
int rows = dgvTrainingSource.Rows.Count;
double[][] input = new double[rows][];
for (int i = 0; i < rows; i++)
{
input[i] = (double[])dgvTrainingSource.Rows[i].Cells["colTrainingFeatures"].Value;
}
Gaussian g = Gaussian.Estimate(input, input.Length / 4);
numSigma.Value = (decimal)g.Sigma;
}
private void btnEstimateGaussian_Click(object sender, EventArgs e)
{
DataTable source = dgvLearningSource.DataSource as DataTable;
// Creates a matrix from the source data table
double[,] sourceMatrix = source.ToMatrix(out columnNames);
// Get only the input vector values (in the first two columns)
double[][] inputs = sourceMatrix.GetColumns(0, 1, 2, 3, 4, 5).ToJagged();
DoubleRange range; // valid range will be returned as an out parameter
Gaussian gaussian = Gaussian.Estimate(inputs, inputs.Length, out range);
numSigma.Value = (decimal)gaussian.Sigma;
}
private IKernel createKernel(double[,] arr)
{
decimal numSigma = 0;
//Data.DataTable source = arr as Data.DataTable;
// Get only the input vector values (in the first two columns)
double[][] inputs = arr.GetColumns(0, 1).ToJagged();
DoubleRange range; // valid range will be returned as an out parameter
Gaussian gaussian = Gaussian.Estimate(inputs, inputs.Length, out range);
numSigma = (decimal)gaussian.Sigma;
return(new Accord.Statistics.Kernels.Gaussian((double)numSigma));
}
private void buttonEstimate_Click(object sender, EventArgs e)
{
if (MessageBox.Show(this, "This action will first save the configuration. Do you wish to Continue?", "Save required", MessageBoxButtons.YesNo, MessageBoxIcon.Question) == DialogResult.Yes)
{
if (save())
{
double[,] sourceMatrix = _model.FeatureTable.ToMatrix <double>(_activeFeatures.ToArray());
double[][] inputs = sourceMatrix.ToArray();
DoubleRange range;
if (groupGaussianKernel.Enabled)
{
Gaussian gaussian = Gaussian.Estimate(inputs, inputs.Length, out range);
tbGaussianSigma.Value = (decimal)gaussian.Sigma;
}
if (groupPolyKernel.Enabled)
{
}
if (groupSigmoidKernel.Enabled)
{
Sigmoid sigmoid = Sigmoid.Estimate(inputs, inputs.Length, out range);
if (sigmoid.Alpha < (double)Decimal.MaxValue && sigmoid.Alpha > (double)Decimal.MinValue)
{
tbSigmoidAlpha.Value = (decimal)sigmoid.Alpha;
}
if (sigmoid.Constant < (double)Decimal.MaxValue && sigmoid.Constant > (double)Decimal.MinValue)
{
tbSigmoidConst.Value = (decimal)sigmoid.Constant;
}
}
if (groupLaplacianKernel.Enabled)
{
Laplacian laplacian = Laplacian.Estimate(inputs, inputs.Length, out range);
tbLaplacianSigma.Value = (decimal)laplacian.Sigma;
}
}
else
{
MessageBox.Show("Failed to fully save the configuration.\nTherefore configuration properties can not be estimated.");
}
}
}
private void setKernalType(KernelType k)
{
switch (k)
{
case KernelType.Linear:
kernel = new Linear();
break;
case KernelType.Quadratic:
kernel = new Quadratic();
break;
case KernelType.Sigmoid:
kernel = Sigmoid.Estimate(independentVls);
break;
case KernelType.Spline:
kernel = new Spline();
break;
case KernelType.ChiSquared:
kernel = new ChiSquare();
break;
case KernelType.Gaussian:
kernel = Gaussian.Estimate(independentVls);
break;
case KernelType.Multiquadric:
kernel = new Multiquadric();
break;
case KernelType.InverseMultquadric:
kernel = new InverseMultiquadric();
break;
case KernelType.Laplacian:
kernel = Laplacian.Estimate(independentVls);
break;
default:
kernel = new Polynomial(2);
break;
}
}
public void ComputeTest5()
{
double[][] inputs = training.Submatrix(null, 0, 3);
int[] labels = Tools.Scale(0, 1, -1, 1, training.GetColumn(4)).ToInt32();
Gaussian kernel = Gaussian.Estimate(inputs);
var machine = new KernelSupportVectorMachine(kernel, inputs[0].Length);
var smo = new SequentialMinimalOptimization(machine, inputs, labels);
smo.Complexity = 1.0;
smo.UseClassProportions = true;
double error = smo.Run();
Assert.AreEqual(1, smo.Complexity);
Assert.AreEqual(0.4, smo.PositiveWeight);
Assert.AreEqual(1.0, smo.NegativeWeight);
Assert.AreEqual(0.4, smo.WeightRatio, 1e-10);
Assert.AreEqual(0.38095238095238093, error);
Assert.AreEqual(265.78327637381551, (machine.Kernel as Gaussian).Sigma);
Assert.AreEqual(32, machine.SupportVectors.Length);
int[] actual = new int[labels.Length];
for (int i = 0; i < actual.Length; i++)
{
actual[i] = Math.Sign(machine.Compute(inputs[i]));
}
ConfusionMatrix matrix = new ConfusionMatrix(actual, labels);
Assert.AreEqual(7, matrix.FalseNegatives);
Assert.AreEqual(9, matrix.FalsePositives);
Assert.AreEqual(5, matrix.TruePositives);
Assert.AreEqual(21, matrix.TrueNegatives);
Assert.AreEqual(0.41666666666666669, matrix.Sensitivity);
Assert.AreEqual(0.7, matrix.Specificity);
}
public void GaussianEstimateTest()
{
// Suppose we have the following data
//
double[][] data =
{
new double[] { 5.1, 3.5, 1.4, 0.2 },
new double[] { 5.0, 3.6, 1.4, 0.2 },
new double[] { 4.9, 3.0, 1.4, 0.2 },
new double[] { 5.8, 4.0, 1.2, 0.2 },
new double[] { 4.7, 3.2, 1.3, 0.2 },
};
// Estimate an appropriate sigma from data
var kernel = Gaussian.Estimate(new Linear(0), data);
double sigma = kernel.Sigma; // 0.36055512
double sigma2 = kernel.SigmaSquared;
Assert.AreEqual(0.36055512754639879, sigma);
Assert.AreEqual(sigma * sigma, sigma2);
}
public void GaussianEstimateTest()
{
// Suppose we have the following data
//
double[][] data =
{
new double[] { 5.1, 3.5, 1.4, 0.2 },
new double[] { 5.0, 3.6, 1.4, 0.2 },
new double[] { 4.9, 3.0, 1.4, 0.2 },
new double[] { 5.8, 4.0, 1.2, 0.2 },
new double[] { 4.7, 3.2, 1.3, 0.2 },
};
// Estimate an appropriate sigma from data
var dtw = new DynamicTimeWarping(1);
var kernel = Gaussian.Estimate(dtw, data);
double sigma = kernel.Sigma;
double sigma2 = kernel.SigmaSquared;
Assert.AreEqual(0.044282096049367413, sigma);
Assert.AreEqual(sigma * sigma, sigma2);
}
public void UseClassProportionsTest()
{
var dataset = KernelSupportVectorMachineTest.training;
var inputs = dataset.Submatrix(null, 0, 3);
var labels = Accord.Math.Tools.Scale(0, 1, -1, 1, dataset.GetColumn(4)).ToInt32();
Gaussian kernel = Gaussian.Estimate(inputs);
var machine = new KernelSupportVectorMachine(kernel, inputs[0].Length);
var smo = new SequentialMinimalOptimization(machine, inputs, labels);
smo.Complexity = 1.0;
smo.UseClassProportions = true;
double error = smo.Run();
Assert.AreEqual(1, smo.Complexity);
Assert.AreEqual(0.4, smo.PositiveWeight);
Assert.AreEqual(1.0, smo.NegativeWeight);
Assert.AreEqual(0.4, smo.WeightRatio, 1e-10);
Assert.AreEqual(0.2857142857142857, error);
Assert.AreEqual(265.78327637381551, ((Gaussian)machine.Kernel).Sigma);
Assert.AreEqual(26, machine.SupportVectors.Length);
int[] actual = new int[labels.Length];
for (int i = 0; i < actual.Length; i++)
{
actual[i] = Math.Sign(machine.Compute(inputs[i]));
}
ConfusionMatrix matrix = new ConfusionMatrix(actual, labels);
Assert.AreEqual(12, matrix.FalseNegatives);
Assert.AreEqual(0, matrix.FalsePositives);
Assert.AreEqual(0, matrix.TruePositives);
Assert.AreEqual(30, matrix.TrueNegatives);
}
public void WeightRatioTest()
{
var dataset = KernelSupportVectorMachineTest.training;
var inputs = dataset.Submatrix(null, 0, 3);
var labels = Accord.Math.Tools.Scale(0, 1, -1, 1, dataset.GetColumn(4)).ToInt32();
Gaussian kernel = Gaussian.Estimate(inputs);
{
var machine = new KernelSupportVectorMachine(kernel, inputs[0].Length);
var smo = new SequentialMinimalOptimization(machine, inputs, labels);
smo.Complexity = 1.0;
smo.WeightRatio = 10;
double error = smo.Run();
Assert.AreEqual(1.0, smo.PositiveWeight);
Assert.AreEqual(0.1, smo.NegativeWeight);
Assert.AreEqual(0.7142857142857143, error);
Assert.AreEqual(265.78327637381551, ((Gaussian)machine.Kernel).Sigma);
Assert.AreEqual(39, machine.SupportVectors.Length);
int[] actual = new int[labels.Length];
for (int i = 0; i < actual.Length; i++)
{
actual[i] = Math.Sign(machine.Compute(inputs[i]));
}
ConfusionMatrix matrix = new ConfusionMatrix(actual, labels);
Assert.AreEqual(12, matrix.TruePositives); // has more importance
Assert.AreEqual(0, matrix.FalseNegatives); // has more importance
Assert.AreEqual(30, matrix.FalsePositives);
Assert.AreEqual(0, matrix.TrueNegatives);
Assert.AreEqual(1.0, matrix.Sensitivity);
Assert.AreEqual(0.0, matrix.Specificity);
Assert.AreEqual(0.44444444444444448, matrix.FScore);
Assert.AreEqual(0.0, matrix.MatthewsCorrelationCoefficient);
}
{
var machine = new KernelSupportVectorMachine(kernel, inputs[0].Length);
var smo = new SequentialMinimalOptimization(machine, inputs, labels);
smo.Complexity = 1.0;
smo.WeightRatio = 0.1;
double error = smo.Run();
Assert.AreEqual(0.1, smo.PositiveWeight);
Assert.AreEqual(1.0, smo.NegativeWeight);
Assert.AreEqual(0.21428571428571427, error);
Assert.AreEqual(265.78327637381551, ((Gaussian)machine.Kernel).Sigma);
Assert.AreEqual(18, machine.SupportVectors.Length);
int[] actual = new int[labels.Length];
for (int i = 0; i < actual.Length; i++)
{
actual[i] = Math.Sign(machine.Compute(inputs[i]));
}
ConfusionMatrix matrix = new ConfusionMatrix(actual, labels);
Assert.AreEqual(8, matrix.FalseNegatives);
Assert.AreEqual(1, matrix.FalsePositives); // has more importance
Assert.AreEqual(4, matrix.TruePositives);
Assert.AreEqual(29, matrix.TrueNegatives); // has more importance
Assert.AreEqual(0.33333333333333331, matrix.Sensitivity);
Assert.AreEqual(0.96666666666666667, matrix.Specificity);
Assert.AreEqual(0.47058823529411764, matrix.FScore);
Assert.AreEqual(0.41849149947774944, matrix.MatthewsCorrelationCoefficient);
}
}
public bool Train()
{
if (type == ClassifierType.SVM)
{
if (trainData != null)
{
float[,] mat = trainData.Data;
double[][] inputs = new double[trainData.Rows][];
for (int i = 0; i < trainData.Rows; i++)
{
int numFeatures = Math.Max(trainData.Cols, 2);
inputs[i] = new double[numFeatures];
for (int j = 0; j < numFeatures; j++)
{
inputs[i][j] = mat[i, Math.Min(trainData.Cols - 1, j)];
}
}
int[] outputs = new int[trainLabels.Rows];
for (int i = 0; i < trainLabels.Rows; i++)
{
outputs[i] = trainLabels[i, 0];
}
int numClasses = nameForID.Count;
if (numClasses <= 1)
{
return(true);
}
IKernel kernel;
switch (kernelType)
{
case KernelType.Linear:
kernel = new Linear();
break;
case KernelType.Poly:
kernel = new Polynomial(3);
break;
case KernelType.Rbf:
if (inputs.Length < 20)
{
kernel = new Gaussian(0.1);
}
else
{
kernel = Gaussian.Estimate(inputs, inputs.Length / 4);
}
break;
case KernelType.Chi2:
kernel = new ChiSquare();
break;
default:
kernel = inputs[0].Length > 20 ? (IKernel)(new ChiSquare()) : (IKernel)(Gaussian.Estimate(inputs, inputs.Length / 4));
break;
}
svm = new MulticlassSupportVectorMachine(inputs: inputs[0].Length, kernel: kernel, classes: numClasses);
var teacher = new MulticlassSupportVectorLearning(svm, inputs, outputs)
{
Algorithm = (machine, classInputs, classOutputs, i, j) => new SequentialMinimalOptimization(machine, classInputs, classOutputs)
{
Tolerance = 1e-6,
UseComplexityHeuristic = true
//Tolerance = 0.001,
//Complexity = 1,
//CacheSize = 200
}
};
try
{
double error = teacher.Run();
}
catch (Exception ex) { Debug.WriteLine("Error training SVM: " + ex.Message); }
teacher = new MulticlassSupportVectorLearning(svm, inputs, outputs)
{
Algorithm = (machine, classInputs, classOutputs, i, j) => new ProbabilisticOutputCalibration(machine, classInputs, classOutputs)
};
try
{
double error = teacher.Run();
}
catch (Exception ex) { Debug.WriteLine("Error calibrating SVM: " + ex.Message); }
return(true);
}
return(false);
}
else if (type == ClassifierType.NeuralNet)
{
float[,] mat = trainData.Data;
double[][] inputs = new double[trainData.Rows][];
List <int> randomOrder = new List <int>(trainData.Rows);
for (int i = 0; i < trainData.Rows; i++)
{
randomOrder.Add(i);
}
randomOrder.Shuffle();
for (int i = 0; i < trainData.Rows; i++)
{
inputs[randomOrder[i]] = new double[trainData.Cols];
for (int j = 0; j < trainData.Cols; j++)
{
inputs[randomOrder[i]][j] = mat[i, j];
}
}
int[] classes = new int[trainLabels.Rows];
for (int i = 0; i < trainLabels.Rows; i++)
{
classes[randomOrder[i]] = trainLabels[i, 0];
}
// First we have to convert this problem into a way that the neural
// network can handle. The first step is to expand the classes into
// indicator vectors, where a 1 into a position signifies that this
// position indicates the class the sample belongs to.
//
double[][] outputs = Accord.Statistics.Tools.Expand(classes, -1, +1);
// Create an activation function for the net
var function = new BipolarSigmoidFunction();
// Create an activation network with the function and
// N inputs, (M+N)/2 hidden neurons and M possible outputs:
int N = inputs[0].Length;
int M = nameForID.Count;
network = new ActivationNetwork(function, N, (M + N) / 2, M);
// Randomly initialize the network
new NguyenWidrow(network).Randomize();
// Teach the network using parallel Rprop:
var teacher = new ParallelResilientBackpropagationLearning(network);
double error = 1.0;
int iter = 0;
while (error > 1e-7 && iter < 100)
{
//for (int iter = 0; iter < 10000 && error > 1e-5; iter++)
error = teacher.RunEpoch(inputs, outputs);
iter++;
}
return(true);
}
else if (type == ClassifierType.KMeans)
{
List <double[]> rows = new List <double[]>();
float[,] data = trainData.Data;
for (int i = 0; i < trainData.Rows; i++)
{
double[] row = new double[trainData.Cols];
for (int j = 0; j < trainData.Cols; j++)
{
row[j] = data[i, j];
}
rows.Add(row);
}
double[][] points = rows.ToArray();
kmeans = new Accord.MachineLearning.KMeans(nameForID.Count * 2);
int[] labels = kmeans.Compute(points);
clusterClasses = new Dictionary <int, Dictionary <int, int> >();
for (int i = 0; i < labels.Length; i++)
{
if (!clusterClasses.ContainsKey(labels[i]))
{
clusterClasses.Add(labels[i], new Dictionary <int, int>());
}
int label = trainLabels.Data[i, 0];
if (!clusterClasses[labels[i]].ContainsKey(label))
{
clusterClasses[labels[i]].Add(label, 0);
}
clusterClasses[labels[i]][label]++;
}
return(true);
}
else
{
return(true);
}
}
public void ComputeTest6()
{
double[][] inputs = training.Submatrix(null, 0, 3);
int[] labels = Tools.Scale(0, 1, -1, 1, training.GetColumn(4)).ToInt32();
Gaussian kernel = Gaussian.Estimate(inputs);
{
var machine = new KernelSupportVectorMachine(kernel, inputs[0].Length);
var smo = new SequentialMinimalOptimization(machine, inputs, labels);
smo.Complexity = 1.0;
smo.WeightRatio = 30 / 12.0;
double error = smo.Run();
Assert.AreEqual(1, smo.PositiveWeight);
Assert.AreEqual(0.4, smo.NegativeWeight);
Assert.AreEqual(0.21428571428571427, error);
Assert.AreEqual(265.78327637381551, (machine.Kernel as Gaussian).Sigma);
Assert.AreEqual(34, machine.SupportVectors.Length);
int[] actual = new int[labels.Length];
for (int i = 0; i < actual.Length; i++)
{
actual[i] = Math.Sign(machine.Compute(inputs[i]));
}
ConfusionMatrix matrix = new ConfusionMatrix(actual, labels);
Assert.AreEqual(9, matrix.FalseNegatives);
Assert.AreEqual(0, matrix.FalsePositives);
Assert.AreEqual(3, matrix.TruePositives);
Assert.AreEqual(30, matrix.TrueNegatives);
Assert.AreEqual(0.25, matrix.Sensitivity);
Assert.AreEqual(1.0, matrix.Specificity);
Assert.AreEqual(0.4, matrix.FScore);
Assert.AreEqual(0.4385290096535146, matrix.MatthewsCorrelationCoefficient);
}
{
var machine = new KernelSupportVectorMachine(kernel, inputs[0].Length);
var smo = new SequentialMinimalOptimization(machine, inputs, labels);
smo.Complexity = 1.0;
smo.WeightRatio = 12 / 30.0;
double error = smo.Run();
Assert.AreEqual(0.4, smo.PositiveWeight);
Assert.AreEqual(1.0, smo.NegativeWeight);
Assert.AreEqual(0.38095238095238093, error);
Assert.AreEqual(265.78327637381551, (machine.Kernel as Gaussian).Sigma);
Assert.AreEqual(32, machine.SupportVectors.Length);
int[] actual = new int[labels.Length];
for (int i = 0; i < actual.Length; i++)
{
actual[i] = Math.Sign(machine.Compute(inputs[i]));
}
ConfusionMatrix matrix = new ConfusionMatrix(actual, labels);
Assert.AreEqual(7, matrix.FalseNegatives);
Assert.AreEqual(9, matrix.FalsePositives);
Assert.AreEqual(5, matrix.TruePositives);
Assert.AreEqual(21, matrix.TrueNegatives);
Assert.AreEqual(0.41666666666666669, matrix.Sensitivity);
Assert.AreEqual(0.7, matrix.Specificity);
Assert.AreEqual(0.38461538461538458, matrix.FScore);
Assert.AreEqual(0.11180339887498948, matrix.MatthewsCorrelationCoefficient);
}
}
