public void RunTest2()
{
var dataset = SequentialMinimalOptimizationTest.yinyang;
double[][] inputs = dataset.Submatrix(null, 0, 1).ToArray();
int[] labels = dataset.GetColumn(2).ToInt32();
var svm = new SupportVectorMachine(inputs: 2);
var teacher = new ProbabilisticCoordinateDescent(svm, inputs, labels);
teacher.Tolerance = 1e-10;
teacher.Complexity = 1e+10;
double error = teacher.Run();
double[] weights = svm.ToWeights();
Assert.AreEqual(0.12, error);
Assert.AreEqual(3, weights.Length);
Assert.AreEqual(-1.3231203367770932, weights[0], 1e-8);
Assert.AreEqual(-3.0227742288788493, weights[1], 1e-8);
Assert.AreEqual(-0.73074823290553259, weights[2], 1e-8);
Assert.AreEqual(svm.Threshold, weights[0]);
}
/// <summary>
/// Creates a Support Vector Machine and teaches it to recognize
/// the previously loaded dataset using the current UI settings.
/// </summary>
///
private void btnCreate_Click(object sender, EventArgs e)
{
if (dgvLearningSource.DataSource == null)
{
MessageBox.Show("Please load some data first.");
return;
}
// Finishes and save any pending changes to the given data
dgvLearningSource.EndEdit();
// Creates a matrix from the entire source data table
double[,] table = (dgvLearningSource.DataSource as DataTable).ToMatrix(out columnNames);
// Get only the input vector values (first two columns)
double[][] inputs = table.GetColumns(0, 1).ToArray();
// Get only the output labels (last column)
int[] outputs = table.GetColumn(2).ToInt32();
// Create a sparse logistic learning algorithm
var pcd = new ProbabilisticCoordinateDescent()
{
// Set learning parameters
Complexity = (double)numC.Value,
Tolerance = (double)numT.Value,
PositiveWeight = (double)numPositiveWeight.Value,
NegativeWeight = (double)numNegativeWeight.Value,
};
try
{
// Run
svm = pcd.Learn(inputs, outputs);
lbStatus.Text = "Training complete!";
}
catch (ConvergenceException)
{
lbStatus.Text = "Convergence could not be attained. " +
"The learned machine might still be usable.";
}
svm.Compress(); // reduce support vectors to a single weight vector
Trace.Assert(svm.SupportVectors.Length == 1);
Trace.Assert(svm.Weights.Length == 1);
createSurface(table);
// Show feature weight importance
double[] weights = svm.SupportVectors[0].Abs();
string[] featureNames = columnNames.RemoveAt(columnNames.Length - 1);
dgvSupportVectors.DataSource = new ArrayDataView(weights, featureNames);
CreateBarGraph(weights, featureNames);
}
public void RunTest()
{
double[][] input =
{
new double[] { 55, 0 }, // 0 - no cancer
new double[] { 28, 0 }, // 0
new double[] { 65, 1 }, // 0
new double[] { 46, 0 }, // 1 - have cancer
new double[] { 86, 1 }, // 1
new double[] { 56, 1 }, // 1
new double[] { 85, 0 }, // 0
new double[] { 33, 0 }, // 0
new double[] { 21, 1 }, // 0
new double[] { 42, 1 }, // 1
};
double[] output =
{
0, 0, 0, 1, 1, 1, 0, 0, 0, 1
};
int[] labels = output.Apply(x => x > 0 ? +1 : -1);
var svm = new SupportVectorMachine(inputs: 2);
var teacher = new ProbabilisticCoordinateDescent(svm, input, labels);
teacher.Tolerance = 1e-10;
teacher.Complexity = 1e+10;
Assert.IsFalse(svm.IsProbabilistic);
double error = teacher.Run();
Assert.IsTrue(svm.IsProbabilistic);
var regression = LogisticRegression.FromWeights(svm.ToWeights());
double[] actual = new double[output.Length];
for (int i = 0; i < actual.Length; i++)
{
actual[i] = regression.Compute(input[i]);
}
double ageOdds = regression.GetOddsRatio(1); // 1.0208597028836701
double smokeOdds = regression.GetOddsRatio(2); // 5.8584748789881331
Assert.AreEqual(0.2, error);
Assert.AreEqual(1.0208597028836701, ageOdds, 1e-4);
Assert.AreEqual(5.8584748789881331, smokeOdds, 1e-4);
Assert.IsFalse(Double.IsNaN(ageOdds));
Assert.IsFalse(Double.IsNaN(smokeOdds));
Assert.AreEqual(-2.4577464307294092, regression.Intercept, 1e-8);
Assert.AreEqual(-2.4577464307294092, regression.Coefficients[0], 1e-8);
Assert.AreEqual(0.020645118265359252, regression.Coefficients[1], 1e-8);
Assert.AreEqual(1.7678893101571855, regression.Coefficients[2], 1e-8);
}
public void KernelTest2()
{
var dataset = SequentialMinimalOptimizationTest.GetYingYang();
var inputs = dataset.Submatrix(null, 0, 1).ToJagged();
var labels = dataset.GetColumn(2).ToInt32();
var svm = new KernelSupportVectorMachine(new Linear(1), inputs: 2);
var p = new ProbabilisticCoordinateDescent(svm, inputs, labels);
Assert.NotNull(p);
}
public void KernelTest1()
{
var dataset = SequentialMinimalOptimizationTest.GetYingYang();
double[][] inputs = dataset.Submatrix(null, 0, 1).ToJagged();
int[] labels = dataset.GetColumn(2).ToInt32();
double e1, e2;
double[] w1, w2;
{
Accord.Math.Random.Generator.Seed = 0;
var svm = new SupportVectorMachine(inputs: 2);
var teacher = new ProbabilisticCoordinateDescent(svm, inputs, labels);
teacher.Tolerance = 1e-10;
teacher.Complexity = 1e+10;
e1 = teacher.Run();
w1 = svm.ToWeights();
}
{
Accord.Math.Random.Generator.Seed = 0;
var svm = new KernelSupportVectorMachine(new Linear(0), inputs: 2);
var teacher = new ProbabilisticCoordinateDescent(svm, inputs, labels);
teacher.Tolerance = 1e-10;
teacher.Complexity = 1e+10;
e2 = teacher.Run();
w2 = svm.ToWeights();
}
Assert.AreEqual(e1, e2);
Assert.AreEqual(w1.Length, w2.Length);
Assert.AreEqual(w1[0], w2[0], 1e-8);
Assert.AreEqual(w1[1], w2[1], 1e-8);
Assert.AreEqual(w1[2], w2[2], 1e-8);
}
public void CalculateLogistic(double[][] x_train, int[] y_train)
{
try
{
var teacher = new ProbabilisticCoordinateDescent()
{
Tolerance = 1e-10,
Complexity = 1e+10,
};
var svm = teacher.Learn(x_train, y_train);
this.regression = (LogisticRegression)svm;
}
catch
{
MessageBox.Show("학습실패LR");
}
}
public void logistic_regression_sparse_test()
{
#region doc_logreg_sparse
// Declare some training data. This is exactly the same
// data used in the LogisticRegression documentation page
// Suppose we have the following data about some patients.
// The first variable is continuous and represent patient
// age. The second variable is dichotomic and give whether
// they smoke or not (This is completely fictional data).
// We also know if they have had lung cancer or not, and
// we would like to know whether smoking has any connection
// with lung cancer (This is completely fictional data).
Sparse <double>[] input =
{ // age, smokes?, had cancer?
Sparse.FromDense(new double[] { 55, 0 }), // false - no cancer
Sparse.FromDense(new double[] { 28, 0 }), // false
Sparse.FromDense(new double[] { 65, 1 }), // false
Sparse.FromDense(new double[] { 46, 0 }), // true - had cancer
Sparse.FromDense(new double[] { 86, 1 }), // true
Sparse.FromDense(new double[] { 56, 1 }), // true
Sparse.FromDense(new double[] { 85, 0 }), // false
Sparse.FromDense(new double[] { 33, 0 }), // false
Sparse.FromDense(new double[] { 21, 1 }), // false
Sparse.FromDense(new double[] { 42, 1 }), // true
};
double[] output = // Whether each patient had lung cancer or not
{
0, 0, 0, 1, 1, 1, 0, 0, 0, 1
};
// Create the L1-regularization learning algorithm
var teacher = new ProbabilisticCoordinateDescent <Linear, Sparse <double> >()
{
Tolerance = 1e-10,
Complexity = 1e+10, // learn a hard-margin model
};
// Learn the L1-regularized machine
var svm = teacher.Learn(input, output);
// Convert the svm to logistic regression
var regression = (LogisticRegression)svm;
// Compute the predicted outcome for inputs
bool[] predicted = regression.Decide(input.ToDense(regression.NumberOfInputs));
// Compute log-likelihood scores for the outputs
double[] scores = regression.LogLikelihood(input.ToDense(regression.NumberOfInputs));
// Compute odds-ratio as in the LogisticRegression example
double ageOdds = regression.GetOddsRatio(1); // 1.0208597029158772
double smokeOdds = regression.GetOddsRatio(2); // 5.8584748789881331
// Compute the classification error as in SVM example
double error = new ZeroOneLoss(output).Loss(predicted);
#endregion
var rsvm = (SupportVectorMachine)regression;
Assert.AreEqual(2, rsvm.NumberOfInputs);
Assert.AreEqual(2, rsvm.NumberOfOutputs); // TODO: Maybe should 1 rather than 2
double[] svmpred = svm.Score(input);
Assert.IsTrue(scores.IsEqual(svmpred, 1e-10));
Assert.AreEqual(0.2, error);
Assert.AreEqual(1.0208597029158772, ageOdds, 1e-4);
Assert.AreEqual(5.8584748789881331, smokeOdds, 1e-4);
Assert.AreEqual(-2.4577464307294092, regression.Intercept, 1e-8);
Assert.AreEqual(-2.4577464307294092, regression.Coefficients[0], 1e-8);
Assert.AreEqual(0.020645118265359252, regression.Coefficients[1], 1e-8);
Assert.AreEqual(1.7678893101571855, regression.Coefficients[2], 1e-8);
}
public static void train_one(Problem prob, Parameters param, out double[] w, double Cp, double Cn)
{
double[][] inputs = prob.Inputs;
int[] labels = prob.Outputs.Apply(x => x >= 0 ? 1 : -1);
double eps = param.Tolerance;
int pos = 0;
for (int i = 0; i < labels.Length; i++)
{
if (labels[i] >= 0)
{
pos++;
}
}
int neg = prob.Outputs.Length - pos;
double primal_solver_tol = eps * Math.Max(Math.Min(pos, neg), 1.0) / prob.Inputs.Length;
SupportVectorMachine svm = new SupportVectorMachine(prob.Dimensions);
ISupportVectorMachineLearning teacher = null;
switch (param.Solver)
{
case LibSvmSolverType.L2RegularizedLogisticRegression:
// l2r_lr_fun
teacher = new ProbabilisticNewtonMethod(svm, inputs, labels)
{
PositiveWeight = Cp,
NegativeWeight = Cn,
Tolerance = primal_solver_tol
}; break;
case LibSvmSolverType.L2RegularizedL2LossSvc:
// fun_obj=new l2r_l2_svc_fun(prob, C);
teacher = new LinearNewtonMethod(svm, inputs, labels)
{
PositiveWeight = Cp,
NegativeWeight = Cn,
Tolerance = primal_solver_tol
}; break;
case LibSvmSolverType.L2RegularizedL2LossSvcDual:
// solve_l2r_l1l2_svc(prob, w, eps, Cp, Cn, L2R_L2LOSS_SVC_DUAL);
teacher = new LinearDualCoordinateDescent(svm, inputs, labels)
{
Loss = Loss.L2,
PositiveWeight = Cp,
NegativeWeight = Cn,
}; break;
case LibSvmSolverType.L2RegularizedL1LossSvcDual:
// solve_l2r_l1l2_svc(prob, w, eps, Cp, Cn, L2R_L1LOSS_SVC_DUAL);
teacher = new LinearDualCoordinateDescent(svm, inputs, labels)
{
Loss = Loss.L1,
PositiveWeight = Cp,
NegativeWeight = Cn,
}; break;
case LibSvmSolverType.L1RegularizedLogisticRegression:
// solve_l1r_lr(&prob_col, w, primal_solver_tol, Cp, Cn);
teacher = new ProbabilisticCoordinateDescent(svm, inputs, labels)
{
PositiveWeight = Cp,
NegativeWeight = Cn,
Tolerance = primal_solver_tol
}; break;
case LibSvmSolverType.L2RegularizedLogisticRegressionDual:
// solve_l2r_lr_dual(prob, w, eps, Cp, Cn);
teacher = new ProbabilisticDualCoordinateDescent(svm, inputs, labels)
{
PositiveWeight = Cp,
NegativeWeight = Cn,
Tolerance = primal_solver_tol,
}; break;
}
Trace.WriteLine("Training " + param.Solver);
// run the learning algorithm
var sw = Stopwatch.StartNew();
double error = teacher.Run();
sw.Stop();
// save the solution
w = svm.ToWeights();
Trace.WriteLine(String.Format("Finished {0}: {1} in {2}",
param.Solver, error, sw.Elapsed));
}
static public int[] ProbabilisticCoordinateDescent(double[][] input1, int[] labels, string SaveFile)
{
// http://accord-framework.net/docs/html/T_Accord_MachineLearning_VectorMachines_Learning_ProbabilisticCoordinateDescent.htm
/* This class implements a SupportVectorMachine learning algorithm specifically crafted for
* probabilistic linear machines only. It provides a L1- regularized coordinate descent learning
* algorithm for optimizing the learning problem. The code has been based on liblinear's method
* solve_l1r_lr method, whose original description is provided below.
*
* Liblinear's solver -s 6: L1R_LR. A coordinate descent algorithm for L1-regularized logistic
* regression (probabilistic svm) problems.
*/
int folds = 5;
Accord.Math.Random.Generator.Seed = 0;
var cv = CrossValidation.Create(
k: folds, // We will be using 10-fold cross validation
// First we define the learning algorithm:
learner: (p) => new ProbabilisticCoordinateDescent(),
// Now we have to specify how the n.b. performance should be measured:
loss: (actual, expected, p) => new ZeroOneLoss(expected).Loss(actual),
// This function can be used to perform any special
// operations before the actual learning is done, but
// here we will just leave it as simple as it can be:
fit: (teach, x, y, w) => teach.Learn(x, y, w),
// Finally, we have to pass the input and output data
// that will be used in cross-validation.
x: input1, y: labels
);
var cvresult = cv.Learn(input1, labels);
GeneralConfusionMatrix gcm = cvresult.ToConfusionMatrix(input1, labels);
var teacher = new ProbabilisticCoordinateDescent()
{
Tolerance = 1e-10,
Complexity = 1e+10,
// learn a hard-margin model
/* Complexity (cost) parameter C. Increasing the value of C forces the creation of a more
* accurate model that may not generalize well. If this value is not set and UseComplexityHeuristic
* is set to true, the framework will automatically guess a value for C. If this value is manually
* set to something else, then UseComplexityHeuristic will be automatically disabled and the given
* value will be used instead.
*/
};
var svm = teacher.Learn(input1, labels);
var svmregression = (LogisticRegression)svm;
ConfusionMatrix cm = ConfusionMatrix.Estimate(svm, input1, labels);
// accuracy, TP, FP, FN, TN and FScore Diagonal
Utility.OutPutStats(cvresult.NumberOfSamples, cvresult.NumberOfInputs, cvresult.Training.Mean,
gcm.Accuracy, cm.FalsePositives, cm.FalseNegatives, cm.FScore);
// Write the model out to a save file
string modelsavefilename = SaveFile.Replace(".csv", ".PCD.save");
svmregression.Save(modelsavefilename, compression: SerializerCompression.None);
bool[] answers = svmregression.Decide(input1);
return(Funcs.Utility.BoolToInt(answers));
}
