private static DecisionFunction SvmTrainOne(SvmProblem prob, SvmParameter param, double cp, double cn)
{
double[] alpha = new double[prob.Count];
SvmSolver.SolutionInfo si = new SvmSolver.SolutionInfo();
switch (param.svmType)
{
case SvmType.CSvc:
SolveCSvc(prob, param, alpha, si, cp, cn);
break;
case SvmType.NuSvc:
SolveNuSvc(prob, param, alpha, si);
break;
case SvmType.OneClass:
SolveOneClass(prob, param, alpha, si);
break;
case SvmType.EpsilonSvr:
SolveEpsilonSvr(prob, param, alpha, si);
break;
case SvmType.NuSvr:
SolveNuSvr(prob, param, alpha, si);
break;
}
Info("obj = " + si.obj + ", rho = " + si.rho + "\n");
// output SVs
int nSv = 0;
int nBsv = 0;
for (int i = 0; i < prob.Count; i++)
{
if (Math.Abs(alpha[i]) > 0)
{
++nSv;
if (prob.y[i] > 0)
{
if (Math.Abs(alpha[i]) >= si.upperBoundP)
{
++nBsv;
}
}
else
{
if (Math.Abs(alpha[i]) >= si.upperBoundN)
{
++nBsv;
}
}
}
}
Info("nSV = " + nSv + ", nBSV = " + nBsv + "\n");
DecisionFunction f = new DecisionFunction {
alpha = alpha, rho = si.rho
};
return(f);
}
private static void Main()
{
// Here we declare that our samples will be 1 dimensional column vectors. The reason for
// using a matrix here is that in general you can use N dimensional vectors as inputs to the
// krls object. But here we only have 1 dimension to make the example simple.
// Now we are making a typedef for the kind of kernel we want to use. I picked the
// radial basis kernel because it only has one parameter and generally gives good
// results without much fiddling.
// Here we declare an instance of the krls object. The first argument to the constructor
// is the kernel we wish to use. The second is a parameter that determines the numerical
// accuracy with which the object will perform part of the regression algorithm. Generally
// smaller values give better results but cause the algorithm to run slower (because it tries
// to use more "dictionary vectors" to represent the function it is learning.
// You just have to play with it to decide what balance of speed and accuracy is right
// for your problem. Here we have set it to 0.001.
//
// The last argument is the maximum number of dictionary vectors the algorithm is allowed
// to use. The default value for this field is 1,000,000 which is large enough that you
// won't ever hit it in practice. However, here we have set it to the much smaller value
// of 7. This means that once the krls object accumulates 7 dictionary vectors it will
// start discarding old ones in favor of new ones as it goes through the training process.
// In other words, the algorithm "forgets" about old training data and focuses on recent
// training samples. So the bigger the maximum dictionary size the longer its memory will
// be. But in this example program we are doing filtering so we only care about the most
// recent data. So using a small value is appropriate here since it will result in much
// faster filtering and won't introduce much error.
using (var rbk = new RadialBasisKernel <double, Matrix <double> >(0.1, 1, 1))
using (var test = new Krls <double, RadialBasisKernel <double, Matrix <double> > >(rbk, 0.001))
{
// now we train our object on a few samples of the sinc function.
using (var m = Matrix <double> .CreateTemplateParameterizeMatrix(1, 1))
{
for (double x = -10; x <= 4; x += 1)
{
m[0] = x;
test.Train(m, Sinc(x));
}
// now we output the value of the sinc function for a few test points as well as the
// value predicted by krls object.
m[0] = 2.5;
Console.WriteLine($"{Sinc(m[0])} {test.Operator(m)}");
m[0] = 0.1;
Console.WriteLine($"{Sinc(m[0])} {test.Operator(m)}");
m[0] = -4;
Console.WriteLine($"{Sinc(m[0])} {test.Operator(m)}");
m[0] = 5.0;
Console.WriteLine($"{Sinc(m[0])} {test.Operator(m)}");
// The output is as follows:
// 0.239389 0.239362
// 0.998334 0.998333
// -0.189201 -0.189201
// -0.191785 -0.197267
// The first column is the true value of t he sinc function and the second
// column is the output from the krls estimate.
// Another thing that is worth knowing is that just about everything in dlib is serializable.
// So for example, you can save the test object to disk and recall it later like so:
Krls <double, RadialBasisKernel <double, Matrix <double> > > .Serialize(test, "saved_krls_object.dat");
// Now let's open that file back up and load the krls object it contains.
using (var rbk2 = new RadialBasisKernel <double, Matrix <double> >(0.1, 1, 1))
{
var test2 = new Krls <double, RadialBasisKernel <double, Matrix <double> > >(rbk2, 0.001);
Krls <double, RadialBasisKernel <double, Matrix <double> > > .Deserialize("saved_krls_object.dat", ref test2);
// If you don't want to save the whole krls object (it might be a bit large)
// you can save just the decision function it has learned so far. You can get
// the decision function out of it by calling test.get_decision_function() and
// then you can serialize that object instead. E.g.
var funct = test2.GetDecisionFunction();
DecisionFunction <double, RadialBasisKernel <double, Matrix <double> > > .Serialize(funct, "saved_krls_function.dat");
}
}
}
}
//
// Interface functions
//
public static SvmModel SvmTrain(SvmProblem prob, SvmParameter param)
{
SvmModel model = new SvmModel {
param = param
};
if (param.svmType == SvmType.OneClass || param.svmType == SvmType.EpsilonSvr || param.svmType == SvmType.NuSvr)
{
// regression or one-class-svm
model.nrClass = 2;
model.label = null;
model.nSv = null;
model.probA = null;
model.probB = null;
model.svCoef = new double[1][];
if (param.probability && (param.svmType == SvmType.EpsilonSvr || param.svmType == SvmType.NuSvr))
{
model.probA = new double[1];
model.probA[0] = SvmSvrProbability(prob, param);
}
DecisionFunction f = SvmTrainOne(prob, param, 0, 0);
model.rho = new double[1];
model.rho[0] = f.rho;
int nSv = 0;
int i;
for (i = 0; i < prob.Count; i++)
{
if (Math.Abs(f.alpha[i]) > 0)
{
++nSv;
}
}
model.l = nSv;
model.sv = new BaseVector[nSv];
model.svCoef[0] = new double[nSv];
int j = 0;
for (i = 0; i < prob.Count; i++)
{
if (Math.Abs(f.alpha[i]) > 0)
{
model.sv[j] = prob.x[i];
model.svCoef[0][j] = f.alpha[i];
++j;
}
}
}
else
{
// classification
int l = prob.Count;
int[] tmpNrClass = new int[1];
int[][] tmpLabel = new int[1][];
int[][] tmpStart = new int[1][];
int[][] tmpCount = new int[1][];
int[] perm = new int[l];
// group training data of the same class
SvmGroupClasses(prob, tmpNrClass, tmpLabel, tmpStart, tmpCount, perm);
int nrClass = tmpNrClass[0];
int[] label = tmpLabel[0];
int[] start = tmpStart[0];
int[] count = tmpCount[0];
if (nrClass == 1)
{
Info("WARNING: training data in only one class. See README for details.\n");
}
BaseVector[] x = new BaseVector[l];
int i;
for (i = 0; i < l; i++)
{
x[i] = prob.x[perm[i]];
}
// calculate weighted C
double[] weightedC = new double[nrClass];
for (i = 0; i < nrClass; i++)
{
weightedC[i] = param.c;
}
for (i = 0; i < param.nrWeight; i++)
{
int j;
for (j = 0; j < nrClass; j++)
{
if (param.weightLabel[i] == label[j])
{
break;
}
}
if (j == nrClass)
{
Info("WARNING: class label " + param.weightLabel[i] + " specified in weight is not found\n");
}
else
{
weightedC[j] *= param.weight[i];
}
}
// train k*(k-1)/2 models
bool[] nonzero = new bool[l];
for (i = 0; i < l; i++)
{
nonzero[i] = false;
}
DecisionFunction[] f = new DecisionFunction[nrClass * (nrClass - 1) / 2];
double[] probA = null, probB = null;
if (param.probability)
{
probA = new double[nrClass * (nrClass - 1) / 2];
probB = new double[nrClass * (nrClass - 1) / 2];
}
int p = 0;
for (i = 0; i < nrClass; i++)
{
for (int j = i + 1; j < nrClass; j++)
{
int si = start[i], sj = start[j];
int ci = count[i], cj = count[j];
int c = ci + cj;
SvmProblem subProb = new SvmProblem {
x = new BaseVector[c], y = new float[c]
};
int k;
for (k = 0; k < ci; k++)
{
subProb.x[k] = x[si + k];
subProb.y[k] = +1;
}
for (k = 0; k < cj; k++)
{
subProb.x[ci + k] = x[sj + k];
subProb.y[ci + k] = -1;
}
if (param.probability)
{
double[] probAb = new double[2];
SvmBinarySvcProbability(subProb, param, weightedC[i], weightedC[j], probAb);
probA[p] = probAb[0];
probB[p] = probAb[1];
}
f[p] = SvmTrainOne(subProb, param, weightedC[i], weightedC[j]);
for (k = 0; k < ci; k++)
{
if (!nonzero[si + k] && Math.Abs(f[p].alpha[k]) > 0)
{
nonzero[si + k] = true;
}
}
for (k = 0; k < cj; k++)
{
if (!nonzero[sj + k] && Math.Abs(f[p].alpha[ci + k]) > 0)
{
nonzero[sj + k] = true;
}
}
++p;
}
}
// build output
model.nrClass = nrClass;
model.label = new int[nrClass];
for (i = 0; i < nrClass; i++)
{
model.label[i] = label[i];
}
model.rho = new double[nrClass * (nrClass - 1) / 2];
for (i = 0; i < nrClass * (nrClass - 1) / 2; i++)
{
model.rho[i] = f[i].rho;
}
if (param.probability)
{
model.probA = new double[nrClass * (nrClass - 1) / 2];
model.probB = new double[nrClass * (nrClass - 1) / 2];
for (i = 0; i < nrClass * (nrClass - 1) / 2; i++)
{
model.probA[i] = probA[i];
model.probB[i] = probB[i];
}
}
else
{
model.probA = null;
model.probB = null;
}
int nnz = 0;
int[] nzCount = new int[nrClass];
model.nSv = new int[nrClass];
for (i = 0; i < nrClass; i++)
{
int nSv = 0;
for (int j = 0; j < count[i]; j++)
{
if (nonzero[start[i] + j])
{
++nSv;
++nnz;
}
}
model.nSv[i] = nSv;
nzCount[i] = nSv;
}
Info("Total nSV = " + nnz + "\n");
model.l = nnz;
model.sv = new BaseVector[nnz];
p = 0;
for (i = 0; i < l; i++)
{
if (nonzero[i])
{
model.sv[p++] = x[i];
}
}
int[] nzStart = new int[nrClass];
nzStart[0] = 0;
for (i = 1; i < nrClass; i++)
{
nzStart[i] = nzStart[i - 1] + nzCount[i - 1];
}
model.svCoef = new double[nrClass - 1][];
for (i = 0; i < nrClass - 1; i++)
{
model.svCoef[i] = new double[nnz];
}
p = 0;
for (i = 0; i < nrClass; i++)
{
for (int j = i + 1; j < nrClass; j++)
{
// classifier (i,j): coefficients with
// i are in sv_coef[j-1][nz_start[i]...],
// j are in sv_coef[i][nz_start[j]...]
int si = start[i];
int sj = start[j];
int ci = count[i];
int cj = count[j];
int q = nzStart[i];
int k;
for (k = 0; k < ci; k++)
{
if (nonzero[si + k])
{
model.svCoef[j - 1][q++] = f[p].alpha[k];
}
}
q = nzStart[j];
for (k = 0; k < cj; k++)
{
if (nonzero[sj + k])
{
model.svCoef[i][q++] = f[p].alpha[ci + k];
}
}
++p;
}
}
}
return(model);
}
//
// Interface functions
//
public static SvmModel SvmTrain(SvmProblem prob, SvmParameter param)
{
SvmModel model = new SvmModel{param = param};
if (param.svmType == SvmType.OneClass || param.svmType == SvmType.EpsilonSvr || param.svmType == SvmType.NuSvr){
// regression or one-class-svm
model.nrClass = 2;
model.label = null;
model.nSv = null;
model.probA = null;
model.probB = null;
model.svCoef = new double[1][];
if (param.probability && (param.svmType == SvmType.EpsilonSvr || param.svmType == SvmType.NuSvr)){
model.probA = new double[1];
model.probA[0] = SvmSvrProbability(prob, param);
}
DecisionFunction f = SvmTrainOne(prob, param, 0, 0);
model.rho = new double[1];
model.rho[0] = f.rho;
int nSv = 0;
int i;
for (i = 0; i < prob.Count; i++){
if (Math.Abs(f.alpha[i]) > 0){
++nSv;
}
}
model.l = nSv;
model.sv = new BaseVector[nSv];
model.svCoef[0] = new double[nSv];
int j = 0;
for (i = 0; i < prob.Count; i++){
if (Math.Abs(f.alpha[i]) > 0){
model.sv[j] = prob.x[i];
model.svCoef[0][j] = f.alpha[i];
++j;
}
}
} else{
// classification
int l = prob.Count;
int[] tmpNrClass = new int[1];
int[][] tmpLabel = new int[1][];
int[][] tmpStart = new int[1][];
int[][] tmpCount = new int[1][];
int[] perm = new int[l];
// group training data of the same class
SvmGroupClasses(prob, tmpNrClass, tmpLabel, tmpStart, tmpCount, perm);
int nrClass = tmpNrClass[0];
int[] label = tmpLabel[0];
int[] start = tmpStart[0];
int[] count = tmpCount[0];
if (nrClass == 1){
Info("WARNING: training data in only one class. See README for details.\n");
}
BaseVector[] x = new BaseVector[l];
int i;
for (i = 0; i < l; i++){
x[i] = prob.x[perm[i]];
}
// calculate weighted C
double[] weightedC = new double[nrClass];
for (i = 0; i < nrClass; i++){
weightedC[i] = param.c;
}
for (i = 0; i < param.nrWeight; i++){
int j;
for (j = 0; j < nrClass; j++){
if (param.weightLabel[i] == label[j]){
break;
}
}
if (j == nrClass){
Info("WARNING: class label " + param.weightLabel[i] + " specified in weight is not found\n");
} else{
weightedC[j] *= param.weight[i];
}
}
// train k*(k-1)/2 models
bool[] nonzero = new bool[l];
for (i = 0; i < l; i++){
nonzero[i] = false;
}
DecisionFunction[] f = new DecisionFunction[nrClass*(nrClass - 1)/2];
double[] probA = null, probB = null;
if (param.probability){
probA = new double[nrClass*(nrClass - 1)/2];
probB = new double[nrClass*(nrClass - 1)/2];
}
int p = 0;
for (i = 0; i < nrClass; i++){
for (int j = i + 1; j < nrClass; j++){
int si = start[i], sj = start[j];
int ci = count[i], cj = count[j];
int c = ci + cj;
SvmProblem subProb = new SvmProblem{x = new BaseVector[c], y = new float[c]};
int k;
for (k = 0; k < ci; k++){
subProb.x[k] = x[si + k];
subProb.y[k] = +1;
}
for (k = 0; k < cj; k++){
subProb.x[ci + k] = x[sj + k];
subProb.y[ci + k] = -1;
}
if (param.probability){
double[] probAb = new double[2];
SvmBinarySvcProbability(subProb, param, weightedC[i], weightedC[j], probAb);
probA[p] = probAb[0];
probB[p] = probAb[1];
}
f[p] = SvmTrainOne(subProb, param, weightedC[i], weightedC[j]);
for (k = 0; k < ci; k++){
if (!nonzero[si + k] && Math.Abs(f[p].alpha[k]) > 0){
nonzero[si + k] = true;
}
}
for (k = 0; k < cj; k++){
if (!nonzero[sj + k] && Math.Abs(f[p].alpha[ci + k]) > 0){
nonzero[sj + k] = true;
}
}
++p;
}
}
// build output
model.nrClass = nrClass;
model.label = new int[nrClass];
for (i = 0; i < nrClass; i++){
model.label[i] = label[i];
}
model.rho = new double[nrClass*(nrClass - 1)/2];
for (i = 0; i < nrClass*(nrClass - 1)/2; i++){
model.rho[i] = f[i].rho;
}
if (param.probability){
model.probA = new double[nrClass*(nrClass - 1)/2];
model.probB = new double[nrClass*(nrClass - 1)/2];
for (i = 0; i < nrClass*(nrClass - 1)/2; i++){
model.probA[i] = probA[i];
model.probB[i] = probB[i];
}
} else{
model.probA = null;
model.probB = null;
}
int nnz = 0;
int[] nzCount = new int[nrClass];
model.nSv = new int[nrClass];
for (i = 0; i < nrClass; i++){
int nSv = 0;
for (int j = 0; j < count[i]; j++){
if (nonzero[start[i] + j]){
++nSv;
++nnz;
}
}
model.nSv[i] = nSv;
nzCount[i] = nSv;
}
Info("Total nSV = " + nnz + "\n");
model.l = nnz;
model.sv = new BaseVector[nnz];
p = 0;
for (i = 0; i < l; i++){
if (nonzero[i]){
model.sv[p++] = x[i];
}
}
int[] nzStart = new int[nrClass];
nzStart[0] = 0;
for (i = 1; i < nrClass; i++){
nzStart[i] = nzStart[i - 1] + nzCount[i - 1];
}
model.svCoef = new double[nrClass - 1][];
for (i = 0; i < nrClass - 1; i++){
model.svCoef[i] = new double[nnz];
}
p = 0;
for (i = 0; i < nrClass; i++){
for (int j = i + 1; j < nrClass; j++){
// classifier (i,j): coefficients with
// i are in sv_coef[j-1][nz_start[i]...],
// j are in sv_coef[i][nz_start[j]...]
int si = start[i];
int sj = start[j];
int ci = count[i];
int cj = count[j];
int q = nzStart[i];
int k;
for (k = 0; k < ci; k++){
if (nonzero[si + k]){
model.svCoef[j - 1][q++] = f[p].alpha[k];
}
}
q = nzStart[j];
for (k = 0; k < cj; k++){
if (nonzero[sj + k]){
model.svCoef[i][q++] = f[p].alpha[ci + k];
}
}
++p;
}
}
}
return model;
}
private static DecisionFunction SvmTrainOne(SvmProblem prob, SvmParameter param, double cp, double cn)
{
double[] alpha = new double[prob.Count];
SvmSolver.SolutionInfo si = new SvmSolver.SolutionInfo();
switch (param.svmType){
case SvmType.CSvc:
SolveCSvc(prob, param, alpha, si, cp, cn);
break;
case SvmType.NuSvc:
SolveNuSvc(prob, param, alpha, si);
break;
case SvmType.OneClass:
SolveOneClass(prob, param, alpha, si);
break;
case SvmType.EpsilonSvr:
SolveEpsilonSvr(prob, param, alpha, si);
break;
case SvmType.NuSvr:
SolveNuSvr(prob, param, alpha, si);
break;
}
Info("obj = " + si.obj + ", rho = " + si.rho + "\n");
// output SVs
int nSv = 0;
int nBsv = 0;
for (int i = 0; i < prob.Count; i++){
if (Math.Abs(alpha[i]) > 0){
++nSv;
if (prob.y[i] > 0){
if (Math.Abs(alpha[i]) >= si.upperBoundP){
++nBsv;
}
} else{
if (Math.Abs(alpha[i]) >= si.upperBoundN){
++nBsv;
}
}
}
}
Info("nSV = " + nSv + ", nBSV = " + nBsv + "\n");
DecisionFunction f = new DecisionFunction{alpha = alpha, rho = si.rho};
return f;
}
