private static DecisionFunction SvmTrainOne(SvmProblem prob, SvmParameter param, double cp, double cn) { double[] alpha = new double[prob.Count]; SvmSolver.SolutionInfo si = new SvmSolver.SolutionInfo(); switch (param.svmType) { case SvmType.CSvc: SolveCSvc(prob, param, alpha, si, cp, cn); break; case SvmType.NuSvc: SolveNuSvc(prob, param, alpha, si); break; case SvmType.OneClass: SolveOneClass(prob, param, alpha, si); break; case SvmType.EpsilonSvr: SolveEpsilonSvr(prob, param, alpha, si); break; case SvmType.NuSvr: SolveNuSvr(prob, param, alpha, si); break; } Info("obj = " + si.obj + ", rho = " + si.rho + "\n"); // output SVs int nSv = 0; int nBsv = 0; for (int i = 0; i < prob.Count; i++) { if (Math.Abs(alpha[i]) > 0) { ++nSv; if (prob.y[i] > 0) { if (Math.Abs(alpha[i]) >= si.upperBoundP) { ++nBsv; } } else { if (Math.Abs(alpha[i]) >= si.upperBoundN) { ++nBsv; } } } } Info("nSV = " + nSv + ", nBSV = " + nBsv + "\n"); DecisionFunction f = new DecisionFunction { alpha = alpha, rho = si.rho }; return(f); }
private static void Main() { // Here we declare that our samples will be 1 dimensional column vectors. The reason for // using a matrix here is that in general you can use N dimensional vectors as inputs to the // krls object. But here we only have 1 dimension to make the example simple. // Now we are making a typedef for the kind of kernel we want to use. I picked the // radial basis kernel because it only has one parameter and generally gives good // results without much fiddling. // Here we declare an instance of the krls object. The first argument to the constructor // is the kernel we wish to use. The second is a parameter that determines the numerical // accuracy with which the object will perform part of the regression algorithm. Generally // smaller values give better results but cause the algorithm to run slower (because it tries // to use more "dictionary vectors" to represent the function it is learning. // You just have to play with it to decide what balance of speed and accuracy is right // for your problem. Here we have set it to 0.001. // // The last argument is the maximum number of dictionary vectors the algorithm is allowed // to use. The default value for this field is 1,000,000 which is large enough that you // won't ever hit it in practice. However, here we have set it to the much smaller value // of 7. This means that once the krls object accumulates 7 dictionary vectors it will // start discarding old ones in favor of new ones as it goes through the training process. // In other words, the algorithm "forgets" about old training data and focuses on recent // training samples. So the bigger the maximum dictionary size the longer its memory will // be. But in this example program we are doing filtering so we only care about the most // recent data. So using a small value is appropriate here since it will result in much // faster filtering and won't introduce much error. using (var rbk = new RadialBasisKernel <double, Matrix <double> >(0.1, 1, 1)) using (var test = new Krls <double, RadialBasisKernel <double, Matrix <double> > >(rbk, 0.001)) { // now we train our object on a few samples of the sinc function. using (var m = Matrix <double> .CreateTemplateParameterizeMatrix(1, 1)) { for (double x = -10; x <= 4; x += 1) { m[0] = x; test.Train(m, Sinc(x)); } // now we output the value of the sinc function for a few test points as well as the // value predicted by krls object. m[0] = 2.5; Console.WriteLine($"{Sinc(m[0])} {test.Operator(m)}"); m[0] = 0.1; Console.WriteLine($"{Sinc(m[0])} {test.Operator(m)}"); m[0] = -4; Console.WriteLine($"{Sinc(m[0])} {test.Operator(m)}"); m[0] = 5.0; Console.WriteLine($"{Sinc(m[0])} {test.Operator(m)}"); // The output is as follows: // 0.239389 0.239362 // 0.998334 0.998333 // -0.189201 -0.189201 // -0.191785 -0.197267 // The first column is the true value of t he sinc function and the second // column is the output from the krls estimate. // Another thing that is worth knowing is that just about everything in dlib is serializable. // So for example, you can save the test object to disk and recall it later like so: Krls <double, RadialBasisKernel <double, Matrix <double> > > .Serialize(test, "saved_krls_object.dat"); // Now let's open that file back up and load the krls object it contains. using (var rbk2 = new RadialBasisKernel <double, Matrix <double> >(0.1, 1, 1)) { var test2 = new Krls <double, RadialBasisKernel <double, Matrix <double> > >(rbk2, 0.001); Krls <double, RadialBasisKernel <double, Matrix <double> > > .Deserialize("saved_krls_object.dat", ref test2); // If you don't want to save the whole krls object (it might be a bit large) // you can save just the decision function it has learned so far. You can get // the decision function out of it by calling test.get_decision_function() and // then you can serialize that object instead. E.g. var funct = test2.GetDecisionFunction(); DecisionFunction <double, RadialBasisKernel <double, Matrix <double> > > .Serialize(funct, "saved_krls_function.dat"); } } } }
// // Interface functions // public static SvmModel SvmTrain(SvmProblem prob, SvmParameter param) { SvmModel model = new SvmModel { param = param }; if (param.svmType == SvmType.OneClass || param.svmType == SvmType.EpsilonSvr || param.svmType == SvmType.NuSvr) { // regression or one-class-svm model.nrClass = 2; model.label = null; model.nSv = null; model.probA = null; model.probB = null; model.svCoef = new double[1][]; if (param.probability && (param.svmType == SvmType.EpsilonSvr || param.svmType == SvmType.NuSvr)) { model.probA = new double[1]; model.probA[0] = SvmSvrProbability(prob, param); } DecisionFunction f = SvmTrainOne(prob, param, 0, 0); model.rho = new double[1]; model.rho[0] = f.rho; int nSv = 0; int i; for (i = 0; i < prob.Count; i++) { if (Math.Abs(f.alpha[i]) > 0) { ++nSv; } } model.l = nSv; model.sv = new BaseVector[nSv]; model.svCoef[0] = new double[nSv]; int j = 0; for (i = 0; i < prob.Count; i++) { if (Math.Abs(f.alpha[i]) > 0) { model.sv[j] = prob.x[i]; model.svCoef[0][j] = f.alpha[i]; ++j; } } } else { // classification int l = prob.Count; int[] tmpNrClass = new int[1]; int[][] tmpLabel = new int[1][]; int[][] tmpStart = new int[1][]; int[][] tmpCount = new int[1][]; int[] perm = new int[l]; // group training data of the same class SvmGroupClasses(prob, tmpNrClass, tmpLabel, tmpStart, tmpCount, perm); int nrClass = tmpNrClass[0]; int[] label = tmpLabel[0]; int[] start = tmpStart[0]; int[] count = tmpCount[0]; if (nrClass == 1) { Info("WARNING: training data in only one class. See README for details.\n"); } BaseVector[] x = new BaseVector[l]; int i; for (i = 0; i < l; i++) { x[i] = prob.x[perm[i]]; } // calculate weighted C double[] weightedC = new double[nrClass]; for (i = 0; i < nrClass; i++) { weightedC[i] = param.c; } for (i = 0; i < param.nrWeight; i++) { int j; for (j = 0; j < nrClass; j++) { if (param.weightLabel[i] == label[j]) { break; } } if (j == nrClass) { Info("WARNING: class label " + param.weightLabel[i] + " specified in weight is not found\n"); } else { weightedC[j] *= param.weight[i]; } } // train k*(k-1)/2 models bool[] nonzero = new bool[l]; for (i = 0; i < l; i++) { nonzero[i] = false; } DecisionFunction[] f = new DecisionFunction[nrClass * (nrClass - 1) / 2]; double[] probA = null, probB = null; if (param.probability) { probA = new double[nrClass * (nrClass - 1) / 2]; probB = new double[nrClass * (nrClass - 1) / 2]; } int p = 0; for (i = 0; i < nrClass; i++) { for (int j = i + 1; j < nrClass; j++) { int si = start[i], sj = start[j]; int ci = count[i], cj = count[j]; int c = ci + cj; SvmProblem subProb = new SvmProblem { x = new BaseVector[c], y = new float[c] }; int k; for (k = 0; k < ci; k++) { subProb.x[k] = x[si + k]; subProb.y[k] = +1; } for (k = 0; k < cj; k++) { subProb.x[ci + k] = x[sj + k]; subProb.y[ci + k] = -1; } if (param.probability) { double[] probAb = new double[2]; SvmBinarySvcProbability(subProb, param, weightedC[i], weightedC[j], probAb); probA[p] = probAb[0]; probB[p] = probAb[1]; } f[p] = SvmTrainOne(subProb, param, weightedC[i], weightedC[j]); for (k = 0; k < ci; k++) { if (!nonzero[si + k] && Math.Abs(f[p].alpha[k]) > 0) { nonzero[si + k] = true; } } for (k = 0; k < cj; k++) { if (!nonzero[sj + k] && Math.Abs(f[p].alpha[ci + k]) > 0) { nonzero[sj + k] = true; } } ++p; } } // build output model.nrClass = nrClass; model.label = new int[nrClass]; for (i = 0; i < nrClass; i++) { model.label[i] = label[i]; } model.rho = new double[nrClass * (nrClass - 1) / 2]; for (i = 0; i < nrClass * (nrClass - 1) / 2; i++) { model.rho[i] = f[i].rho; } if (param.probability) { model.probA = new double[nrClass * (nrClass - 1) / 2]; model.probB = new double[nrClass * (nrClass - 1) / 2]; for (i = 0; i < nrClass * (nrClass - 1) / 2; i++) { model.probA[i] = probA[i]; model.probB[i] = probB[i]; } } else { model.probA = null; model.probB = null; } int nnz = 0; int[] nzCount = new int[nrClass]; model.nSv = new int[nrClass]; for (i = 0; i < nrClass; i++) { int nSv = 0; for (int j = 0; j < count[i]; j++) { if (nonzero[start[i] + j]) { ++nSv; ++nnz; } } model.nSv[i] = nSv; nzCount[i] = nSv; } Info("Total nSV = " + nnz + "\n"); model.l = nnz; model.sv = new BaseVector[nnz]; p = 0; for (i = 0; i < l; i++) { if (nonzero[i]) { model.sv[p++] = x[i]; } } int[] nzStart = new int[nrClass]; nzStart[0] = 0; for (i = 1; i < nrClass; i++) { nzStart[i] = nzStart[i - 1] + nzCount[i - 1]; } model.svCoef = new double[nrClass - 1][]; for (i = 0; i < nrClass - 1; i++) { model.svCoef[i] = new double[nnz]; } p = 0; for (i = 0; i < nrClass; i++) { for (int j = i + 1; j < nrClass; j++) { // classifier (i,j): coefficients with // i are in sv_coef[j-1][nz_start[i]...], // j are in sv_coef[i][nz_start[j]...] int si = start[i]; int sj = start[j]; int ci = count[i]; int cj = count[j]; int q = nzStart[i]; int k; for (k = 0; k < ci; k++) { if (nonzero[si + k]) { model.svCoef[j - 1][q++] = f[p].alpha[k]; } } q = nzStart[j]; for (k = 0; k < cj; k++) { if (nonzero[sj + k]) { model.svCoef[i][q++] = f[p].alpha[ci + k]; } } ++p; } } } return(model); }
// // Interface functions // public static SvmModel SvmTrain(SvmProblem prob, SvmParameter param) { SvmModel model = new SvmModel{param = param}; if (param.svmType == SvmType.OneClass || param.svmType == SvmType.EpsilonSvr || param.svmType == SvmType.NuSvr){ // regression or one-class-svm model.nrClass = 2; model.label = null; model.nSv = null; model.probA = null; model.probB = null; model.svCoef = new double[1][]; if (param.probability && (param.svmType == SvmType.EpsilonSvr || param.svmType == SvmType.NuSvr)){ model.probA = new double[1]; model.probA[0] = SvmSvrProbability(prob, param); } DecisionFunction f = SvmTrainOne(prob, param, 0, 0); model.rho = new double[1]; model.rho[0] = f.rho; int nSv = 0; int i; for (i = 0; i < prob.Count; i++){ if (Math.Abs(f.alpha[i]) > 0){ ++nSv; } } model.l = nSv; model.sv = new BaseVector[nSv]; model.svCoef[0] = new double[nSv]; int j = 0; for (i = 0; i < prob.Count; i++){ if (Math.Abs(f.alpha[i]) > 0){ model.sv[j] = prob.x[i]; model.svCoef[0][j] = f.alpha[i]; ++j; } } } else{ // classification int l = prob.Count; int[] tmpNrClass = new int[1]; int[][] tmpLabel = new int[1][]; int[][] tmpStart = new int[1][]; int[][] tmpCount = new int[1][]; int[] perm = new int[l]; // group training data of the same class SvmGroupClasses(prob, tmpNrClass, tmpLabel, tmpStart, tmpCount, perm); int nrClass = tmpNrClass[0]; int[] label = tmpLabel[0]; int[] start = tmpStart[0]; int[] count = tmpCount[0]; if (nrClass == 1){ Info("WARNING: training data in only one class. See README for details.\n"); } BaseVector[] x = new BaseVector[l]; int i; for (i = 0; i < l; i++){ x[i] = prob.x[perm[i]]; } // calculate weighted C double[] weightedC = new double[nrClass]; for (i = 0; i < nrClass; i++){ weightedC[i] = param.c; } for (i = 0; i < param.nrWeight; i++){ int j; for (j = 0; j < nrClass; j++){ if (param.weightLabel[i] == label[j]){ break; } } if (j == nrClass){ Info("WARNING: class label " + param.weightLabel[i] + " specified in weight is not found\n"); } else{ weightedC[j] *= param.weight[i]; } } // train k*(k-1)/2 models bool[] nonzero = new bool[l]; for (i = 0; i < l; i++){ nonzero[i] = false; } DecisionFunction[] f = new DecisionFunction[nrClass*(nrClass - 1)/2]; double[] probA = null, probB = null; if (param.probability){ probA = new double[nrClass*(nrClass - 1)/2]; probB = new double[nrClass*(nrClass - 1)/2]; } int p = 0; for (i = 0; i < nrClass; i++){ for (int j = i + 1; j < nrClass; j++){ int si = start[i], sj = start[j]; int ci = count[i], cj = count[j]; int c = ci + cj; SvmProblem subProb = new SvmProblem{x = new BaseVector[c], y = new float[c]}; int k; for (k = 0; k < ci; k++){ subProb.x[k] = x[si + k]; subProb.y[k] = +1; } for (k = 0; k < cj; k++){ subProb.x[ci + k] = x[sj + k]; subProb.y[ci + k] = -1; } if (param.probability){ double[] probAb = new double[2]; SvmBinarySvcProbability(subProb, param, weightedC[i], weightedC[j], probAb); probA[p] = probAb[0]; probB[p] = probAb[1]; } f[p] = SvmTrainOne(subProb, param, weightedC[i], weightedC[j]); for (k = 0; k < ci; k++){ if (!nonzero[si + k] && Math.Abs(f[p].alpha[k]) > 0){ nonzero[si + k] = true; } } for (k = 0; k < cj; k++){ if (!nonzero[sj + k] && Math.Abs(f[p].alpha[ci + k]) > 0){ nonzero[sj + k] = true; } } ++p; } } // build output model.nrClass = nrClass; model.label = new int[nrClass]; for (i = 0; i < nrClass; i++){ model.label[i] = label[i]; } model.rho = new double[nrClass*(nrClass - 1)/2]; for (i = 0; i < nrClass*(nrClass - 1)/2; i++){ model.rho[i] = f[i].rho; } if (param.probability){ model.probA = new double[nrClass*(nrClass - 1)/2]; model.probB = new double[nrClass*(nrClass - 1)/2]; for (i = 0; i < nrClass*(nrClass - 1)/2; i++){ model.probA[i] = probA[i]; model.probB[i] = probB[i]; } } else{ model.probA = null; model.probB = null; } int nnz = 0; int[] nzCount = new int[nrClass]; model.nSv = new int[nrClass]; for (i = 0; i < nrClass; i++){ int nSv = 0; for (int j = 0; j < count[i]; j++){ if (nonzero[start[i] + j]){ ++nSv; ++nnz; } } model.nSv[i] = nSv; nzCount[i] = nSv; } Info("Total nSV = " + nnz + "\n"); model.l = nnz; model.sv = new BaseVector[nnz]; p = 0; for (i = 0; i < l; i++){ if (nonzero[i]){ model.sv[p++] = x[i]; } } int[] nzStart = new int[nrClass]; nzStart[0] = 0; for (i = 1; i < nrClass; i++){ nzStart[i] = nzStart[i - 1] + nzCount[i - 1]; } model.svCoef = new double[nrClass - 1][]; for (i = 0; i < nrClass - 1; i++){ model.svCoef[i] = new double[nnz]; } p = 0; for (i = 0; i < nrClass; i++){ for (int j = i + 1; j < nrClass; j++){ // classifier (i,j): coefficients with // i are in sv_coef[j-1][nz_start[i]...], // j are in sv_coef[i][nz_start[j]...] int si = start[i]; int sj = start[j]; int ci = count[i]; int cj = count[j]; int q = nzStart[i]; int k; for (k = 0; k < ci; k++){ if (nonzero[si + k]){ model.svCoef[j - 1][q++] = f[p].alpha[k]; } } q = nzStart[j]; for (k = 0; k < cj; k++){ if (nonzero[sj + k]){ model.svCoef[i][q++] = f[p].alpha[ci + k]; } } ++p; } } } return model; }
private static DecisionFunction SvmTrainOne(SvmProblem prob, SvmParameter param, double cp, double cn) { double[] alpha = new double[prob.Count]; SvmSolver.SolutionInfo si = new SvmSolver.SolutionInfo(); switch (param.svmType){ case SvmType.CSvc: SolveCSvc(prob, param, alpha, si, cp, cn); break; case SvmType.NuSvc: SolveNuSvc(prob, param, alpha, si); break; case SvmType.OneClass: SolveOneClass(prob, param, alpha, si); break; case SvmType.EpsilonSvr: SolveEpsilonSvr(prob, param, alpha, si); break; case SvmType.NuSvr: SolveNuSvr(prob, param, alpha, si); break; } Info("obj = " + si.obj + ", rho = " + si.rho + "\n"); // output SVs int nSv = 0; int nBsv = 0; for (int i = 0; i < prob.Count; i++){ if (Math.Abs(alpha[i]) > 0){ ++nSv; if (prob.y[i] > 0){ if (Math.Abs(alpha[i]) >= si.upperBoundP){ ++nBsv; } } else{ if (Math.Abs(alpha[i]) >= si.upperBoundN){ ++nBsv; } } } } Info("nSV = " + nSv + ", nBSV = " + nBsv + "\n"); DecisionFunction f = new DecisionFunction{alpha = alpha, rho = si.rho}; return f; }