public static double get_w_value(Model model_, int idx, int label_idx) { int nr_class = model_.nr_class; SOLVER_TYPE solver_type = model_.param.solver_type; double[] w = model_.w; if (idx < 0 || idx > model_.nr_feature) { return(0); } if (model_.param.check_regression_model()) { return(w[idx]); } else { if (label_idx < 0 || label_idx >= nr_class) { return(0); } if (nr_class == 2 && solver_type != SOLVER_TYPE.MCSVM_CS) { if (label_idx == 0) { return(w[idx]); } else { return(-w[idx]); } } else { return(w[idx * nr_class + label_idx]); } } }
public void Testget_w_value_L2R_L2Loss(SOLVER_TYPE st) { Model m = new Model(); Parameter p = new Parameter(); double [] w = { 1.0, 2.0, 3.0 }; int idx = 2; int label_idx = 2; p.solver_type = st; m.param = p; m.nr_class = 2; m.nr_feature = 3; m.w = w; // IF // solver_type==SOLVER_TYPE.L2R_L2LOSS_SVR || // solver_type==SOLVER_TYPE.L2R_L1LOSS_SVR_DUAL || // solver_type==SOLVER_TYPE.L2R_L2LOSS_SVR_DUAL); // Shuld return w[idx] double t = Model.get_w_value(m, idx, label_idx); Assert.Equal(w[2], t); }
public double[] solve(Problem prob, double[] w, Parameter param, SOLVER_TYPE solver_type) { int l = prob.l; double C = param.C; double p = param.p; int w_size = prob.n; double eps = param.eps; int i, s, iter = 0; int max_iter = 1000; int active_size = l; int[] index = new int[l]; double d, G, H; double Gmax_old = double.PositiveInfinity; double Gmax_new, Gnorm1_new; double Gnorm1_init = -1.0; // Gnorm1_init is initialized at the first iteration double[] beta = new double[l]; double[] QD = new double[l]; double[] y = prob.y; // L2R_L2LOSS_SVR_DUAL double[] lambda = { 0.5 / C }; double[] upper_bound = { double.PositiveInfinity }; if (solver_type == SOLVER_TYPE.L2R_L1LOSS_SVR_DUAL) { lambda[0] = 0; upper_bound[0] = C; } // Initial beta can be set here. Note that // -upper_bound <= beta[i] <= upper_bound // for(i=0; i<l; i++) // beta[i] = 0; Array.Clear(beta, 0, l); // for(i=0; i<w_size; i++) // w[i] = 0; Array.Clear(w, 0, w_size); for (i = 0; i < l; i++) { // feature_node[] xi = prob.x[i]; // QD[i] = sparse_operator.nrm2_sq(xi); // sparse_operator.axpy(beta[i], xi, w); QD[i] = prob.x.nrm2_sq(i); prob.x.axpy(beta[i], i, w); index[i] = i; } while (iter < max_iter) { Gmax_new = 0; Gnorm1_new = 0; for (i = 0; i < active_size; i++) { int j = i + rand.Next(active_size - i); Helper.swap <int>(ref index[i], ref index[j]); } for (s = 0; s < active_size; s++) { i = index[s]; G = -y[i] + lambda[0] * beta[i]; //GETI(i) (0) H = QD[i] + lambda[0]; //GETI(i) (0) // feature_node[] xi = prob.x[i]; // G += sparse_operator.dot(w, xi); G += prob.x.dot(i, w); double Gp = G + p; double Gn = G - p; double violation = 0; if (beta[i] == 0) { if (Gp < 0) { violation = -Gp; } else if (Gn > 0) { violation = Gn; } else if (Gp > Gmax_old && Gn < -Gmax_old) { active_size--; Helper.swap <int>(ref index[s], ref index[active_size]); s--; continue; } } else if (beta[i] >= upper_bound[0]) //GETI(i) (0) { if (Gp > 0) { violation = Gp; } else if (Gp < -Gmax_old) { active_size--; Helper.swap <int>(ref index[s], ref index[active_size]); s--; continue; } } else if (beta[i] <= -upper_bound[0]) //GETI(i) (0) { if (Gn < 0) { violation = -Gn; } else if (Gn > Gmax_old) { active_size--; Helper.swap <int>(ref index[s], ref index[active_size]); s--; continue; } } else if (beta[i] > 0) { violation = Math.Abs(Gp); } else { violation = Math.Abs(Gn); } Gmax_new = Math.Max(Gmax_new, violation); Gnorm1_new += violation; // obtain Newton direction d if (Gp < H * beta[i]) { d = -Gp / H; } else if (Gn > H * beta[i]) { d = -Gn / H; } else { d = -beta[i]; } if (Math.Abs(d) < CONSTANTS.SMALL_ERR) { continue; } double beta_old = beta[i]; beta[i] = Math.Min(Math.Max(beta[i] + d, -upper_bound[0]), upper_bound[0]); //GETI(i) (0) d = beta[i] - beta_old; if (d != 0) { prob.x.axpy(d, i, w); } } if (iter == 0) { Gnorm1_init = Gnorm1_new; } iter++; if (iter % 10 == 0) { _logger.LogInformation("."); } if (Gnorm1_new <= eps * Gnorm1_init) { if (active_size == l) { break; } else { active_size = l; _logger.LogInformation("*"); Gmax_old = double.PositiveInfinity; continue; } } Gmax_old = Gmax_new; } _logger.LogInformation("\noptimization finished, #iter = {0}\n", iter); if (iter >= max_iter) { _logger.LogInformation("\nWARNING: reaching max number of iterations\nUsing -s 11 may be faster\n\n"); } // calculate objective value double v = 0; int nSV = 0; for (i = 0; i < w_size; i++) { v += w[i] * w[i]; } v = 0.5 * v; for (i = 0; i < l; i++) { v += p * Math.Abs(beta[i]) - y[i] * beta[i] + 0.5 * lambda[0] * beta[i] * beta[i]; //GETI(i) (0) if (beta[i] != 0) { nSV++; } } _logger.LogInformation("Objective value = {0}\n", v); _logger.LogInformation("nSV = {0}\n", nSV); return(w); }
// A coordinate descent algorithm for // L1-loss and L2-loss SVM dual problems // // min_\alpha 0.5(\alpha^T (Q + D)\alpha) - e^T \alpha, // s.t. 0 <= \alpha_i <= upper_bound_i, // // where Qij = yi yj xi^T xj and // D is a diagonal matrix // // In L1-SVM case: // upper_bound_i = Cp if y_i = 1 // upper_bound_i = Cn if y_i = -1 // D_ii = 0 // In L2-SVM case: // upper_bound_i = INF // D_ii = 1/(2*Cp) if y_i = 1 // D_ii = 1/(2*Cn) if y_i = -1 // // Given: // x, y, Cp, Cn // eps is the stopping tolerance // // solution will be put in w // // See Algorithm 3 of Hsieh et al., ICML 2008 // // #undef GETI // #define GETI(i) (y[i]+1) // To support weights for instances, use GETI(i) (i) // Adjusted for BIAS not in matrix public double[] solve(Problem prob, double[] w, double eps, double Cp, double Cn, SOLVER_TYPE solver_type) { int l = prob.l; int w_size = prob.n; int i, s, iter = 0; double C, d, G; double[] QD = new double[l]; int max_iter = 1000; int[] index = new int[l]; double[] alpha = new double[l]; sbyte[] y = new sbyte[l]; int active_size = l; Console.WriteLine("prob n = " + prob.n); // PG: projected gradient, for shrinking and stopping double PG; double PGmax_old = double.PositiveInfinity; double PGmin_old = double.NegativeInfinity; double PGmax_new; double PGmin_new; // default solver_type: L2R_L2LOSS_SVC_DUAL double[] diag = { 0.5 / Cn, 0, 0.5 / Cp }; double[] upper_bound = { double.PositiveInfinity, 0, double.PositiveInfinity }; if (solver_type == SOLVER_TYPE.L2R_L1LOSS_SVC_DUAL) { diag[0] = 0; diag[2] = 0; upper_bound[0] = Cn; upper_bound[2] = Cp; } for (i = 0; i < l; i++) { y[i] = (sbyte)((prob.y[i] > 0) ? 1 : -1); } // Initial alpha can be set here. Note that // 0 <= alpha[i] <= upper_bound[GETI(i)] // for(i=0; i<l; i++) // alpha[i] = 0; Array.Clear(alpha, 0, l); // for(i=0; i<w_size; i++) // w[i] = 0; Array.Clear(w, 0, w_size); for (i = 0; i < l; i++) { QD[i] = diag[(y[i] + 1)]; //GET(i) //feature_node[] xi = prob.x[i]; //QD[i] += sparse_operator.nrm2_sq(xi); //sparse_operator.axpy(y[i]*alpha[i], xi, w); QD[i] += prob.x.nrm2_sq(i); prob.x.axpy(y[i] * alpha[i], i, w); index[i] = i; } while (iter < max_iter) { PGmax_new = double.NegativeInfinity; PGmin_new = double.PositiveInfinity; for (i = 0; i < active_size; i++) { int j = i + rand.Next(active_size - i); Helper.swap <int>(ref index[i], ref index[j]); } for (s = 0; s < active_size; s++) { i = index[s]; sbyte yi = y[i]; //feature_node[] xi = prob.x[i]; G = (yi * prob.x.dot(i, w)) - 1; // G = yi*sparse_operator.dot(w, xi)-1; C = upper_bound[(yi + 1)]; // GETI G += alpha[i] * diag[(yi + 1)]; // GETI PG = 0; if (Math.Abs(alpha[i]) < Double.Epsilon) //if (alpha[i] == 0) { if (G > PGmax_old) { active_size--; Helper.swap <int>(ref index[s], ref index[active_size]); s--; continue; } else if (G < 0) { PG = G; } } else if (Math.Abs(alpha[i] - C) < Double.Epsilon) //if (alpha[i] == C) { if (G < PGmin_old) { active_size--; Helper.swap <int>(ref index[s], ref index[active_size]); s--; continue; } else if (G > 0) { PG = G; } } else { PG = G; } PGmax_new = Math.Max(PGmax_new, PG); PGmin_new = Math.Min(PGmin_new, PG); if (Math.Abs(PG) > CONSTANTS.SMALL_ERR) { double alpha_old = alpha[i]; alpha[i] = Math.Min(Math.Max(alpha[i] - G / QD[i], 0.0), C); d = (alpha[i] - alpha_old) * yi; prob.x.axpy(d, i, w); } } iter++; if (iter % 10 == 0) { _logger.LogInformation("."); } if (PGmax_new - PGmin_new <= eps) { if (active_size == l) { break; } else { active_size = l; _logger.LogInformation("*"); PGmax_old = double.PositiveInfinity; PGmin_old = double.NegativeInfinity; continue; } } PGmax_old = PGmax_new; PGmin_old = PGmin_new; if (PGmax_old <= 0) { PGmax_old = double.PositiveInfinity; } if (PGmin_old >= 0) { PGmin_old = double.NegativeInfinity; } } _logger.LogInformation("\noptimization finished, #iter = {0}\n", iter); if (iter >= max_iter) { _logger.LogInformation("\nWARNING: reaching max number of iterations\nUsing -s 2 may be faster (also see FAQ)\n\n"); } // calculate objective value double v = 0; int nSV = 0; for (i = 0; i < w_size; i++) { v += w[i] * w[i]; } for (i = 0; i < l; i++) { v += alpha[i] * (alpha[i] * diag[(y[i] + 1)] - 2); //GETI if (alpha[i] > 0) { ++nSV; } } _logger.LogInformation("Objective value = {0}\n", v / 2); _logger.LogInformation("nSV = {0}\n", nSV); return(w); }