public static double get_w_value(Model model_, int idx, int label_idx)
{
int nr_class = model_.nr_class;
SOLVER_TYPE solver_type = model_.param.solver_type;
double[] w = model_.w;
if (idx < 0 || idx > model_.nr_feature)
{
return(0);
}
if (model_.param.check_regression_model())
{
return(w[idx]);
}
else
{
if (label_idx < 0 || label_idx >= nr_class)
{
return(0);
}
if (nr_class == 2 && solver_type != SOLVER_TYPE.MCSVM_CS)
{
if (label_idx == 0)
{
return(w[idx]);
}
else
{
return(-w[idx]);
}
}
else
{
return(w[idx * nr_class + label_idx]);
}
}
}
public void Testget_w_value_L2R_L2Loss(SOLVER_TYPE st)
{
Model m = new Model();
Parameter p = new Parameter();
double [] w = { 1.0, 2.0, 3.0 };
int idx = 2;
int label_idx = 2;
p.solver_type = st;
m.param = p;
m.nr_class = 2;
m.nr_feature = 3;
m.w = w;
// IF
// solver_type==SOLVER_TYPE.L2R_L2LOSS_SVR ||
// solver_type==SOLVER_TYPE.L2R_L1LOSS_SVR_DUAL ||
// solver_type==SOLVER_TYPE.L2R_L2LOSS_SVR_DUAL);
// Shuld return w[idx]
double t = Model.get_w_value(m, idx, label_idx);
Assert.Equal(w[2], t);
}
public double[] solve(Problem prob, double[] w, Parameter param, SOLVER_TYPE solver_type)
{
int l = prob.l;
double C = param.C;
double p = param.p;
int w_size = prob.n;
double eps = param.eps;
int i, s, iter = 0;
int max_iter = 1000;
int active_size = l;
int[] index = new int[l];
double d, G, H;
double Gmax_old = double.PositiveInfinity;
double Gmax_new, Gnorm1_new;
double Gnorm1_init = -1.0; // Gnorm1_init is initialized at the first iteration
double[] beta = new double[l];
double[] QD = new double[l];
double[] y = prob.y;
// L2R_L2LOSS_SVR_DUAL
double[] lambda = { 0.5 / C };
double[] upper_bound = { double.PositiveInfinity };
if (solver_type == SOLVER_TYPE.L2R_L1LOSS_SVR_DUAL)
{
lambda[0] = 0;
upper_bound[0] = C;
}
// Initial beta can be set here. Note that
// -upper_bound <= beta[i] <= upper_bound
// for(i=0; i<l; i++)
// beta[i] = 0;
Array.Clear(beta, 0, l);
// for(i=0; i<w_size; i++)
// w[i] = 0;
Array.Clear(w, 0, w_size);
for (i = 0; i < l; i++)
{
// feature_node[] xi = prob.x[i];
// QD[i] = sparse_operator.nrm2_sq(xi);
// sparse_operator.axpy(beta[i], xi, w);
QD[i] = prob.x.nrm2_sq(i);
prob.x.axpy(beta[i], i, w);
index[i] = i;
}
while (iter < max_iter)
{
Gmax_new = 0;
Gnorm1_new = 0;
for (i = 0; i < active_size; i++)
{
int j = i + rand.Next(active_size - i);
Helper.swap <int>(ref index[i], ref index[j]);
}
for (s = 0; s < active_size; s++)
{
i = index[s];
G = -y[i] + lambda[0] * beta[i]; //GETI(i) (0)
H = QD[i] + lambda[0]; //GETI(i) (0)
// feature_node[] xi = prob.x[i];
// G += sparse_operator.dot(w, xi);
G += prob.x.dot(i, w);
double Gp = G + p;
double Gn = G - p;
double violation = 0;
if (beta[i] == 0)
{
if (Gp < 0)
{
violation = -Gp;
}
else if (Gn > 0)
{
violation = Gn;
}
else if (Gp > Gmax_old && Gn < -Gmax_old)
{
active_size--;
Helper.swap <int>(ref index[s], ref index[active_size]);
s--;
continue;
}
}
else if (beta[i] >= upper_bound[0]) //GETI(i) (0)
{
if (Gp > 0)
{
violation = Gp;
}
else if (Gp < -Gmax_old)
{
active_size--;
Helper.swap <int>(ref index[s], ref index[active_size]);
s--;
continue;
}
}
else if (beta[i] <= -upper_bound[0]) //GETI(i) (0)
{
if (Gn < 0)
{
violation = -Gn;
}
else if (Gn > Gmax_old)
{
active_size--;
Helper.swap <int>(ref index[s], ref index[active_size]);
s--;
continue;
}
}
else if (beta[i] > 0)
{
violation = Math.Abs(Gp);
}
else
{
violation = Math.Abs(Gn);
}
Gmax_new = Math.Max(Gmax_new, violation);
Gnorm1_new += violation;
// obtain Newton direction d
if (Gp < H * beta[i])
{
d = -Gp / H;
}
else if (Gn > H * beta[i])
{
d = -Gn / H;
}
else
{
d = -beta[i];
}
if (Math.Abs(d) < CONSTANTS.SMALL_ERR)
{
continue;
}
double beta_old = beta[i];
beta[i] = Math.Min(Math.Max(beta[i] + d, -upper_bound[0]), upper_bound[0]); //GETI(i) (0)
d = beta[i] - beta_old;
if (d != 0)
{
prob.x.axpy(d, i, w);
}
}
if (iter == 0)
{
Gnorm1_init = Gnorm1_new;
}
iter++;
if (iter % 10 == 0)
{
_logger.LogInformation(".");
}
if (Gnorm1_new <= eps * Gnorm1_init)
{
if (active_size == l)
{
break;
}
else
{
active_size = l;
_logger.LogInformation("*");
Gmax_old = double.PositiveInfinity;
continue;
}
}
Gmax_old = Gmax_new;
}
_logger.LogInformation("\noptimization finished, #iter = {0}\n", iter);
if (iter >= max_iter)
{
_logger.LogInformation("\nWARNING: reaching max number of iterations\nUsing -s 11 may be faster\n\n");
}
// calculate objective value
double v = 0;
int nSV = 0;
for (i = 0; i < w_size; i++)
{
v += w[i] * w[i];
}
v = 0.5 * v;
for (i = 0; i < l; i++)
{
v += p * Math.Abs(beta[i]) - y[i] * beta[i] + 0.5 * lambda[0] * beta[i] * beta[i]; //GETI(i) (0)
if (beta[i] != 0)
{
nSV++;
}
}
_logger.LogInformation("Objective value = {0}\n", v);
_logger.LogInformation("nSV = {0}\n", nSV);
return(w);
}
// A coordinate descent algorithm for
// L1-loss and L2-loss SVM dual problems
//
// min_\alpha 0.5(\alpha^T (Q + D)\alpha) - e^T \alpha,
// s.t. 0 <= \alpha_i <= upper_bound_i,
//
// where Qij = yi yj xi^T xj and
// D is a diagonal matrix
//
// In L1-SVM case:
// upper_bound_i = Cp if y_i = 1
// upper_bound_i = Cn if y_i = -1
// D_ii = 0
// In L2-SVM case:
// upper_bound_i = INF
// D_ii = 1/(2*Cp) if y_i = 1
// D_ii = 1/(2*Cn) if y_i = -1
//
// Given:
// x, y, Cp, Cn
// eps is the stopping tolerance
//
// solution will be put in w
//
// See Algorithm 3 of Hsieh et al., ICML 2008
//
// #undef GETI
// #define GETI(i) (y[i]+1)
// To support weights for instances, use GETI(i) (i)
// Adjusted for BIAS not in matrix
public double[] solve(Problem prob, double[] w, double eps, double Cp, double Cn, SOLVER_TYPE solver_type)
{
int l = prob.l;
int w_size = prob.n;
int i, s, iter = 0;
double C, d, G;
double[] QD = new double[l];
int max_iter = 1000;
int[] index = new int[l];
double[] alpha = new double[l];
sbyte[] y = new sbyte[l];
int active_size = l;
Console.WriteLine("prob n = " + prob.n);
// PG: projected gradient, for shrinking and stopping
double PG;
double PGmax_old = double.PositiveInfinity;
double PGmin_old = double.NegativeInfinity;
double PGmax_new;
double PGmin_new;
// default solver_type: L2R_L2LOSS_SVC_DUAL
double[] diag = { 0.5 / Cn, 0, 0.5 / Cp };
double[] upper_bound = { double.PositiveInfinity, 0, double.PositiveInfinity };
if (solver_type == SOLVER_TYPE.L2R_L1LOSS_SVC_DUAL)
{
diag[0] = 0;
diag[2] = 0;
upper_bound[0] = Cn;
upper_bound[2] = Cp;
}
for (i = 0; i < l; i++)
{
y[i] = (sbyte)((prob.y[i] > 0) ? 1 : -1);
}
// Initial alpha can be set here. Note that
// 0 <= alpha[i] <= upper_bound[GETI(i)]
// for(i=0; i<l; i++)
// alpha[i] = 0;
Array.Clear(alpha, 0, l);
// for(i=0; i<w_size; i++)
// w[i] = 0;
Array.Clear(w, 0, w_size);
for (i = 0; i < l; i++)
{
QD[i] = diag[(y[i] + 1)]; //GET(i)
//feature_node[] xi = prob.x[i];
//QD[i] += sparse_operator.nrm2_sq(xi);
//sparse_operator.axpy(y[i]*alpha[i], xi, w);
QD[i] += prob.x.nrm2_sq(i);
prob.x.axpy(y[i] * alpha[i], i, w);
index[i] = i;
}
while (iter < max_iter)
{
PGmax_new = double.NegativeInfinity;
PGmin_new = double.PositiveInfinity;
for (i = 0; i < active_size; i++)
{
int j = i + rand.Next(active_size - i);
Helper.swap <int>(ref index[i], ref index[j]);
}
for (s = 0; s < active_size; s++)
{
i = index[s];
sbyte yi = y[i];
//feature_node[] xi = prob.x[i];
G = (yi * prob.x.dot(i, w)) - 1;
// G = yi*sparse_operator.dot(w, xi)-1;
C = upper_bound[(yi + 1)]; // GETI
G += alpha[i] * diag[(yi + 1)]; // GETI
PG = 0;
if (Math.Abs(alpha[i]) < Double.Epsilon)
//if (alpha[i] == 0)
{
if (G > PGmax_old)
{
active_size--;
Helper.swap <int>(ref index[s], ref index[active_size]);
s--;
continue;
}
else if (G < 0)
{
PG = G;
}
}
else if (Math.Abs(alpha[i] - C) < Double.Epsilon) //if (alpha[i] == C)
{
if (G < PGmin_old)
{
active_size--;
Helper.swap <int>(ref index[s], ref index[active_size]);
s--;
continue;
}
else if (G > 0)
{
PG = G;
}
}
else
{
PG = G;
}
PGmax_new = Math.Max(PGmax_new, PG);
PGmin_new = Math.Min(PGmin_new, PG);
if (Math.Abs(PG) > CONSTANTS.SMALL_ERR)
{
double alpha_old = alpha[i];
alpha[i] = Math.Min(Math.Max(alpha[i] - G / QD[i], 0.0), C);
d = (alpha[i] - alpha_old) * yi;
prob.x.axpy(d, i, w);
}
}
iter++;
if (iter % 10 == 0)
{
_logger.LogInformation(".");
}
if (PGmax_new - PGmin_new <= eps)
{
if (active_size == l)
{
break;
}
else
{
active_size = l;
_logger.LogInformation("*");
PGmax_old = double.PositiveInfinity;
PGmin_old = double.NegativeInfinity;
continue;
}
}
PGmax_old = PGmax_new;
PGmin_old = PGmin_new;
if (PGmax_old <= 0)
{
PGmax_old = double.PositiveInfinity;
}
if (PGmin_old >= 0)
{
PGmin_old = double.NegativeInfinity;
}
}
_logger.LogInformation("\noptimization finished, #iter = {0}\n", iter);
if (iter >= max_iter)
{
_logger.LogInformation("\nWARNING: reaching max number of iterations\nUsing -s 2 may be faster (also see FAQ)\n\n");
}
// calculate objective value
double v = 0;
int nSV = 0;
for (i = 0; i < w_size; i++)
{
v += w[i] * w[i];
}
for (i = 0; i < l; i++)
{
v += alpha[i] * (alpha[i] * diag[(y[i] + 1)] - 2); //GETI
if (alpha[i] > 0)
{
++nSV;
}
}
_logger.LogInformation("Objective value = {0}\n", v / 2);
_logger.LogInformation("nSV = {0}\n", nSV);
return(w);
}
