public void setSolverType(SolverType solverType)
{
if (solverType == null)
{
throw new ArgumentException("solver type must not be null");
}
this.solverType = solverType;
}
public Parameter(SolverType solver, double C, double eps) : this(solver, C, eps, 0.1)
{
}
/**
* A coordinate descent algorithm for
* L1-loss and L2-loss SVM dual problems
*<pre>
* min_\alpha 0.5(\alpha^T (Q + D)\alpha) - e^T \alpha,
* s.t. 0 <= \alpha_i <= upper_bound_i,
*
* where Qij = yi yj xi^T xj and
* D is a diagonal matrix
*
* In L1-SVM case:
* upper_bound_i = Cp if y_i = 1
* upper_bound_i = Cn if y_i = -1
* D_ii = 0
* In L2-SVM case:
* upper_bound_i = INF
* D_ii = 1/(2*Cp) if y_i = 1
* D_ii = 1/(2*Cn) if y_i = -1
*
* Given:
* x, y, Cp, Cn
* eps is the stopping tolerance
*
* solution will be put in w
*
* See Algorithm 3 of Hsieh et al., ICML 2008
*</pre>
*/
private static void solve_l2r_l1l2_svc(Problem prob, double[] w, double eps, double Cp, double Cn, SolverType solver_type)
{
int l = prob.l;
int w_size = prob.n;
int i, s, iter = 0;
double C, d, G;
double[] QD = new double[l];
int max_iter = 1000;
int[] index = new int[l];
double[] alpha = new double[l];
sbyte[] y = new sbyte[l];
int active_size = l;
// PG: projected gradient, for shrinking and stopping
double PG;
double PGmax_old = Double.PositiveInfinity;
double PGmin_old = Double.NegativeInfinity;
double PGmax_new, PGmin_new;
// default solver_type: L2R_L2LOSS_SVC_DUAL
double[] diag = new[] { 0.5 / Cn, 0, 0.5 / Cp };
double[] upper_bound = new double[] { Double.PositiveInfinity, 0, Double.PositiveInfinity };
if (solver_type.getId() == SolverType.L2R_L1LOSS_SVC_DUAL)
{
diag[0] = 0;
diag[2] = 0;
upper_bound[0] = Cn;
upper_bound[2] = Cp;
}
for (i = 0; i < l; i++)
{
if (prob.y[i] > 0)
{
y[i] = +1;
}
else
{
y[i] = -1;
}
}
// Initial alpha can be set here. Note that
// 0 <= alpha[i] <= upper_bound[GETI(i)]
for (i = 0; i < l; i++)
alpha[i] = 0;
for (i = 0; i < w_size; i++)
w[i] = 0;
for (i = 0; i < l; i++)
{
QD[i] = diag[GETI(y, i)];
foreach (Feature xi in prob.x[i])
{
double val = xi.Value;
QD[i] += val * val;
w[xi.Index - 1] += y[i] * alpha[i] * val;
}
index[i] = i;
}
while (iter < max_iter)
{
PGmax_new = Double.NegativeInfinity;
PGmin_new = Double.PositiveInfinity;
for (i = 0; i < active_size; i++)
{
int j = i + random.Next(active_size - i);
swap(index, i, j);
}
for (s = 0; s < active_size; s++)
{
i = index[s];
G = 0;
sbyte yi = y[i];
foreach (Feature xi in prob.x[i])
{
G += w[xi.Index - 1] * xi.Value;
}
G = G * yi - 1;
C = upper_bound[GETI(y, i)];
G += alpha[i] * diag[GETI(y, i)];
PG = 0;
if (alpha[i] == 0)
{
if (G > PGmax_old)
{
active_size--;
swap(index, s, active_size);
s--;
continue;
}
else if (G < 0)
{
PG = G;
}
}
else if (alpha[i] == C)
{
if (G < PGmin_old)
{
active_size--;
swap(index, s, active_size);
s--;
continue;
}
else if (G > 0)
{
PG = G;
}
}
else
{
PG = G;
}
PGmax_new = Math.Max(PGmax_new, PG);
PGmin_new = Math.Min(PGmin_new, PG);
if (Math.Abs(PG) > 1.0e-12)
{
double alpha_old = alpha[i];
alpha[i] = Math.Min(Math.Max(alpha[i] - G / QD[i], 0.0), C);
d = (alpha[i] - alpha_old) * yi;
foreach (Feature xi in prob.x[i])
{
w[xi.Index - 1] += d * xi.Value;
}
}
}
iter++;
if (iter % 10 == 0) info(".");
if (PGmax_new - PGmin_new <= eps)
{
if (active_size == l)
break;
else
{
active_size = l;
info("*");
PGmax_old = Double.PositiveInfinity;
PGmin_old = Double.NegativeInfinity;
continue;
}
}
PGmax_old = PGmax_new;
PGmin_old = PGmin_new;
if (PGmax_old <= 0) PGmax_old = Double.PositiveInfinity;
if (PGmin_old >= 0) PGmin_old = Double.NegativeInfinity;
}
info("\noptimization finished, #iter = {0}", iter);
if (iter >= max_iter) info("\nWARNING: reaching max number of iterations\nUsing -s 2 may be faster (also see FAQ)\n");
// calculate objective value
double v = 0;
int nSV = 0;
for (i = 0; i < w_size; i++)
v += w[i] * w[i];
for (i = 0; i < l; i++)
{
v += alpha[i] * (alpha[i] * diag[GETI(y, i)] - 2);
if (alpha[i] > 0) ++nSV;
}
info("Objective value = {0}", v / 2);
info("nSV = {0}", nSV);
}
public Parameter(SolverType solverType, double C, double eps, double p ) {
setSolverType(solverType);
setC(C);
setEps(eps);
setP(p);
}
public Parameter(SolverType solver, double C, double eps ):this(solver, C, eps, 0.1) {
}
public void setSolverType(SolverType solverType) {
if (solverType == null) throw new ArgumentException("solver type must not be null");
this.solverType = solverType;
}
internal void parse_command_line(string[] argv)
{
int i;
// eps: see setting below
Parameter = new Parameter(SolverType.getById(SolverType.L2R_L2LOSS_SVC_DUAL), 1, Double.PositiveInfinity, 0.1);
// default values
Bias = -1;
cross_validation = false;
// parse options
for (i = 0; i < argv.Length; i++)
{
if (argv[i][0] != '-')
{
break;
}
if (++i >= argv.Length)
{
exit_with_help();
}
switch (argv[i - 1][1])
{
case 's':
Parameter.solverType = SolverType.getById(Linear.atoi(argv[i]));
break;
case 'c':
Parameter.setC(Linear.atof(argv[i]));
break;
case 'p':
Parameter.setP(Linear.atof(argv[i]));
break;
case 'e':
Parameter.setEps(Linear.atof(argv[i]));
break;
case 'B':
Bias = Linear.atof(argv[i]);
break;
case 'w':
int weightLabel = int.Parse(argv[i - 1].Substring(2));
double weight = double.Parse(argv[i]);
Parameter.weightLabel = addToArray(Parameter.weightLabel, weightLabel);
Parameter.weight = addToArray(Parameter.weight, weight);
break;
case 'v':
cross_validation = true;
nr_fold = int.Parse(argv[i]);
if (nr_fold < 2)
{
Console.Error.WriteLine("n-fold cross validation: n must >= 2");
exit_with_help();
}
break;
case 'q':
i--;
Linear.disableDebugOutput();
break;
default:
Console.Error.WriteLine("unknown option");
exit_with_help();
break;
}
}
// determine filenames
if (i >= argv.Length)
{
exit_with_help();
}
inputFilename = argv[i];
if (i < argv.Length - 1)
{
modelFilename = argv[i + 1];
}
else
{
int p = argv[i].LastIndexOf('/');
++p; // whew...
modelFilename = argv[i].Substring(p) + ".model";
}
if (Parameter.eps == Double.PositiveInfinity)
{
switch (Parameter.solverType.getId())
{
case SolverType.L2R_LR:
case SolverType.L2R_L2LOSS_SVC:
Parameter.setEps(0.01);
break;
case SolverType.L2R_L2LOSS_SVR:
Parameter.setEps(0.001);
break;
case SolverType.L2R_L2LOSS_SVC_DUAL:
case SolverType.L2R_L1LOSS_SVC_DUAL:
case SolverType.MCSVM_CS:
case SolverType.L2R_LR_DUAL:
Parameter.setEps(0.1);
break;
case SolverType.L1R_L2LOSS_SVC:
case SolverType.L1R_LR:
Parameter.setEps(0.01);
break;
case SolverType.L2R_L1LOSS_SVR_DUAL:
case SolverType.L2R_L2LOSS_SVR_DUAL:
Parameter.setEps(0.1);
break;
default:
throw new InvalidOperationException("unknown solver type: " + Parameter.solverType);
}
}
}