public void find_parameter_C(Problem prob, Parameter param1, int nr_fold, double start_C, double max_C, ref double best_C, ref double best_rate) { // variables for CV int i; int l = prob.l; int[] perm = new int[l]; double[] target = new double[prob.l]; Problem[] subprob = new Problem[nr_fold]; // variables for warm start double ratio = 2; double[][] prev_w = new double[nr_fold][]; int num_unchanged_w = 0; //parameter param1 = param; if (nr_fold > l) { nr_fold = l; _logger.LogWarning("WARNING: # folds > # data. Will use # folds = # data instead (i.e., leave-one-out cross validation)"); } int[] fold_start = new int[nr_fold + 1]; for (i = 0; i < l; i++) { perm[i] = i; } for (i = 0; i < l; i++) { int j = i + rand.Next(l - i); Helper.swap(perm, i, j); } for (i = 0; i <= nr_fold; i++) { fold_start[i] = i * l / nr_fold; } for (i = 0; i < nr_fold; i++) { int begin = fold_start[i]; int end = fold_start[i + 1]; int j, k; subprob[i].bias = prob.bias; subprob[i].n = prob.n; subprob[i].l = l - (end - begin); int[] newperm = new int[subprob[i].l]; subprob[i].y = new double[subprob[i].l]; k = 0; for (j = 0; j < begin; j++) { newperm[k] = perm[j]; subprob[i].y[k] = prob.y[perm[j]]; ++k; } for (j = end; j < l; j++) { newperm[k] = perm[j]; subprob[i].y[k] = prob.y[perm[j]]; ++k; } subprob[i].x = SparseMatrixFactory.CreateFacadeMatrix(prob.x, newperm); } //best_C = Double.NaN; best_rate = 0; if (start_C <= 0) { start_C = calc_start_C(prob, param1); } //start_C = calc_start_C(prob, param); param1.C = start_C; Console.WriteLine("Param1.C {0}", param1.C); while (param1.C <= max_C) { for (i = 0; i < nr_fold; i++) { int j; int begin = fold_start[i]; int end = fold_start[i + 1]; param1.init_sol = prev_w[i]; Model submodel = train(subprob[i], param1); int total_w_size; if (submodel.nr_class == 2) { total_w_size = subprob[i].n; } else { total_w_size = subprob[i].n * submodel.nr_class; } if (prev_w[i] == null) { prev_w[i] = new double[total_w_size]; for (j = 0; j < total_w_size; j++) { prev_w[i][j] = submodel.w[j]; } } else if (num_unchanged_w >= 0) { double norm_w_diff = 0; for (j = 0; j < total_w_size; j++) { norm_w_diff += (submodel.w[j] - prev_w[i][j]) * (submodel.w[j] - prev_w[i][j]); prev_w[i][j] = submodel.w[j]; } //norm_w_diff = Math.Sqrt(norm_w_diff); if (norm_w_diff > CONSTANTS.SMALL_EPS_SQ) { num_unchanged_w = -1; } } else { for (j = 0; j < total_w_size; j++) { prev_w[i][j] = submodel.w[j]; } } for (j = begin; j < end; j++) { target[perm[j]] = predict(submodel, prob.x.getRowPtr(perm[j])); } } int total_correct = 0; for (i = 0; i < prob.l; i++) { if (target[i] == prob.y[i]) { ++total_correct; } } double current_rate = (double)total_correct / prob.l; if (current_rate > best_rate) { best_C = param1.C; best_rate = current_rate; } _logger.LogInformation("log2c={0}\trate={1}", Math.Log(param1.C) / Math.Log(2.0), 100.0 * current_rate); num_unchanged_w++; if (num_unchanged_w == 3) { break; } param1.C = param1.C * ratio; } if (param1.C > max_C && max_C > start_C) { _logger.LogInformation("warning: maximum C reached.\n"); } }
public void cross_validation(Problem prob, Parameter param, ref int nr_fold, ref double[] target) { int i; int l = prob.l; int[] perm = new int[l]; if (nr_fold > l) { nr_fold = l; _logger.LogWarning("WARNING: # folds > # data. Will use # folds = # data instead (i.e., leave-one-out cross validation)"); } int[] fold_start = new int[nr_fold + 1]; for (i = 0; i < l; i++) { perm[i] = i; } for (i = 0; i < l; i++) { int j = i + rand.Next(l - i); Helper.swap(perm, i, j); } for (i = 0; i <= nr_fold; i++) { fold_start[i] = i * l / nr_fold; } for (i = 0; i < nr_fold; i++) { int begin = fold_start[i]; int end = fold_start[i + 1]; int j, k; Problem subprob = new Problem(); subprob.bias = prob.bias; subprob.n = prob.n; subprob.l = l - (end - begin); subprob.y = new double[subprob.l]; int[] newperm = new int[subprob.l]; k = 0; for (j = 0; j < begin; j++) { newperm[k] = perm[j]; subprob.y[k] = prob.y[perm[j]]; ++k; } for (j = end; j < l; j++) { newperm[k] = perm[j]; subprob.y[k] = prob.y[perm[j]]; ++k; } subprob.x = SparseMatrixFactory.CreateFacadeMatrix(prob.x, newperm); Model submodel = train(subprob, param); for (j = begin; j < end; j++) { target[perm[j]] = predict(submodel, prob.x.getRowPtr(perm[j])); } } }
// // Interface functions // public Model train(Problem prob, Parameter param) { int i, j; int l = prob.l; int n = prob.n; int w_size = prob.n; Model model_ = new Model(); if (prob.bias >= 0) { model_.nr_feature = n - 1; } else { model_.nr_feature = n; } model_.param = param; model_.bias = prob.bias; if (model_.param.check_regression_model()) { model_.w = new double[w_size]; for (i = 0; i < w_size; i++) { model_.w[i] = 0; } model_.nr_class = 2; model_.label = null; double[] w = model_.w; train_one(prob, param, ref w, 0, 0); model_.w = w; } else { int nr_class = 0; int[] label = null; int[] start = null; int[] count = null; int[] perm = new int[l]; // group training data of the same class group_classes(prob, ref nr_class, ref label, ref start, ref count, ref perm); model_.nr_class = nr_class; model_.label = new int[nr_class]; for (i = 0; i < nr_class; i++) { model_.label[i] = label[i]; } // calculate weighted C double[] weighted_C = new double[nr_class]; for (i = 0; i < nr_class; i++) { weighted_C[i] = param.C; } for (i = 0; i < param.nr_weight; i++) { for (j = 0; j < nr_class; j++) { if (param.weight_label[i] == label[j]) { break; } } if (j == nr_class) { _logger.LogError(string.Format("WARNING:ok class label {0} specified in weight is not found\n", param.weight_label[i])); } else { weighted_C[j] *= param.weight[i]; } } // constructing the subproblem int k; Problem sub_prob = new Problem(); sub_prob.l = l; sub_prob.n = n; sub_prob.y = new double[sub_prob.l]; sub_prob.x = SparseMatrixFactory.CreateFacadeMatrix(prob.x, perm); // multi-class svm by Crammer and Singer if (param.solver_type == SOLVER_TYPE.MCSVM_CS) { model_.w = new double[n * nr_class]; for (i = 0; i < nr_class; i++) { for (j = start[i]; j < start[i] + count[i]; j++) { sub_prob.y[j] = i; } } Solver_MCSVM_CS Solver = new Solver_MCSVM_CS(sub_prob, nr_class, weighted_C, param.eps); Solver.Solve(model_.w); } else { if (nr_class == 2) { model_.w = new double[w_size]; int e0 = start[0] + count[0]; k = 0; for (; k < e0; k++) { sub_prob.y[k] = +1; } for (; k < sub_prob.l; k++) { sub_prob.y[k] = -1; } if (param.init_sol != null) { for (i = 0; i < w_size; i++) { model_.w[i] = param.init_sol[i]; } } else { for (i = 0; i < w_size; i++) { model_.w[i] = 0; } } double [] w = model_.w; train_one(sub_prob, param, ref w, weighted_C[0], weighted_C[1]); model_.w = w; } else { model_.w = new double[w_size * nr_class]; double[] w = new double[w_size]; for (i = 0; i < nr_class; i++) { int si = start[i]; int ei = si + count[i]; k = 0; for (; k < si; k++) { sub_prob.y[k] = -1; } for (; k < ei; k++) { sub_prob.y[k] = +1; } for (; k < sub_prob.l; k++) { sub_prob.y[k] = -1; } if (param.init_sol != null) { for (j = 0; j < w_size; j++) { w[j] = param.init_sol[j * nr_class + i]; } } else { for (j = 0; j < w_size; j++) { w[j] = 0; } } train_one(sub_prob, param, ref w, weighted_C[i], param.C); for (j = 0; j < w_size; j++) { model_.w[j * nr_class + i] = w[j]; } } } } } return(model_); }
// read in a problem (in libsvm format) public static Problem read_problem(StreamReader fp, double bias, ILogger _logger) { _logger.LogInformation("Starting "); char[] delimiters = { ' ', '\t' }; int elements = 0; Problem p = new Problem(); p.bias = bias; int maxCol = 0; try { _logger.LogInformation("Opening File"); string line = fp.ReadLine(); _logger.LogInformation("reading file phase 1"); while (line != null) { string[] tok = line.Split(delimiters); elements += (String.IsNullOrWhiteSpace(tok[tok.Length - 1])) ? tok.Length - 2 : tok.Length - 1; for (int i = 0; i < tok.Length; i++) { if (tok[i].Contains(":")) { string[] t = tok[i].Split(":"); maxCol = Math.Max(maxCol, int.Parse(t[0])); } } line = fp.ReadLine(); p.l++; if (p.bias >= 0) { elements++; } } if (p.bias >= 0) { maxCol++; // Attempt to add back BIAS System.Console.WriteLine("Bias {0}", p.bias); } fp.BaseStream.Position = 0; fp.DiscardBufferedData(); _logger.LogInformation("reading file phase 2 p.l={0}, colcount={1}, element={2}", p.l, maxCol, elements); p.y = new double[p.l]; p.x = SparseMatrixFactory.CreateMatrix(p.l, maxCol, elements); p.n = maxCol; // Sequential Loader mode on teh Matrix is a way to prevent some housekeeping operation as loading rows and columns in order. p.x.SequentialLoad = true; int[] cols = new int[2000]; double[] vals = new double[2000]; for (int i = 0; i < p.l; i++) { line = fp.ReadLine(); string[] tok = line.Split(delimiters); int len = 0; for (int j = 0; j < tok.Length; j++) { string[] t = tok[j].Split(":", 2); if (t.Length > 1) { cols[len] = int.Parse(t[0]) - 1; vals[len] = double.Parse(t[1]); len++; } else if (!String.IsNullOrWhiteSpace(tok[j])) { p.y[i] = double.Parse(tok[j]); } } if (p.bias >= 0) { // Attempt to add back BIAS cols[len] = maxCol - 1; vals[len] = p.bias; len++; } p.x.LoadRow(i, cols, vals, len); } p.x.SequentialLoad = false; fp.Close(); } catch (Exception e) { _logger.LogError(e.Message); throw new Exception("Badly formated input"); } return(p); }