public static double svm_predict(Model model, Node[] x) { if (model.Parameter.SvmType == SvmType.ONE_CLASS || model.Parameter.SvmType == SvmType.EPSILON_SVR || model.Parameter.SvmType == SvmType.NU_SVR) { double[] res = new double[1]; svm_predict_values(model, x, res); if (model.Parameter.SvmType == SvmType.ONE_CLASS) return (res[0] > 0) ? 1 : -1; else return res[0]; } else { int i; int nr_class = model.NumberOfClasses; double[] dec_values = new double[nr_class * (nr_class - 1) / 2]; svm_predict_values(model, x, dec_values); int[] vote = new int[nr_class]; for (i = 0; i < nr_class; i++) vote[i] = 0; int pos = 0; for (i = 0; i < nr_class; i++) for (int j = i + 1; j < nr_class; j++) { if (dec_values[pos++] > 0) ++vote[i]; else ++vote[j]; } int vote_Max_idx = 0; for (i = 1; i < nr_class; i++) if (vote[i] > vote[vote_Max_idx]) vote_Max_idx = i; return model.ClassLabels[vote_Max_idx]; } }
public static double svm_predict_probability(Model model, Node[] x, double[] prob_estimates) { if ((model.Parameter.SvmType == SvmType.C_SVC || model.Parameter.SvmType == SvmType.NU_SVC) && model.PairwiseProbabilityA != null && model.PairwiseProbabilityB != null) { int i; int nr_class = model.NumberOfClasses; double[] dec_values = new double[nr_class * (nr_class - 1) / 2]; svm_predict_values(model, x, dec_values); double Min_prob = 1e-7; double[,] pairwise_prob = new double[nr_class, nr_class]; int k = 0; for (i = 0; i < nr_class; i++) { for (int j = i + 1; j < nr_class; j++) { pairwise_prob[i, j] = Math.Min(Math.Max(sigmoid_predict(dec_values[k], model.PairwiseProbabilityA[k], model.PairwiseProbabilityB[k]), Min_prob), 1 - Min_prob); pairwise_prob[j, i] = 1 - pairwise_prob[i, j]; k++; } } multiclass_probability(nr_class, pairwise_prob, prob_estimates); int prob_Max_idx = 0; for (i = 1; i < nr_class; i++) if (prob_estimates[i] > prob_estimates[prob_Max_idx]) prob_Max_idx = i; return model.ClassLabels[prob_Max_idx]; } else return svm_predict(model, x); }
public static SvmType svm_get_svm_type(Model model) { return model.Parameter.SvmType; }
public static double svm_get_svr_probability(Model model) { if ((model.Parameter.SvmType == SvmType.EPSILON_SVR || model.Parameter.SvmType == SvmType.NU_SVR) && model.PairwiseProbabilityA != null) return model.PairwiseProbabilityA[0]; else { Console.Error.WriteLine("Model doesn't contain information for SVR probability inference"); return 0; } }
public static void svm_get_labels(Model model, int[] label) { if (model.ClassLabels != null) for (int i = 0; i < model.NumberOfClasses; i++) label[i] = model.ClassLabels[i]; }
public static int svm_get_nr_class(Model model) { return model.NumberOfClasses; }
/// <summary> /// Predicts the class memberships of all the vectors in the problem. /// </summary> /// <param name="problem">The SVM Problem to solve</param> /// <param name="outputFile">File for result output</param> /// <param name="model">The Model to use</param> /// <param name="predict_probability">Whether to output a distribution over the classes</param> /// <returns>Percentage correctly labelled</returns> public static double Predict( Problem problem, string outputFile, Model model, bool predict_probability) { int correct = 0; int total = 0; double error = 0; double sumv = 0, sumy = 0, sumvv = 0, sumyy = 0, sumvy = 0; StreamWriter output = outputFile != null ? new StreamWriter(outputFile) : null; SvmType svm_type = Procedures.svm_get_svm_type(model); int nr_class = Procedures.svm_get_nr_class(model); int[] labels = new int[nr_class]; double[] prob_estimates = null; if (predict_probability) { if (svm_type == SvmType.EPSILON_SVR || svm_type == SvmType.NU_SVR) { Console.WriteLine("Prob. model for test data: target value = predicted value + z,\nz: Laplace distribution e^(-|z|/sigma)/(2sigma),sigma=" + Procedures.svm_get_svr_probability(model)); } else { Procedures.svm_get_labels(model, labels); prob_estimates = new double[nr_class]; if (output != null) { output.Write("labels"); for (int j = 0; j < nr_class; j++) { output.Write(" " + labels[j]); } output.Write("\n"); } } } for (int i = 0; i < problem.Count; i++) { double target = problem.Y[i]; Node[] x = problem.X[i]; double v; if (predict_probability && (svm_type == SvmType.C_SVC || svm_type == SvmType.NU_SVC)) { v = Procedures.svm_predict_probability(model, x, prob_estimates); if (output != null) { output.Write(v + " "); for (int j = 0; j < nr_class; j++) { output.Write(prob_estimates[j] + " "); } output.Write("\n"); } } else { v = Procedures.svm_predict(model, x); if(output != null) output.Write(v + "\n"); } if (v == target) ++correct; error += (v - target) * (v - target); sumv += v; sumy += target; sumvv += v * v; sumyy += target * target; sumvy += v * target; ++total; } if(output != null) output.Close(); return (double)correct / total; }
public static double PredictRaw(Model model, Node[] x) { var d = new double[1]; Procedures.svm_predict_values(model, x, d); return d[0]; }
/// <summary> /// Predict the class for a single input vector. /// </summary> /// <param name="model">The Model to use for prediction</param> /// <param name="x">The vector for which to predict class</param> /// <returns>The result</returns> public static double Predict(Model model, Node[] x) { return Procedures.svm_predict(model, x); }
/// <summary> /// Predicts a class distribution for the single input vector. /// </summary> /// <param name="model">Model to use for prediction</param> /// <param name="x">The vector for which to predict the class distribution</param> /// <returns>A probability distribtion over classes</returns> public static double[] PredictProbability(Model model, Node[] x) { SvmType svm_type = Procedures.svm_get_svm_type(model); if (svm_type != SvmType.C_SVC && svm_type != SvmType.NU_SVC) throw new Exception("Model type " + svm_type + " unable to predict probabilities."); int nr_class = Procedures.svm_get_nr_class(model); double[] probEstimates = new double[nr_class]; Procedures.svm_predict_probability(model, x, probEstimates); return probEstimates; }
/// <summary> /// Writes a model to the provided stream. /// </summary> /// <param name="stream">The output stream</param> /// <param name="model">The model to write</param> public static void Write(Stream stream, Model model) { TemporaryCulture.Start(); StreamWriter output = new StreamWriter(stream); Parameter param = model.Parameter; output.Write("svm_type " + param.SvmType + "\n"); output.Write("kernel_type " + param.KernelType + "\n"); if (param.KernelType == KernelType.POLY) output.Write("degree " + param.Degree + "\n"); if (param.KernelType == KernelType.POLY || param.KernelType == KernelType.RBF || param.KernelType == KernelType.SIGMOID) output.Write("gamma " + param.Gamma + "\n"); if (param.KernelType == KernelType.POLY || param.KernelType == KernelType.SIGMOID) output.Write("coef0 " + param.Coefficient0 + "\n"); int nr_class = model.NumberOfClasses; int l = model.SupportVectorCount; output.Write("nr_class " + nr_class + "\n"); output.Write("total_sv " + l + "\n"); { output.Write("rho"); for (int i = 0; i < nr_class * (nr_class - 1) / 2; i++) output.Write(" " + model.Rho[i]); output.Write("\n"); } if (model.ClassLabels != null) { output.Write("label"); for (int i = 0; i < nr_class; i++) output.Write(" " + model.ClassLabels[i]); output.Write("\n"); } if (model.PairwiseProbabilityA != null) // regression has probA only { output.Write("probA"); for (int i = 0; i < nr_class * (nr_class - 1) / 2; i++) output.Write(" " + model.PairwiseProbabilityA[i]); output.Write("\n"); } if (model.PairwiseProbabilityB != null) { output.Write("probB"); for (int i = 0; i < nr_class * (nr_class - 1) / 2; i++) output.Write(" " + model.PairwiseProbabilityB[i]); output.Write("\n"); } if (model.NumberOfSVPerClass != null) { output.Write("nr_sv"); for (int i = 0; i < nr_class; i++) output.Write(" " + model.NumberOfSVPerClass[i]); output.Write("\n"); } output.Write("SV\n"); double[][] sv_coef = model.SupportVectorCoefficients; Node[][] SV = model.SupportVectors; for (int i = 0; i < l; i++) { for (int j = 0; j < nr_class - 1; j++) output.Write(sv_coef[j][i] + " "); Node[] p = SV[i]; if (p.Length == 0) { output.WriteLine(); continue; } if (param.KernelType == KernelType.PRECOMPUTED) output.Write("0:{0}", (int)p[0].Value); else { output.Write("{0}:{1}", p[0].Index, p[0].Value); for (int j = 1; j < p.Length; j++) output.Write(" {0}:{1}", p[j].Index, p[j].Value); } output.WriteLine(); } output.Flush(); TemporaryCulture.Stop(); }
/// <summary> /// Writes a model to the provided filename. This will overwrite any previous data in the file. /// </summary> /// <param name="filename">The desired file</param> /// <param name="model">The Model to write</param> public static void Write(string filename, Model model) { FileStream stream = File.Open(filename, FileMode.Create); try { Write(stream, model); } finally { stream.Close(); } }
/// <summary> /// Reads a Model from the provided stream. /// </summary> /// <param name="stream">The stream from which to read the Model.</param> /// <returns>the Model</returns> public static Model Read(Stream stream) { TemporaryCulture.Start(); StreamReader input = new StreamReader(stream); // read parameters Model model = new Model(); Parameter param = new Parameter(); model.Parameter = param; model.Rho = null; model.PairwiseProbabilityA = null; model.PairwiseProbabilityB = null; model.ClassLabels = null; model.NumberOfSVPerClass = null; bool headerFinished = false; while (!headerFinished) { string line = input.ReadLine(); string cmd, arg; int splitIndex = line.IndexOf(' '); if (splitIndex >= 0) { cmd = line.Substring(0, splitIndex); arg = line.Substring(splitIndex + 1); } else { cmd = line; arg = ""; } arg = arg.ToLower(); int i, n; switch (cmd) { case "svm_type": param.SvmType = (SvmType)Enum.Parse(typeof(SvmType), arg.ToUpper()); break; case "kernel_type": param.KernelType = (KernelType)Enum.Parse(typeof(KernelType), arg.ToUpper()); break; case "degree": param.Degree = int.Parse(arg); break; case "gamma": param.Gamma = double.Parse(arg); break; case "coef0": param.Coefficient0 = double.Parse(arg); break; case "nr_class": model.NumberOfClasses = int.Parse(arg); break; case "total_sv": model.SupportVectorCount = int.Parse(arg); break; case "rho": n = model.NumberOfClasses * (model.NumberOfClasses - 1) / 2; model.Rho = new double[n]; string[] rhoParts = arg.Split(); for (i = 0; i < n; i++) model.Rho[i] = double.Parse(rhoParts[i]); break; case "label": n = model.NumberOfClasses; model.ClassLabels = new int[n]; string[] labelParts = arg.Split(); for (i = 0; i < n; i++) model.ClassLabels[i] = int.Parse(labelParts[i]); break; case "probA": n = model.NumberOfClasses * (model.NumberOfClasses - 1) / 2; model.PairwiseProbabilityA = new double[n]; string[] probAParts = arg.Split(); for (i = 0; i < n; i++) model.PairwiseProbabilityA[i] = double.Parse(probAParts[i]); break; case "probB": n = model.NumberOfClasses * (model.NumberOfClasses - 1) / 2; model.PairwiseProbabilityB = new double[n]; string[] probBParts = arg.Split(); for (i = 0; i < n; i++) model.PairwiseProbabilityB[i] = double.Parse(probBParts[i]); break; case "nr_sv": n = model.NumberOfClasses; model.NumberOfSVPerClass = new int[n]; string[] nrsvParts = arg.Split(); for (i = 0; i < n; i++) model.NumberOfSVPerClass[i] = int.Parse(nrsvParts[i]); break; case "SV": headerFinished = true; break; default: throw new Exception("Unknown text in model file"); } } // read sv_coef and SV int m = model.NumberOfClasses - 1; int l = model.SupportVectorCount; model.SupportVectorCoefficients = new double[m][]; for (int i = 0; i < m; i++) { model.SupportVectorCoefficients[i] = new double[l]; } model.SupportVectors = new Node[l][]; for (int i = 0; i < l; i++) { string[] parts = input.ReadLine().Trim().Split(); for (int k = 0; k < m; k++) model.SupportVectorCoefficients[k][i] = double.Parse(parts[k]); int n = parts.Length - m; model.SupportVectors[i] = new Node[n]; for (int j = 0; j < n; j++) { string[] nodeParts = parts[m + j].Split(':'); model.SupportVectors[i][j] = new Node(); model.SupportVectors[i][j].Index = int.Parse(nodeParts[0]); model.SupportVectors[i][j].Value = double.Parse(nodeParts[1]); } } TemporaryCulture.Stop(); return model; }
public static void svm_predict_values(Model model, Node[] x, double[] dec_values) { if (model.Parameter.SvmType == SvmType.ONE_CLASS || model.Parameter.SvmType == SvmType.EPSILON_SVR || model.Parameter.SvmType == SvmType.NU_SVR) { double[] sv_coef = model.SupportVectorCoefficients[0]; double sum = 0; for (int i = 0; i < model.SupportVectorCount; i++) sum += sv_coef[i] * Kernel.KernelFunction(x, model.SupportVectors[i], model.Parameter); sum -= model.Rho[0]; dec_values[0] = sum; } else { int i; int nr_class = model.NumberOfClasses; int l = model.SupportVectorCount; double[] kvalue = new double[l]; for (i = 0; i < l; i++) kvalue[i] = Kernel.KernelFunction(x, model.SupportVectors[i], model.Parameter); int[] start = new int[nr_class]; start[0] = 0; for (i = 1; i < nr_class; i++) start[i] = start[i - 1] + model.NumberOfSVPerClass[i - 1]; int p = 0; for (i = 0; i < nr_class; i++) for (int j = i + 1; j < nr_class; j++) { double sum = 0; int si = start[i]; int sj = start[j]; int ci = model.NumberOfSVPerClass[i]; int cj = model.NumberOfSVPerClass[j]; int k; double[] coef1 = model.SupportVectorCoefficients[j - 1]; double[] coef2 = model.SupportVectorCoefficients[i]; for (k = 0; k < ci; k++) sum += coef1[si + k] * kvalue[si + k]; for (k = 0; k < cj; k++) sum += coef2[sj + k] * kvalue[sj + k]; sum -= model.Rho[p]; dec_values[p] = sum; p++; } } }
public static int svm_check_probability_model(Model model) { if (((model.Parameter.SvmType == SvmType.C_SVC || model.Parameter.SvmType == SvmType.NU_SVC) && model.PairwiseProbabilityA != null && model.PairwiseProbabilityB != null) || ((model.Parameter.SvmType == SvmType.EPSILON_SVR || model.Parameter.SvmType == SvmType.NU_SVR) && model.PairwiseProbabilityA != null)) return 1; else return 0; }
// // Interface functions // public static Model svm_train(Problem prob, Parameter param) { Model model = new Model(); model.Parameter = param; if (param.SvmType == SvmType.ONE_CLASS || param.SvmType == SvmType.EPSILON_SVR || param.SvmType == SvmType.NU_SVR) { // regression or one-class-svm model.NumberOfClasses = 2; model.ClassLabels = null; model.NumberOfSVPerClass = null; model.PairwiseProbabilityA = null; model.PairwiseProbabilityB = null; model.SupportVectorCoefficients = new double[1][]; if (param.Probability && (param.SvmType == SvmType.EPSILON_SVR || param.SvmType == SvmType.NU_SVR)) { model.PairwiseProbabilityA = new double[1]; model.PairwiseProbabilityA[0] = svm_svr_probability(prob, param); } decision_function f = svm_train_one(prob, param, 0, 0); model.Rho = new double[1]; model.Rho[0] = f.rho; int nSV = 0; int i; for (i = 0; i < prob.Count; i++) if (Math.Abs(f.alpha[i]) > 0) ++nSV; model.SupportVectorCount = nSV; model.SupportVectors = new Node[nSV][]; model.SupportVectorCoefficients[0] = new double[nSV]; int j = 0; for (i = 0; i < prob.Count; i++) if (Math.Abs(f.alpha[i]) > 0) { model.SupportVectors[j] = prob.X[i]; model.SupportVectorCoefficients[0][j] = f.alpha[i]; ++j; } } else { // classification int l = prob.Count; int[] tmp_nr_class = new int[1]; int[][] tmp_label = new int[1][]; int[][] tmp_start = new int[1][]; int[][] tmp_count = new int[1][]; int[] perm = new int[l]; // group training data of the same class svm_group_classes(prob, tmp_nr_class, tmp_label, tmp_start, tmp_count, perm); int nr_class = tmp_nr_class[0]; int[] label = tmp_label[0]; int[] start = tmp_start[0]; int[] count = tmp_count[0]; Node[][] x = new Node[l][]; int i; for (i = 0; i < l; i++) x[i] = prob.X[perm[i]]; // calculate weighted C double[] weighted_C = new double[nr_class]; for (i = 0; i < nr_class; i++) weighted_C[i] = param.C; foreach (int weightedLabel in param.Weights.Keys) { int index = Array.IndexOf<int>(label, weightedLabel); if (index < 0) Console.Error.WriteLine("warning: class label " + weightedLabel + " specified in weight is not found"); else weighted_C[index] *= param.Weights[weightedLabel]; } // train k*(k-1)/2 models bool[] nonzero = new bool[l]; for (i = 0; i < l; i++) nonzero[i] = false; decision_function[] f = new decision_function[nr_class * (nr_class - 1) / 2]; double[] probA = null, probB = null; if (param.Probability) { probA = new double[nr_class * (nr_class - 1) / 2]; probB = new double[nr_class * (nr_class - 1) / 2]; } int p = 0; for (i = 0; i < nr_class; i++) for (int j = i + 1; j < nr_class; j++) { Problem sub_prob = new Problem(); int si = start[i], sj = start[j]; int ci = count[i], cj = count[j]; sub_prob.Count = ci + cj; sub_prob.X = new Node[sub_prob.Count][]; sub_prob.Y = new double[sub_prob.Count]; int k; for (k = 0; k < ci; k++) { sub_prob.X[k] = x[si + k]; sub_prob.Y[k] = +1; } for (k = 0; k < cj; k++) { sub_prob.X[ci + k] = x[sj + k]; sub_prob.Y[ci + k] = -1; } if (param.Probability) { double[] probAB = new double[2]; svm_binary_svc_probability(sub_prob, param, weighted_C[i], weighted_C[j], probAB); probA[p] = probAB[0]; probB[p] = probAB[1]; } f[p] = svm_train_one(sub_prob, param, weighted_C[i], weighted_C[j]); for (k = 0; k < ci; k++) if (!nonzero[si + k] && Math.Abs(f[p].alpha[k]) > 0) nonzero[si + k] = true; for (k = 0; k < cj; k++) if (!nonzero[sj + k] && Math.Abs(f[p].alpha[ci + k]) > 0) nonzero[sj + k] = true; ++p; } // build output model.NumberOfClasses = nr_class; model.ClassLabels = new int[nr_class]; for (i = 0; i < nr_class; i++) model.ClassLabels[i] = label[i]; model.Rho = new double[nr_class * (nr_class - 1) / 2]; for (i = 0; i < nr_class * (nr_class - 1) / 2; i++) model.Rho[i] = f[i].rho; if (param.Probability) { model.PairwiseProbabilityA = new double[nr_class * (nr_class - 1) / 2]; model.PairwiseProbabilityB = new double[nr_class * (nr_class - 1) / 2]; for (i = 0; i < nr_class * (nr_class - 1) / 2; i++) { model.PairwiseProbabilityA[i] = probA[i]; model.PairwiseProbabilityB[i] = probB[i]; } } else { model.PairwiseProbabilityA = null; model.PairwiseProbabilityB = null; } int nnz = 0; int[] nz_count = new int[nr_class]; model.NumberOfSVPerClass = new int[nr_class]; for (i = 0; i < nr_class; i++) { int nSV = 0; for (int j = 0; j < count[i]; j++) if (nonzero[start[i] + j]) { ++nSV; ++nnz; } model.NumberOfSVPerClass[i] = nSV; nz_count[i] = nSV; } Procedures.info("Total nSV = " + nnz + "\n"); model.SupportVectorCount = nnz; model.SupportVectors = new Node[nnz][]; p = 0; for (i = 0; i < l; i++) if (nonzero[i]) model.SupportVectors[p++] = x[i]; int[] nz_start = new int[nr_class]; nz_start[0] = 0; for (i = 1; i < nr_class; i++) nz_start[i] = nz_start[i - 1] + nz_count[i - 1]; model.SupportVectorCoefficients = new double[nr_class - 1][]; for (i = 0; i < nr_class - 1; i++) model.SupportVectorCoefficients[i] = new double[nnz]; p = 0; for (i = 0; i < nr_class; i++) for (int j = i + 1; j < nr_class; j++) { // classifier (i,j): coefficients with // i are in sv_coef[j-1][nz_start[i]...], // j are in sv_coef[i][nz_start[j]...] int si = start[i]; int sj = start[j]; int ci = count[i]; int cj = count[j]; int q = nz_start[i]; int k; for (k = 0; k < ci; k++) if (nonzero[si + k]) model.SupportVectorCoefficients[j - 1][q++] = f[p].alpha[k]; q = nz_start[j]; for (k = 0; k < cj; k++) if (nonzero[sj + k]) model.SupportVectorCoefficients[i][q++] = f[p].alpha[ci + k]; ++p; } } return model; }
public override ClassifierResult Build() { //For this example (and indeed, many scenarios), the default //parameters will suffice. Parameter parameters = new Parameter(); double C; double Gamma; //This will do a grid optimization to find the best parameters //and store them in C and Gamma, outputting the entire //search to params.txt. ParameterSelection.Grid(_trainProblem, parameters, "params.txt", out C, out Gamma); parameters.C = C; parameters.Gamma = Gamma; //Train the model using the optimal parameters. _model = Training.Train(_trainProblem, parameters); // пробегаем по всем клиентски данным и сохраняем результат var probDictList = new Dictionary<string, Dictionary<int, double>>(); foreach (string id in _testDataDict.Keys) { if (!probDictList.ContainsKey(id)) probDictList.Add(id, new Dictionary<int, double>()); foreach (var sarr in _testDataDict[id]) { var y = PredictProba(sarr); double prob = y.Probs[1]; int kmax = probDictList[id].Keys.Count == 0 ? 0 : probDictList[id].Keys.Max() + 1; probDictList[id].Add(kmax, prob); } } // вероятность дефолта определяется как среднее по записям для клиента var probDict = new Dictionary<string, double>(); foreach (var id in probDictList.Keys) { int cnt = probDictList[id].Keys.Count(); double prob = 0; foreach (var d in probDictList[id].Keys) { prob += probDictList[id][d]; } if (!probDict.ContainsKey(id)) probDict.Add(id, prob / cnt); } // находим статистики классификации var rlist = new RocItem[_resultDict.Count]; // массив для оценки результата int idx = 0; foreach (string id in probDict.Keys) { if (rlist[idx] == null) rlist[idx] = new RocItem(); rlist[idx].Prob = probDict[id]; // среднее по наблюдениям rlist[idx].Target = _resultDict[id]; rlist[idx].Predicted = 0; idx++; } Array.Sort(rlist, (o1, o2) => (1 - o1.Prob).CompareTo(1 - o2.Prob)); var cres = ResultCalc.GetResult(rlist, 0.05); var clsRes = new ClassifierResult(); clsRes.BestResult = cres; clsRes.LastResult = cres; clsRes.ResDict.Add(0, cres); return clsRes; }