/// <summary>Extracts a cross validation set from a given set</summary> /// <param name="Set">Set to extract cross validation from</param> /// <param name="CrossValidationSetPercent">Percentage of elements to extract</param> public static TrainingSet GetCrossValidationSet(TrainingSet Set, float CrossValidationSetPercent) { TrainingSet CrossValidationSet = new TrainingSet(); int nCrossSet = (int)(CrossValidationSetPercent * (float)Set.getN); Random rnd = new Random(); for (int i = 0; i < nCrossSet; i++) { int ind = rnd.Next(0, Set.trainingArray.Count - 1); TrainingUnit u = Set.trainingArray[ind]; Set.trainingArray.Remove(u); CrossValidationSet.addTrainingUnit(u); } return CrossValidationSet; }
/// <summary>Trains current SVM with cross-validation, adjusting kernel parameter lambda and box parameter C. /// Returns best achieved efficiency.</summary> /// <param name="CrossValidationSet">Cross validation set</param> /// <param name="LambdaSet">Lambda set</param> /// <param name="CSet">C values set</param> public float TrainWithCrossValidation(TrainingSet CrossValidationSet, float[] LambdaSet, float[] CSet) { foreach (float _lambda in LambdaSet) { for (int i = 0; i < SVMs.Count; i++) { SVMs[i].ProblemCfg.lambda = _lambda; } foreach (float _c in CSet) { for (int i = 0; i < SVMs.Count; i++) { SVMs[i].ProblemCfg.c = _c; SVMs[i].Train(); } float performance = this.GetHitRate(CrossValidationSet); if (CrossValParams == null) { CrossValParams = new float[3]; } if (performance > CrossValParams[0]) { CrossValParams[0] = performance; CrossValParams[1] = _lambda; CrossValParams[2] = _c; } } } //Train with best parameters for (int i = 0; i < SVMs.Count; i++) { SVMs[i].ProblemCfg.lambda = CrossValParams[1]; SVMs[i].ProblemCfg.c = CrossValParams[2]; SVMs[i].Train(); } return(CrossValParams[0]); }
/// <summary>Computes the i-th line of matrix K[i][j]</summary> /// <param name="problemSolution">SVM to solve</param> /// <param name="i">Kernel line number to compute</param> private static void ComputeKernels(SVM problemSolution, int i) { if (problemSolution.TrainingSet.IsKernelCalculated[i]) { return; } TrainingSet trainingSet = problemSolution.TrainingSet; problemSolution.TrainingSet.kernels[i] = new float[problemSolution.TrainingSet.getN]; trainingSet.kernels[i] = new float[trainingSet.getN]; problemSolution.TrainingSet.IsKernelCalculated[i] = true; for (int j = 0; j < trainingSet.getN; j++) { trainingSet.kernels[i][j] = calculateSingleKernel(trainingSet.trainingArray[i], trainingSet.trainingArray[j], problemSolution); //trainingSet.kernels[j][i] = trainingSet.kernels[i][j]; } }
/// <summary>Computes All kernels and errors accelerating with OpenCL</summary> /// <param name="problemSolution">Problem solution SVM</param> public static void CLcalculateAllKernels(SVM problemSolution) { TrainingSet trainingSet = problemSolution.TrainingSet; ProblemConfig problemConfig = problemSolution.ProblemCfg; trainingSet.errors = new float[trainingSet.getN]; trainingSet.kernels = new float[trainingSet.getN][]; trainingSet.IsKernelCalculated = new bool[trainingSet.getN]; // Caching kernels for (int i = 0; i < trainingSet.getN; i++) { if (problemSolution.alphaList[i] != 0) { CLComputeKernels(problemSolution, i); } } }
private static void CLupdateErrorsCache(TrainingSet trainingSet, SVM svm, float oldAlphai, float newAlphai, int iIndex, float oldAlphaj, float newAlphaj, int jIndex, float oldB, float newB) { float alphaiDif = newAlphai - oldAlphai; float alphajDif = newAlphaj - oldAlphaj; float BDif = newB - oldB; if (trainingSet.trainingArray[iIndex].y < 0) { alphaiDif = -alphaiDif; } if (trainingSet.trainingArray[jIndex].y < 0) { alphajDif = -alphajDif; } lock (CLResource) { //Writes kernel values if (svm.CLKi == null || svm.CLKi.OriginalVarLength != svm.TrainingSet.errors.Length) { svm.CLKi = new CLCalc.Program.Variable(svm.TrainingSet.kernels[iIndex]); svm.CLKj = new CLCalc.Program.Variable(svm.TrainingSet.kernels[jIndex]); } else { svm.CLKi.WriteToDevice(svm.TrainingSet.kernels[iIndex]); svm.CLKj.WriteToDevice(svm.TrainingSet.kernels[jIndex]); } float[] p = new float[3] { alphaiDif, BDif, alphajDif }; svm.CLUpdtErrParams.WriteToDevice(p); //Executes update using GPU kernelUpdateErr.Execute(new CLCalc.Program.Variable[] { svm.CLerr, svm.CLKi, svm.CLKj, svm.CLUpdtErrParams }, svm.TrainingSet.getN); svm.CLerr.ReadFromDeviceTo(svm.TrainingSet.errors); } }
private static float calculateFx(int indexX, SVM currentSolution) { TrainingSet trainingSet = currentSolution.TrainingSet; ProblemConfig problemConfig = currentSolution.ProblemCfg; float sum = 0; for (int i = 0; i < trainingSet.getN; i++) { if (trainingSet.trainingArray[i].y > 0) { sum += currentSolution.alphaList[i] * trainingSet.kernels[i][indexX]; } else { sum -= currentSolution.alphaList[i] * trainingSet.kernels[i][indexX]; } } return(sum + currentSolution.b); }
/// <summary>Loads and classifies dataset</summary> private void LoadMITFaceClassifier() { /* * * CBCL Face Database #1 * MIT Center For Biological and Computation Learning * */ string p = System.Windows.Forms.Application.StartupPath; string fileTrain = p + "\\svm.train.normgrey"; string fileTest = p + "\\svm.test.normgrey"; tSet = new TrainingSet(); //Fills both, we're not testing the results FillTrainingSet(fileTrain, tSet); FillTrainingSet(fileTest, tSet); SVM = new MultiClassSVM(tSet); }
/// <summary>Trains current SVM with cross-validation, adjusting kernel parameter lambda and box parameter C. /// Returns best achieved efficiency.</summary> /// <param name="CrossValidationSet">Cross validation set</param> public float TrainWithCrossValidation(TrainingSet CrossValidationSet) { Random rnd = new Random(); float[] lambdaSet = new float[12]; //lambdaSet[0] = 3E-9f * ((float)rnd.NextDouble() + 1); lambdaSet[0] = 3E-3f * ((float)rnd.NextDouble() + 1); for (int i = 1; i < lambdaSet.Length; i++) { lambdaSet[i] = 4.5f * lambdaSet[i - 1]; } float[] cSet = new float[13]; cSet[0] = 1E-5f * ((float)rnd.NextDouble() + 1); for (int i = 1; i < cSet.Length; i++) { cSet[i] = 2.0f * cSet[i - 1]; } return(TrainWithCrossValidation(CrossValidationSet, lambdaSet, cSet)); }
/// <summary>Adds a new self training example</summary> public void AddSelfTraining(int[] sbFrames, int faceIndex, Bitmap bmp) { if (SelfTSet == null) { SelfTSet = new TrainingSet(); } float[] subF = new float[(sbFrames.Length / 3) * 364]; ExtractFeatures(sbFrames, subF, bmp); for (int i = 0; i < sbFrames.Length / 3; i++) { float[] x = new float[364]; for (int k = 0; k < 364; k++) { x[k] = subF[k + i * 364]; } TrainingUnit tu = new TrainingUnit(x, i == faceIndex ? 1.0f : -1.0f); SelfTSet.addTrainingUnit(tu); } }
/// <summary> /// Predicts the output of a single entry, given a previous problem, solution and correspondent training set /// </summary> /// <param name="problemSolution">Correspondent problem solution</param> /// <param name="untrainedUnit">Input features from which the output will be predicted</param> /// <returns>The y classification (true/false = positive/negative)</returns> public static float predictOutput(SVM problemSolution, TrainingUnit untrainedUnit) { TrainingSet trainingSet = problemSolution.TrainingSet; ProblemConfig problemConfig = problemSolution.ProblemCfg; // F(x) = sum + b // sum = summation of alpha_i * y_i * kernel(untrained unit, i) for all i in the training set float sum = 0; for (int i = 0; i < trainingSet.getN; i++) { if (trainingSet.trainingArray[i].y > 0) { sum += problemSolution.alphaList[i] * calculateSingleKernel(trainingSet.trainingArray[i], untrainedUnit, problemSolution); } else { sum -= problemSolution.alphaList[i] * calculateSingleKernel(trainingSet.trainingArray[i], untrainedUnit, problemSolution); } } return(sum + problemSolution.b); }
/// <summary>Creates a new multiclass SVM using desired outputs from training set. Classifications -1.0f are negative for all sets</summary> /// <param name="TSet">Training set</param> /// <param name="SVMCfg">Configuration parameters</param> /// <param name="PreCalibrate">Precalibrate RBF parameter lambda? This will ignore the given value</param> private void initMultiSVM(TrainingSet TSet, ProblemConfig SVMCfg, bool PreCalibrate) { //Determines how many different classifications are there Classifications = new List <float>(); foreach (TrainingUnit tu in TSet.trainingArray) { if (Classifications.IndexOf(tu.y) < 0 && tu.y != -1.0f) { Classifications.Add(tu.y); } } //For each different possible classification, create a different SVM SVMs = new List <SVM>(); foreach (float c in Classifications) { SVM svm = new SVM(); svm.TrainingSet = new TrainingSet(); svm.ProblemCfg = SVMCfg.Clone(); SVMs.Add(svm); foreach (TrainingUnit tu in TSet.trainingArray) { TrainingUnit newTu = tu.Clone(); newTu.y = tu.y == c ? 1 : -1; svm.TrainingSet.addTrainingUnit(newTu); } //Train svm if (PreCalibrate) { svm.PreCalibrateCfg(0.8f / (float)Math.Sqrt(svm.TrainingSet.getN), 0.3f / (float)Math.Sqrt(svm.TrainingSet.getN)); } svm.Train(); svm.RemoveNonSupportVectors(); } }
/// <summary>Adds a new self training example</summary> public void AddSelfTraining(int[] sbFrames, int faceIndex, Bitmap bmp) { if (SelfTSet == null) SelfTSet = new TrainingSet(); float[] subF = new float[(sbFrames.Length / 3) * 364]; ExtractFeatures(sbFrames, subF, bmp); for (int i = 0; i < sbFrames.Length / 3; i++) { float[] x = new float[364]; for (int k = 0; k < 364; k++) x[k] = subF[k + i * 364]; TrainingUnit tu = new TrainingUnit(x, i == faceIndex ? 1.0f : -1.0f); SelfTSet.addTrainingUnit(tu); } }
/// <summary>Adds training units to a set from a file</summary> /// <param name="filename">File containing features</param> /// <param name="TrSet">Training set to be populated</param> private void FillTrainingSet(string filename, TrainingSet TrSet) { string sepdec = (1.5).ToString().Substring(1, 1); using (StreamReader sr = new StreamReader(filename)) { string line; line = sr.ReadLine(); int n = int.Parse(line); line = sr.ReadLine(); int dim = (int)Math.Sqrt(double.Parse(line)); for (int i = 0; i < n; i++) { line = sr.ReadLine().Replace(".", sepdec); string[] s = line.Split(new string[] { " " }, StringSplitOptions.RemoveEmptyEntries); float[] x = new float[364]; float y; for (int j = 0; j < s.Length - 1; j++) x[j] = float.Parse(s[j]); y = float.Parse(s[s.Length - 1]); TrSet.addTrainingUnit(new TrainingUnit(x, y)); /* * Features Haar * [04:05:53] Edmundo ITA05(FOX.Howler): 1,1 [04:05:57] Edmundo ITA05(FOX.Howler): 7,4 [04:06:55] Edmundo ITA05(FOX.Howler): new Rectangle(1, 1, 7, 4) [04:07:06] Edmundo ITA05(FOX.Howler): 7 x 4 [04:08:15] Edmundo ITA05(FOX.Howler): new Rectangle(11, 4, 7, 4) [04:08:24] Edmundo ITA05(FOX.Howler): divide por 7*4 [04:09:04] Edmundo ITA05(FOX.Howler): subtrai da área do Rect(8, 1, 3, 4) [04:09:13] Edmundo ITA05(FOX.Howler): dividido pela 3 *4 * * * * [04:05:53] Edmundo ITA05(FOX.Howler): 1,1 [04:05:57] Edmundo ITA05(FOX.Howler): 7,4 [04:06:55] Edmundo ITA05(FOX.Howler): new Rectangle(1, 1, 7, 4) [04:07:06] Edmundo ITA05(FOX.Howler): 7 x 4 [04:08:15] Edmundo ITA05(FOX.Howler): new Rectangle(11, 4, 7, 4) [04:08:24] Edmundo ITA05(FOX.Howler): divide por 7*4 [04:09:04] Edmundo ITA05(FOX.Howler): subtrai da área do Rect(8, 1, 3, 4) [04:09:13] Edmundo ITA05(FOX.Howler): dividido pela 3 *4 [04:13:38] Edmundo ITA05(FOX.Howler): ((1,6,5,5) + (13,6,5,5)) / 2 - (7,6,5,6) * * * [04:17:15] Edmundo ITA05(FOX.Howler): (1,1,17,5) - (1,6,17,8) dividir pela area */ } } }
/// <summary>Trains current SVM with cross-validation, adjusting kernel parameter lambda and box parameter C. Returns best performance so far</summary> /// <param name="CrossValidationSetPercent">Percentage of training examples that should be used as cross validation set</param> /// <param name="lambdaSet">Values of lambda to try</param> /// <param name="CSet">Values of c to try</param> public float TrainWithCrossValidation(float CrossValidationSetPercent, float[] lambdaSet, float[] CSet) { if (alphaList == null || alphaList.Count != TrainingSet.getN) { //Problem changed, previous values dont make sense initializeWithZeros(); CrossValParams = null; } #region Constructs cross validation set TrainingSet CrossValidationSet = new TrainingSet(); int nCrossSet = (int)(CrossValidationSetPercent * (float)this.TrainingSet.getN); Random rnd = new Random(); for (int i = 0; i < nCrossSet; i++) { int ind = rnd.Next(0, this.TrainingSet.trainingArray.Count - 1); TrainingUnit u = this.TrainingSet.trainingArray[ind]; this.TrainingSet.trainingArray.Remove(u); CrossValidationSet.addTrainingUnit(u); } #endregion #region Loops through lambdas and Cs and finds maximum crossvalidation foreach (float _lambda in lambdaSet) { this.ProblemCfg.lambda = _lambda; this.initializeWithZeros(); PreComputeKernels(); foreach (float _c in CSet) { this.ProblemCfg.c = _c; //ProblemSolver.solveSMOStartingFromPreviousSolution(this); ProblemSolver.solveSMOStartingFromZero(this); float performance = this.GetHitRate(CrossValidationSet); if (CrossValParams == null) CrossValParams = new float[3]; if (performance > CrossValParams[0]) { CrossValParams[0] = performance; CrossValParams[1] = _lambda; CrossValParams[2] = _c; } } } #endregion #region Trains with best parameters so far this.ProblemCfg.lambda = CrossValParams[1]; this.ProblemCfg.c = CrossValParams[2]; this.Train(); #endregion return CrossValParams[0]; }
/// <summary>Computes hit rates for a given test set</summary> /// <param name="samples">Test set to be used</param> public float GetHitRate(TrainingSet samples) { float rate = 0; foreach (TrainingUnit tu in samples.trainingArray) { bool c = Classify(tu); if ((c && tu.y == 1) || ((!c) && tu.y == -1)) rate++; } return rate / (float)samples.getN; }
/* /// <summary> /// Copy all values from another solution /// </summary> /// <param name="sourceSolution">The source to copy from</param> public void Load(SVM sourceSolution) { dimension = sourceSolution.dimension; alphaList = new float[dimension]; for (int i = 0; i < dimension; i++) { alphaList[i] = sourceSolution.alphaList[i]; } b = sourceSolution.b; } */ /// <summary> /// Copy all values from another solution /// </summary> /// <param name="FileName">File containing alpha's data</param> public void Load(string FileName) { DataSet d = new DataSet(); d.ReadXml(FileName); DataTable t = d.Tables["Solution"]; dimension = t.Rows.Count; //Configuration DataTable TblCfg = d.Tables["Config"]; float valC, valTol; int valKernel, valMaxP; valC = (float)((double)TblCfg.Rows[0]["dblValues"]); valKernel = (int)((double)TblCfg.Rows[1]["dblValues"]); valTol = (float)((double)TblCfg.Rows[2]["dblValues"]); valMaxP = (int)((double)TblCfg.Rows[3]["dblValues"]); this.b = (float)((double)TblCfg.Rows[4]["dblValues"]); float Lambda = (float)((double)TblCfg.Rows[5]["dblValues"]); int xDim = (int)((double)TblCfg.Rows[6]["dblValues"]); //Reads classifications DataTable TblClassif = d.Tables["Classifications"]; alphaList = new List<float>(); TrainingSet = new TrainingSet(); for (int i = 0; i < dimension; i++) { TrainingSet.addTrainingUnit(new TrainingUnit(new float[xDim], -1)); } for (int i = 0; i < dimension; i++) { alphaList.Add((float)((double)t.Rows[i]["dblValues"])); TrainingSet.trainingArray[i].y = (float)((double)TblClassif.Rows[i]["dblValues"]) > 0 ? 1 : -1; } //Reads training set //Creates datatables for training examples DataTable Tbl = d.Tables["Examples"]; for (int i = 0; i < dimension; i ++) { for (int j = 0; j < xDim; j++) { TrainingSet.trainingArray[i].xVector[j] = (float)((double)Tbl.Rows[j + i*xDim]["dblValues"]); } } this.ProblemCfg = new ProblemConfig(Lambda, valC, valTol, valMaxP, (ProblemConfig.KernelType)valKernel); if (OpenCLTemplate.CLCalc.CLAcceleration == OpenCLTemplate.CLCalc.CLAccelerationType.UsingCL) { this.WriteToDevice(); } }
/// <summary>Attempts to pre-calibrate configuration parameters. /// Finds an alpha that enhances similarities between positive examples /// and reduces similarities between positive and negative examples. /// Assumes that decreasing lambda increases kernel match. /// </summary> /// <param name="tolPositive">Positive kernels average should be greater than tolPositive</param> /// <param name="tolNegative">Negative kernels average should be lesser than tolNegative</param> public void PreCalibrateCfg(float tolPositive, float tolNegative) { #region Checks if there are positive and negative examples bool posSamples = false; bool negSamples = false; for (int i = 0; i < TrainingSet.trainingArray.Count; i++) { if (TrainingSet.trainingArray[i].y > 0) posSamples = true; if (TrainingSet.trainingArray[i].y < 0) negSamples = true; if (posSamples && negSamples) i = TrainingSet.trainingArray.Count; } if ((!posSamples) || (!negSamples)) throw new Exception("Training set must contain positive and negative samples"); #endregion Random rnd = new Random(); int nSet = (int)(20 * Math.Log(TrainingSet.getN, 2)); TrainingSet PositiveExamples1 = new TrainingSet(); TrainingSet PositiveExamples2 = new TrainingSet(); TrainingSet NegativeExamples = new TrainingSet(); //Kernel average for positive and negative samples float positiveAvg = 0, negativeAvg = 0; float invN = 1 / (float)nSet; int count = 0; float bestLambda = ProblemCfg.lambda; float maxPosNegAvg = -1.0f; while ((positiveAvg <= tolPositive || negativeAvg >= tolNegative) && count < nSet) { //Populates training sets PositiveExamples1.trainingArray.Clear(); PositiveExamples2.trainingArray.Clear(); NegativeExamples.trainingArray.Clear(); while (PositiveExamples1.getN < nSet || PositiveExamples2.getN < nSet || NegativeExamples.getN < nSet) { TrainingUnit tu = TrainingSet.trainingArray[rnd.Next(TrainingSet.trainingArray.Count - 1)]; if (tu.y > 0 && PositiveExamples1.getN < nSet) PositiveExamples1.addTrainingUnit(tu); else if (tu.y > 0 && PositiveExamples2.getN < nSet) PositiveExamples2.addTrainingUnit(tu); if (tu.y < 0 && NegativeExamples.getN < nSet) NegativeExamples.addTrainingUnit(tu); } count++; positiveAvg = 0; negativeAvg = 0; for (int i = 0; i < nSet; i++) { positiveAvg += ProblemSolver.calculateSingleKernel(PositiveExamples1.trainingArray[i], PositiveExamples2.trainingArray[i], this); negativeAvg += ProblemSolver.calculateSingleKernel(PositiveExamples1.trainingArray[i], NegativeExamples.trainingArray[i], this); } positiveAvg *= invN; negativeAvg *= invN; if (maxPosNegAvg < positiveAvg - negativeAvg) { bestLambda = ProblemCfg.lambda; maxPosNegAvg = positiveAvg - negativeAvg; } //Desired: positiveAvg=1, negativeAvg = 0 if (positiveAvg <= tolPositive) this.ProblemCfg.lambda *= 0.15f; else if (negativeAvg >= tolNegative) this.ProblemCfg.lambda *= 1.2f; } ProblemCfg.lambda = bestLambda; }
private static void updateErrorsCache(TrainingSet trainingSet, SVM currentSolution, float oldAlphai, float newAlphai, int iIndex, float oldAlphaj, float newAlphaj, int jIndex, float oldB, float newB) { float alphaiDif = newAlphai - oldAlphai; float alphajDif = newAlphaj - oldAlphaj; float BDif = newB - oldB; if (trainingSet.trainingArray[iIndex].y < 0) alphaiDif = -alphaiDif; if (trainingSet.trainingArray[jIndex].y < 0) alphajDif = -alphajDif; for (int t = 0; t < trainingSet.getN; t++) { float variation = alphaiDif * trainingSet.kernels[iIndex][t]; variation += alphajDif * trainingSet.kernels[jIndex][t]; variation += BDif; trainingSet.errors[t] += variation; } }
private static void updateSingleError(TrainingSet trainingSet, SVM currentSolution, float newAlphai, int iIndex) { for (int t = 0; t < trainingSet.getN; t++) { float variation = 0; if (trainingSet.trainingArray[iIndex].y > 0) { variation += newAlphai * trainingSet.kernels[iIndex][t]; } else { variation -= newAlphai * trainingSet.kernels[iIndex][t]; } trainingSet.errors[t] += variation; } }
/// <summary>Creates a new multiclass SVM using desired outputs from training set. Classifications -1.0f are negative for all sets</summary> /// <param name="TSet">Training set</param> /// <param name="SVMCfg">Configuration parameters</param> private void initMultiSVM(TrainingSet TSet, ProblemConfig SVMCfg) { //Determines how many different classifications are there Classifications = new List<float>(); foreach (TrainingUnit tu in TSet.trainingArray) { if (Classifications.IndexOf(tu.y) < 0 && tu.y != -1.0f) Classifications.Add(tu.y); } //For each different possible classification, create a different SVM SVMs = new List<SVM>(); foreach (float c in Classifications) { SVM svm = new SVM(); svm.TrainingSet = new TrainingSet(); svm.ProblemCfg = SVMCfg.Clone(); SVMs.Add(svm); foreach (TrainingUnit tu in TSet.trainingArray) { TrainingUnit newTu = tu.Clone(); newTu.y = tu.y == c ? 1 : -1; svm.TrainingSet.addTrainingUnit(newTu); } //Train svm svm.PreCalibrateCfg(0.8f / (float)Math.Sqrt(svm.TrainingSet.getN), 0.3f / (float)Math.Sqrt(svm.TrainingSet.getN)); svm.Train(); svm.RemoveNonSupportVectors(); } }
/// <summary>Creates a new multiclass SVM using desired outputs from training set. Classifications -1.0f are negative for all sets</summary> /// <param name="TSet">Training set</param> /// <param name="SVMCfg">Configuration parameters</param> public MultiClassSVM(TrainingSet TSet, ProblemConfig SVMCfg) { initMultiSVM(TSet, SVMCfg); }
/// <summary>Trains current SVM with cross-validation, adjusting kernel parameter lambda and box parameter C. /// Returns best achieved efficiency.</summary> /// <param name="CrossValidationSet">Cross validation set</param> public float TrainWithCrossValidation(TrainingSet CrossValidationSet) { Random rnd = new Random(); float[] lambdaSet = new float[12]; //lambdaSet[0] = 3E-9f * ((float)rnd.NextDouble() + 1); lambdaSet[0] = 3E-3f * ((float)rnd.NextDouble() + 1); for (int i = 1; i < lambdaSet.Length; i++) lambdaSet[i] = 4.5f * lambdaSet[i - 1]; float[] cSet = new float[13]; cSet[0] = 1E-5f * ((float)rnd.NextDouble() + 1); for (int i = 1; i < cSet.Length; i++) cSet[i] = 2.0f * cSet[i - 1]; return TrainWithCrossValidation(CrossValidationSet, lambdaSet, cSet); }
/// <summary>Loads and classifies dataset</summary> private void LoadMITFaceClassifier() { /* CBCL Face Database #1 MIT Center For Biological and Computation Learning * */ string p = System.Windows.Forms.Application.StartupPath; string fileTrain = p + "\\svm.train.normgrey"; string fileTest = p + "\\svm.test.normgrey"; tSet = new TrainingSet(); //Fills both, we're not testing the results FillTrainingSet(fileTrain, tSet); FillTrainingSet(fileTest, tSet); SVM = new MultiClassSVM(tSet); }
private static void CLupdateErrorsCache(TrainingSet trainingSet, SVM svm, float oldAlphai, float newAlphai, int iIndex, float oldAlphaj, float newAlphaj, int jIndex, float oldB, float newB) { float alphaiDif = newAlphai - oldAlphai; float alphajDif = newAlphaj - oldAlphaj; float BDif = newB - oldB; if (trainingSet.trainingArray[iIndex].y < 0) alphaiDif = -alphaiDif; if (trainingSet.trainingArray[jIndex].y < 0) alphajDif = -alphajDif; lock (CLResource) { //Writes kernel values if (svm.CLKi == null || svm.CLKi.OriginalVarLength != svm.TrainingSet.errors.Length) { svm.CLKi = new CLCalc.Program.Variable(svm.TrainingSet.kernels[iIndex]); svm.CLKj = new CLCalc.Program.Variable(svm.TrainingSet.kernels[jIndex]); } else { svm.CLKi.WriteToDevice(svm.TrainingSet.kernels[iIndex]); svm.CLKj.WriteToDevice(svm.TrainingSet.kernels[jIndex]); } float[] p = new float[3] { alphaiDif, BDif, alphajDif }; svm.CLUpdtErrParams.WriteToDevice(p); //Executes update using GPU kernelUpdateErr.Execute(new CLCalc.Program.Variable[] { svm.CLerr, svm.CLKi, svm.CLKj, svm.CLUpdtErrParams }, svm.TrainingSet.getN); svm.CLerr.ReadFromDeviceTo(svm.TrainingSet.errors); } }
/// <summary>Attempts to pre-calibrate configuration parameters. /// Finds an alpha that enhances similarities between positive examples /// and reduces similarities between positive and negative examples. /// Assumes that decreasing lambda increases kernel match. /// </summary> /// <param name="tolPositive">Positive kernels average should be greater than tolPositive</param> /// <param name="tolNegative">Negative kernels average should be lesser than tolNegative</param> public void PreCalibrateCfg(float tolPositive, float tolNegative) { #region Checks if there are positive and negative examples bool posSamples = false; bool negSamples = false; for (int i = 0; i < TrainingSet.trainingArray.Count; i++) { if (TrainingSet.trainingArray[i].y > 0) { posSamples = true; } if (TrainingSet.trainingArray[i].y < 0) { negSamples = true; } if (posSamples && negSamples) { i = TrainingSet.trainingArray.Count; } } if ((!posSamples) || (!negSamples)) { throw new Exception("Training set must contain positive and negative samples"); } #endregion Random rnd = new Random(); int nSet = (int)(20 * Math.Log(TrainingSet.getN, 2)); TrainingSet PositiveExamples1 = new TrainingSet(); TrainingSet PositiveExamples2 = new TrainingSet(); TrainingSet NegativeExamples = new TrainingSet(); //Kernel average for positive and negative samples float positiveAvg = 0, negativeAvg = 0; float invN = 1 / (float)nSet; int count = 0; float bestLambda = ProblemCfg.lambda; float maxPosNegAvg = -1.0f; while ((positiveAvg <= tolPositive || negativeAvg >= tolNegative) && count < nSet) { //Populates training sets PositiveExamples1.trainingArray.Clear(); PositiveExamples2.trainingArray.Clear(); NegativeExamples.trainingArray.Clear(); while (PositiveExamples1.getN < nSet || PositiveExamples2.getN < nSet || NegativeExamples.getN < nSet) { TrainingUnit tu = TrainingSet.trainingArray[rnd.Next(TrainingSet.trainingArray.Count - 1)]; if (tu.y > 0 && PositiveExamples1.getN < nSet) { PositiveExamples1.addTrainingUnit(tu); } else if (tu.y > 0 && PositiveExamples2.getN < nSet) { PositiveExamples2.addTrainingUnit(tu); } if (tu.y < 0 && NegativeExamples.getN < nSet) { NegativeExamples.addTrainingUnit(tu); } } count++; positiveAvg = 0; negativeAvg = 0; for (int i = 0; i < nSet; i++) { positiveAvg += ProblemSolver.calculateSingleKernel(PositiveExamples1.trainingArray[i], PositiveExamples2.trainingArray[i], this); negativeAvg += ProblemSolver.calculateSingleKernel(PositiveExamples1.trainingArray[i], NegativeExamples.trainingArray[i], this); } positiveAvg *= invN; negativeAvg *= invN; if (maxPosNegAvg < positiveAvg - negativeAvg) { bestLambda = ProblemCfg.lambda; maxPosNegAvg = positiveAvg - negativeAvg; } //Desired: positiveAvg=1, negativeAvg = 0 if (positiveAvg <= tolPositive) { this.ProblemCfg.lambda *= 0.15f; } else if (negativeAvg >= tolNegative) { this.ProblemCfg.lambda *= 1.2f; } } ProblemCfg.lambda = bestLambda; }
/// <summary>Trains current SVM with cross-validation, adjusting kernel parameter lambda and box parameter C. Returns best performance so far</summary> /// <param name="CrossValidationSetPercent">Percentage of training examples that should be used as cross validation set</param> /// <param name="lambdaSet">Values of lambda to try</param> /// <param name="CSet">Values of c to try</param> public float TrainWithCrossValidation(float CrossValidationSetPercent, float[] lambdaSet, float[] CSet) { if (alphaList == null || alphaList.Count != TrainingSet.getN) { //Problem changed, previous values dont make sense initializeWithZeros(); CrossValParams = null; } #region Constructs cross validation set TrainingSet CrossValidationSet = new TrainingSet(); int nCrossSet = (int)(CrossValidationSetPercent * (float)this.TrainingSet.getN); Random rnd = new Random(); for (int i = 0; i < nCrossSet; i++) { int ind = rnd.Next(0, this.TrainingSet.trainingArray.Count - 1); TrainingUnit u = this.TrainingSet.trainingArray[ind]; this.TrainingSet.trainingArray.Remove(u); CrossValidationSet.addTrainingUnit(u); } #endregion #region Loops through lambdas and Cs and finds maximum crossvalidation foreach (float _lambda in lambdaSet) { this.ProblemCfg.lambda = _lambda; this.initializeWithZeros(); PreComputeKernels(); foreach (float _c in CSet) { this.ProblemCfg.c = _c; //ProblemSolver.solveSMOStartingFromPreviousSolution(this); ProblemSolver.solveSMOStartingFromZero(this); float performance = this.GetHitRate(CrossValidationSet); if (CrossValParams == null) { CrossValParams = new float[3]; } if (performance > CrossValParams[0]) { CrossValParams[0] = performance; CrossValParams[1] = _lambda; CrossValParams[2] = _c; } } } #endregion #region Trains with best parameters so far this.ProblemCfg.lambda = CrossValParams[1]; this.ProblemCfg.c = CrossValParams[2]; this.Train(); #endregion return(CrossValParams[0]); }
/// <summary>Gets SVM hit rate</summary> /// <param name="TestSet">Test set</param> public float GetHitRate(TrainingSet TestSet) { float rate = 0, val; foreach (TrainingUnit tu in TestSet.trainingArray) { float resp = Classify(tu, out val); if (resp == tu.y) rate++; } return rate / (float)TestSet.trainingArray.Count; }
/// <summary> /// Solves the SMO considering no previous knowledge about the problem /// </summary> /// <param name="problemSolution">Known solution</param> /// <returns>Solution of the problem with alphas and threshold</returns> public static SVM solveSMOStartingFromPreviousSolution(SVM problemSolution) { System.Diagnostics.Stopwatch swTotalTime = new System.Diagnostics.Stopwatch(); System.Diagnostics.Stopwatch swHeuristica = new System.Diagnostics.Stopwatch(); System.Diagnostics.Stopwatch swComputeKernel = new System.Diagnostics.Stopwatch(); System.Diagnostics.Stopwatch swUpdateError = new System.Diagnostics.Stopwatch(); swTotalTime.Start(); ProblemConfig problemConfig = problemSolution.ProblemCfg; if (problemSolution.alphaList == null) { problemSolution.initializeWithZeros(); } ProblemSolver.calculateErrors(problemSolution); //Initializes GPU error vector if (OpenCLTemplate.CLCalc.CLAcceleration == OpenCLTemplate.CLCalc.CLAccelerationType.UsingCL) { WriteCLErr(problemSolution); } TrainingSet trainingSet = problemSolution.TrainingSet; int passes = 0; int m = trainingSet.getN; while (passes < problemConfig.maxPasses) { int changedAlphas = 0; for (int i = 0; i < m; i++) { float yi = trainingSet.trainingArray[i].y; float alpha_i = problemSolution.alphaList[i]; // Error between the SVM output on the ith training unit and the true ith output float ei = trainingSet.errors[i]; // KKT conditions for ith element if ( ((yi * ei < -problemConfig.tol && alpha_i < problemConfig.c) || (yi * ei > problemConfig.tol && alpha_i > 0)) ) { swHeuristica.Start(); #region Computes J using maximum variation heuristics // Get a number from 0 to m - 1 not equal to i int j = 0; if (trainingSet.errors[i] >= 0) { if (OpenCLTemplate.CLCalc.CLAcceleration == OpenCLTemplate.CLCalc.CLAccelerationType.UsingCL) { j = CLFindMinError(problemSolution); } else { float minError = trainingSet.errors[0]; for (int k = 1; k < trainingSet.getN; k++) { if (minError > trainingSet.errors[k]) { minError = trainingSet.errors[k]; j = k; } } } } else { if (OpenCLTemplate.CLCalc.CLAcceleration == OpenCLTemplate.CLCalc.CLAccelerationType.UsingCL) { j = CLFindMaxError(problemSolution); } else { float maxError = trainingSet.errors[0]; for (int k = 1; k < trainingSet.getN; k++) { if (maxError < trainingSet.errors[k]) { maxError = trainingSet.errors[k]; j = k; } } } } #endregion swHeuristica.Stop(); float yj = trainingSet.trainingArray[j].y; float alpha_j = problemSolution.alphaList[j]; // Error between the SVM output on the jth training unit and the true jth output float ej = trainingSet.errors[j]; // Save old alphas float oldAlpha_i = problemSolution.alphaList[i]; float oldAlpha_j = problemSolution.alphaList[j]; #region Compute lower and higher bounds of alpha_j float lowerBound; float higherBound; if (yi != yj) { lowerBound = Math.Max(0, alpha_j - alpha_i); higherBound = Math.Min(problemConfig.c, problemConfig.c + alpha_j - alpha_i); } else { lowerBound = Math.Max(0, alpha_j + alpha_i - problemConfig.c); higherBound = Math.Min(problemConfig.c, alpha_j + alpha_i); } #endregion // Nothing to adjust if we can't set any value between those bounds if (lowerBound == higherBound) { continue; } #region Compute eta float kernel_xi_xj; float kernel_xi_xi; float kernel_xj_xj; if (trainingSet.IsKernelCalculated[i]) { kernel_xi_xj = trainingSet.kernels[i][j]; } else if (trainingSet.IsKernelCalculated[j]) { kernel_xi_xj = trainingSet.kernels[j][i]; } else { kernel_xi_xj = calculateSingleKernel(trainingSet.trainingArray[i], trainingSet.trainingArray[j], problemSolution); //trainingSet.kernels[i][j]; } if (trainingSet.IsKernelCalculated[i]) { kernel_xi_xi = trainingSet.kernels[i][i]; } else { kernel_xi_xi = calculateSingleKernel(trainingSet.trainingArray[i], trainingSet.trainingArray[i], problemSolution); //trainingSet.kernels[i][i]; } if (trainingSet.IsKernelCalculated[j]) { kernel_xj_xj = trainingSet.kernels[j][j]; } else { kernel_xj_xj = calculateSingleKernel(trainingSet.trainingArray[j], trainingSet.trainingArray[j], problemSolution); //trainingSet.kernels[j][j]; } float eta = 2 * kernel_xi_xj - kernel_xi_xi - kernel_xj_xj; #endregion if (eta >= 0) { continue; } // Compute new alpha_j alpha_j = alpha_j - yj * (ei - ej) / eta; // Clip alpha_j if necessary if (alpha_j > higherBound) { alpha_j = higherBound; } else if (alpha_j < lowerBound) { alpha_j = lowerBound; } // If the changes are not big enough, just continue if (Math.Abs(oldAlpha_j - alpha_j) < MIN_ALPHA_CHANGE) { continue; } swComputeKernel.Start(); //Needs to compute lines K[i][] and K[j][] since the alphas will change if (OpenCLTemplate.CLCalc.CLAcceleration == OpenCLTemplate.CLCalc.CLAccelerationType.UsingCL) { CLComputeKernels(problemSolution, i); CLComputeKernels(problemSolution, j); } else { ComputeKernels(problemSolution, i); ComputeKernels(problemSolution, j); } swComputeKernel.Stop(); // Compute value for alpha_i alpha_i = alpha_i + yi * yj * (oldAlpha_j - alpha_j); // Compute b1, b2 and new b (threshold) float oldB = problemSolution.b; if (0 < alpha_i && alpha_i < problemConfig.c) { // b1 is enough in this case float b1 = problemSolution.b - ei - yi * (alpha_i - oldAlpha_i) * kernel_xi_xi - yj * (alpha_j - oldAlpha_j) * kernel_xi_xj; problemSolution.b = b1; } else if (0 < alpha_j && alpha_j < problemConfig.c) { // b2 is enough in this case float b2 = problemSolution.b - ej - yi * (alpha_i - oldAlpha_i) * kernel_xi_xj - yj * (alpha_j - oldAlpha_j) * kernel_xj_xj; problemSolution.b = b2; } else { // b is the average between b1 and b2 float b1 = problemSolution.b - ei - yi * (alpha_i - oldAlpha_i) * kernel_xi_xi - yj * (alpha_j - oldAlpha_j) * kernel_xi_xj; float b2 = problemSolution.b - ej - yi * (alpha_i - oldAlpha_i) * kernel_xi_xj - yj * (alpha_j - oldAlpha_j) * kernel_xj_xj; problemSolution.b = (b1 + b2) * 0.5f; } // Update the changed alphas in the solution problemSolution.alphaList[i] = alpha_i; problemSolution.alphaList[j] = alpha_j; // Update errors cache swUpdateError.Start(); if (OpenCLTemplate.CLCalc.CLAcceleration == OpenCLTemplate.CLCalc.CLAccelerationType.UsingCL) { CLupdateErrorsCache(trainingSet, problemSolution, oldAlpha_i, alpha_i, i, oldAlpha_j, alpha_j, j, oldB, problemSolution.b); } else { updateErrorsCache(trainingSet, problemSolution, oldAlpha_i, alpha_i, i, oldAlpha_j, alpha_j, j, oldB, problemSolution.b); } swUpdateError.Stop(); changedAlphas++; } } if (changedAlphas == 0) { passes++; } else { passes = 0; } } return(problemSolution); }
/// <summary>Trains current SVM with cross-validation, adjusting kernel parameter lambda and box parameter C. /// Returns best achieved efficiency.</summary> /// <param name="CrossValidationSet">Cross validation set</param> /// <param name="LambdaSet">Lambda set</param> /// <param name="CSet">C values set</param> public float TrainWithCrossValidation(TrainingSet CrossValidationSet, float[] LambdaSet, float[] CSet) { foreach (float _lambda in LambdaSet) { for (int i = 0; i < SVMs.Count; i++) SVMs[i].ProblemCfg.lambda = _lambda; foreach (float _c in CSet) { for (int i = 0; i < SVMs.Count; i++) { SVMs[i].ProblemCfg.c = _c; SVMs[i].Train(); } float performance = this.GetHitRate(CrossValidationSet); if (CrossValParams == null) CrossValParams = new float[3]; if (performance > CrossValParams[0]) { CrossValParams[0] = performance; CrossValParams[1] = _lambda; CrossValParams[2] = _c; } } } //Train with best parameters for (int i = 0; i < SVMs.Count; i++) { SVMs[i].ProblemCfg.lambda = CrossValParams[1]; SVMs[i].ProblemCfg.c = CrossValParams[2]; SVMs[i].Train(); } return CrossValParams[0]; }
/// <summary>Creates a new multiclass SVM using desired outputs from training set. Classifications -1.0f are negative for all sets</summary> /// <param name="TSet">Training set</param> public MultiClassSVM(TrainingSet TSet) { ProblemConfig cfg = new ProblemConfig(2.529822E-8f * (float)Math.Sqrt(TSet.getN), 127.922182f, 1e-3f, 1, ProblemConfig.KernelType.RBF); initMultiSVM(TSet, cfg); }
/// <summary>Creates a new multiclass SVM using desired outputs from training set. Classifications -1.0f are negative for all sets</summary> /// <param name="TSet">Training set</param> /// <param name="SVMCfg">Configuration parameters</param> /// <param name="PreCalibrate">Precalibrate RBF parameter lambda? This will ignore the given value</param> public MultiClassSVM(TrainingSet TSet, ProblemConfig SVMCfg, bool PreCalibrate) { initMultiSVM(TSet, SVMCfg, PreCalibrate); }