public static float calculateSingleKernel(TrainingUnit xi, TrainingUnit xj,SVM ProblemSolution) { ProblemConfig problemConfig = ProblemSolution.ProblemCfg; // Vectors size check //if (xi.getDimension() != xj.getDimension()) return 0; // Linear: u'*v (inner product) if (problemConfig.kernelType == ProblemConfig.KernelType.Linear) { float sum = 0; for (int i = 0; i < xi.getDimension(); i++) { sum += xi.xVector[i] * xj.xVector[i]; } return sum; } // Radial basis function: exp(-gamma*|u-v|^2) if (problemConfig.kernelType == ProblemConfig.KernelType.RBF) { // Gamma is, by choice, 1 / (number of features). float sum = 0, temp; for (int i = 0; i < xi.getDimension(); i++) { temp = xi.xVector[i] - xj.xVector[i]; sum += temp * temp; } return (float)Math.Exp(-ProblemSolution.ProblemCfg.lambda * sum); } return 0; }
public static float calculateSingleKernel(TrainingUnit xi, TrainingUnit xj, SVM ProblemSolution) { ProblemConfig problemConfig = ProblemSolution.ProblemCfg; // Vectors size check //if (xi.getDimension() != xj.getDimension()) return 0; // Linear: u'*v (inner product) if (problemConfig.kernelType == ProblemConfig.KernelType.Linear) { float sum = 0; for (int i = 0; i < xi.getDimension(); i++) { sum += xi.xVector[i] * xj.xVector[i]; } return(sum); } // Radial basis function: exp(-gamma*|u-v|^2) if (problemConfig.kernelType == ProblemConfig.KernelType.RBF) { // Gamma is, by choice, 1 / (number of features). float sum = 0, temp; for (int i = 0; i < xi.getDimension(); i++) { temp = xi.xVector[i] - xj.xVector[i]; sum += temp * temp; } return((float)Math.Exp(-ProblemSolution.ProblemCfg.lambda * sum)); } return(0); }
/// <summary>Creates a new multiclass SVM using desired outputs from training set. Classifications -1.0f are negative for all sets</summary> /// <param name="TSet">Training set</param> /// <param name="SVMCfg">Configuration parameters</param> private void initMultiSVM(TrainingSet TSet, ProblemConfig SVMCfg) { //Determines how many different classifications are there Classifications = new List <float>(); foreach (TrainingUnit tu in TSet.trainingArray) { if (Classifications.IndexOf(tu.y) < 0 && tu.y != -1.0f) { Classifications.Add(tu.y); } } //For each different possible classification, create a different SVM SVMs = new List <SVM>(); foreach (float c in Classifications) { SVM svm = new SVM(); svm.TrainingSet = new TrainingSet(); svm.ProblemCfg = SVMCfg.Clone(); SVMs.Add(svm); foreach (TrainingUnit tu in TSet.trainingArray) { TrainingUnit newTu = tu.Clone(); newTu.y = tu.y == c ? 1 : -1; svm.TrainingSet.addTrainingUnit(newTu); } //Train svm svm.PreCalibrateCfg(0.8f / (float)Math.Sqrt(svm.TrainingSet.getN), 0.3f / (float)Math.Sqrt(svm.TrainingSet.getN)); svm.Train(); svm.RemoveNonSupportVectors(); } }
/// <summary>Classifies a sample within a given category even if all SVMs predict it doesn`t belong to any.</summary> /// <param name="Sample">Sample to classify</param> /// <param name="maxVal">Maximum classification value found</param> public float Classify(TrainingUnit Sample, out float maxVal) { sample = Sample; if (ClassificationValues == null) { ClassificationValues = new float[SVMs.Count]; } for (int i = 0; i < SVMs.Count; i++) { Classify(i); } //Finds maximum value maxVal = ClassificationValues[0]; float classification = Classifications[0]; for (int i = 1; i < ClassificationValues.Length; i++) { if (ClassificationValues[i] > maxVal) { maxVal = ClassificationValues[i]; classification = Classifications[i]; } } return(classification); }
/// <summary>Classifies a training unit with a float. The bigger, the more positive the sample. Values greater than zero /// are assumed to be positive samples</summary> /// <param name="Sample">Sample to be classified</param> public float ClassificationValue(TrainingUnit Sample) { if (OpenCLTemplate.CLCalc.CLAcceleration == OpenCLTemplate.CLCalc.CLAccelerationType.UsingCL) { return (CLpredictOutput(this, Sample)); } else return (ProblemSolver.predictOutput(this, Sample)); }
/// <summary>Classifies a training unit with a float. The bigger, the more positive the sample. Values greater than zero /// are assumed to be positive samples</summary> /// <param name="Sample">Sample to be classified</param> public float ClassificationValue(TrainingUnit Sample) { if (OpenCLTemplate.CLCalc.CLAcceleration == OpenCLTemplate.CLCalc.CLAccelerationType.UsingCL) { return(CLpredictOutput(this, Sample)); } else { return(ProblemSolver.predictOutput(this, Sample)); } }
/// <summary>Attempts to classify a sample within a given category. Returns -1 if no classification was achieved.</summary> public float ClassifyWithRejection(TrainingUnit Sample) { float maxVal; float resp = Classify(Sample, out maxVal); if (maxVal >= 0) { return(resp); } else { return(-1.0f); } }
/// <summary>Extracts a cross validation set from a given set</summary> /// <param name="Set">Set to extract cross validation from</param> /// <param name="CrossValidationSetPercent">Percentage of elements to extract</param> public static TrainingSet GetCrossValidationSet(TrainingSet Set, float CrossValidationSetPercent) { TrainingSet CrossValidationSet = new TrainingSet(); int nCrossSet = (int)(CrossValidationSetPercent * (float)Set.getN); Random rnd = new Random(); for (int i = 0; i < nCrossSet; i++) { int ind = rnd.Next(0, Set.trainingArray.Count - 1); TrainingUnit u = Set.trainingArray[ind]; Set.trainingArray.Remove(u); CrossValidationSet.addTrainingUnit(u); } return(CrossValidationSet); }
/// <summary> /// Predicts the output of a single entry, given a previous problem, solution and correspondent training set /// </summary> /// <param name="problemSolution">Correspondent problem solution</param> /// <param name="untrainedUnit">Input features from which the output will be predicted</param> /// <returns>The y classification (true/false = positive/negative)</returns> public static float CLpredictOutput(SVM problemSolution, TrainingUnit untrainedUnit) { TrainingSet trainingSet = problemSolution.TrainingSet; ProblemConfig problemConfig = problemSolution.ProblemCfg; #region Compute kernel float[] K = new float[problemSolution.TrainingSet.getN]; CLCalc.Program.MemoryObject[] args = new CLCalc.Program.MemoryObject[] { problemSolution.CLTrainingFeatures, problemSolution.CLXVecLen, problemSolution.CLSample, problemSolution.CLKernelValues, problemSolution.CLLambda }; for (int j = 0; j < untrainedUnit.xVector.Length; j++) { problemSolution.HostSample[j] = untrainedUnit.xVector[j]; } problemSolution.CLSample.WriteToDevice(problemSolution.HostSample); lock (CLResource) { kernelComputeKernelRBF.Execute(args, problemSolution.TrainingSet.getN); problemSolution.CLKernelValues.ReadFromDeviceTo(K); } #endregion // F(x) = sum + b // sum = summation of alpha_i * y_i * kernel(untrained unit, i) for all i in the training set float sum = 0; for (int i = 0; i < trainingSet.getN; i++) { if (trainingSet.trainingArray[i].y > 0) { sum += problemSolution.alphaList[i] * K[i]; } else { sum -= problemSolution.alphaList[i] * K[i]; } } return(sum + problemSolution.b); }
/// <summary>Adds a new training unit to the set</summary> /// <param name="newTrainingUnit">New training unit to add</param> public void addTrainingUnit(TrainingUnit newTrainingUnit) { if (p != 0) { if (p == newTrainingUnit.getDimension()) { trainingArray.Add(newTrainingUnit); } else { // Invalid entry, not equal in size to the others training units // Do nothing } } else // The first training set is being added { p = newTrainingUnit.getDimension(); trainingArray.Add(newTrainingUnit); } }
/// <summary>Adds a new training unit to the set</summary> /// <param name="newTrainingUnit">New training unit to add</param> public void addTrainingUnit(TrainingUnit newTrainingUnit) { if (p != 0) { if (p == newTrainingUnit.getDimension()) { trainingArray.Add(newTrainingUnit); } else { // Invalid entry, not equal in size to the others training units // Do nothing } } else // The first training set is being added { p = newTrainingUnit.getDimension(); trainingArray.Add(newTrainingUnit); } }
/// <summary>Classifies a sample within a given category even if all SVMs predict it doesn`t belong to any.</summary> /// <param name="Sample">Sample to classify</param> /// <param name="maxVal">Maximum classification value found</param> public float Classify(TrainingUnit Sample, out float maxVal) { sample = Sample; if (ClassificationValues == null) ClassificationValues = new float[SVMs.Count]; for (int i = 0; i < SVMs.Count; i++) Classify(i); //Finds maximum value maxVal = ClassificationValues[0]; float classification = Classifications[0]; for (int i = 1; i < ClassificationValues.Length; i++) { if (ClassificationValues[i] > maxVal) { maxVal = ClassificationValues[i]; classification = Classifications[i]; } } return classification; }
/// <summary>Adds a new self training example</summary> public void AddSelfTraining(int[] sbFrames, int faceIndex, Bitmap bmp) { if (SelfTSet == null) { SelfTSet = new TrainingSet(); } float[] subF = new float[(sbFrames.Length / 3) * 364]; ExtractFeatures(sbFrames, subF, bmp); for (int i = 0; i < sbFrames.Length / 3; i++) { float[] x = new float[364]; for (int k = 0; k < 364; k++) { x[k] = subF[k + i * 364]; } TrainingUnit tu = new TrainingUnit(x, i == faceIndex ? 1.0f : -1.0f); SelfTSet.addTrainingUnit(tu); } }
/// <summary> /// Predicts the output of a single entry, given a previous problem, solution and correspondent training set /// </summary> /// <param name="problemSolution">Correspondent problem solution</param> /// <param name="untrainedUnit">Input features from which the output will be predicted</param> /// <returns>The y classification (true/false = positive/negative)</returns> public static float predictOutput(SVM problemSolution, TrainingUnit untrainedUnit) { TrainingSet trainingSet = problemSolution.TrainingSet; ProblemConfig problemConfig = problemSolution.ProblemCfg; // F(x) = sum + b // sum = summation of alpha_i * y_i * kernel(untrained unit, i) for all i in the training set float sum = 0; for (int i = 0; i < trainingSet.getN; i++) { if (trainingSet.trainingArray[i].y > 0) { sum += problemSolution.alphaList[i] * calculateSingleKernel(trainingSet.trainingArray[i], untrainedUnit, problemSolution); } else { sum -= problemSolution.alphaList[i] * calculateSingleKernel(trainingSet.trainingArray[i], untrainedUnit, problemSolution); } } return(sum + problemSolution.b); }
/// <summary>Attempts to pre-calibrate configuration parameters. /// Finds an alpha that enhances similarities between positive examples /// and reduces similarities between positive and negative examples. /// Assumes that decreasing lambda increases kernel match. /// </summary> /// <param name="tolPositive">Positive kernels average should be greater than tolPositive</param> /// <param name="tolNegative">Negative kernels average should be lesser than tolNegative</param> public void PreCalibrateCfg(float tolPositive, float tolNegative) { #region Checks if there are positive and negative examples bool posSamples = false; bool negSamples = false; for (int i = 0; i < TrainingSet.trainingArray.Count; i++) { if (TrainingSet.trainingArray[i].y > 0) { posSamples = true; } if (TrainingSet.trainingArray[i].y < 0) { negSamples = true; } if (posSamples && negSamples) { i = TrainingSet.trainingArray.Count; } } if ((!posSamples) || (!negSamples)) { throw new Exception("Training set must contain positive and negative samples"); } #endregion Random rnd = new Random(); int nSet = (int)(20 * Math.Log(TrainingSet.getN, 2)); TrainingSet PositiveExamples1 = new TrainingSet(); TrainingSet PositiveExamples2 = new TrainingSet(); TrainingSet NegativeExamples = new TrainingSet(); //Kernel average for positive and negative samples float positiveAvg = 0, negativeAvg = 0; float invN = 1 / (float)nSet; int count = 0; float bestLambda = ProblemCfg.lambda; float maxPosNegAvg = -1.0f; while ((positiveAvg <= tolPositive || negativeAvg >= tolNegative) && count < nSet) { //Populates training sets PositiveExamples1.trainingArray.Clear(); PositiveExamples2.trainingArray.Clear(); NegativeExamples.trainingArray.Clear(); while (PositiveExamples1.getN < nSet || PositiveExamples2.getN < nSet || NegativeExamples.getN < nSet) { TrainingUnit tu = TrainingSet.trainingArray[rnd.Next(TrainingSet.trainingArray.Count - 1)]; if (tu.y > 0 && PositiveExamples1.getN < nSet) { PositiveExamples1.addTrainingUnit(tu); } else if (tu.y > 0 && PositiveExamples2.getN < nSet) { PositiveExamples2.addTrainingUnit(tu); } if (tu.y < 0 && NegativeExamples.getN < nSet) { NegativeExamples.addTrainingUnit(tu); } } count++; positiveAvg = 0; negativeAvg = 0; for (int i = 0; i < nSet; i++) { positiveAvg += ProblemSolver.calculateSingleKernel(PositiveExamples1.trainingArray[i], PositiveExamples2.trainingArray[i], this); negativeAvg += ProblemSolver.calculateSingleKernel(PositiveExamples1.trainingArray[i], NegativeExamples.trainingArray[i], this); } positiveAvg *= invN; negativeAvg *= invN; if (maxPosNegAvg < positiveAvg - negativeAvg) { bestLambda = ProblemCfg.lambda; maxPosNegAvg = positiveAvg - negativeAvg; } //Desired: positiveAvg=1, negativeAvg = 0 if (positiveAvg <= tolPositive) { this.ProblemCfg.lambda *= 0.15f; } else if (negativeAvg >= tolNegative) { this.ProblemCfg.lambda *= 1.2f; } } ProblemCfg.lambda = bestLambda; }
/// <summary>Attempts to classify a sample within a given category. Returns -1 if no classification was achieved.</summary> public float ClassifyWithRejection(TrainingUnit Sample) { float maxVal; float resp = Classify(Sample, out maxVal); if (maxVal >= 0) return resp; else return -1.0f; }
/// <summary>Adds a new self training example</summary> public void AddSelfTraining(int[] sbFrames, int faceIndex, Bitmap bmp) { if (SelfTSet == null) SelfTSet = new TrainingSet(); float[] subF = new float[(sbFrames.Length / 3) * 364]; ExtractFeatures(sbFrames, subF, bmp); for (int i = 0; i < sbFrames.Length / 3; i++) { float[] x = new float[364]; for (int k = 0; k < 364; k++) x[k] = subF[k + i * 364]; TrainingUnit tu = new TrainingUnit(x, i == faceIndex ? 1.0f : -1.0f); SelfTSet.addTrainingUnit(tu); } }
/// <summary>Classifies a training unit as positive or negative (true or false)</summary> /// <param name="Sample">Sample to be classified</param> public bool Classify(TrainingUnit Sample) { if (OpenCLTemplate.CLCalc.CLAcceleration == OpenCLTemplate.CLCalc.CLAccelerationType.UsingCL) { return (CLpredictOutput(this, Sample)>=0); } else return (ProblemSolver.predictOutput(this, Sample)>=0); }
/// <summary> /// Predicts the output of a single entry, given a previous problem, solution and correspondent training set /// </summary> /// <param name="problemSolution">Correspondent problem solution</param> /// <param name="untrainedUnit">Input features from which the output will be predicted</param> /// <returns>The y classification (true/false = positive/negative)</returns> public static float CLpredictOutput(SVM problemSolution, TrainingUnit untrainedUnit) { TrainingSet trainingSet = problemSolution.TrainingSet; ProblemConfig problemConfig = problemSolution.ProblemCfg; #region Compute kernel float[] K = new float[problemSolution.TrainingSet.getN]; CLCalc.Program.MemoryObject[] args = new CLCalc.Program.MemoryObject[] { problemSolution.CLTrainingFeatures, problemSolution.CLXVecLen, problemSolution.CLSample, problemSolution.CLKernelValues, problemSolution.CLLambda }; for (int j = 0; j < untrainedUnit.xVector.Length; j++) problemSolution.HostSample[j] = untrainedUnit.xVector[j]; problemSolution.CLSample.WriteToDevice(problemSolution.HostSample); lock (CLResource) { kernelComputeKernelRBF.Execute(args, problemSolution.TrainingSet.getN); problemSolution.CLKernelValues.ReadFromDeviceTo(K); } #endregion // F(x) = sum + b // sum = summation of alpha_i * y_i * kernel(untrained unit, i) for all i in the training set float sum = 0; for (int i = 0; i < trainingSet.getN; i++) { if (trainingSet.trainingArray[i].y > 0) sum += problemSolution.alphaList[i] * K[i]; else sum -= problemSolution.alphaList[i] * K[i]; } return sum + problemSolution.b; }
/// <summary> /// Predicts the output of a single entry, given a previous problem, solution and correspondent training set /// </summary> /// <param name="problemSolution">Correspondent problem solution</param> /// <param name="untrainedUnit">Input features from which the output will be predicted</param> /// <returns>The y classification (true/false = positive/negative)</returns> public static float predictOutput(SVM problemSolution, TrainingUnit untrainedUnit) { TrainingSet trainingSet = problemSolution.TrainingSet; ProblemConfig problemConfig = problemSolution.ProblemCfg; // F(x) = sum + b // sum = summation of alpha_i * y_i * kernel(untrained unit, i) for all i in the training set float sum = 0; for (int i = 0; i < trainingSet.getN; i++) { if (trainingSet.trainingArray[i].y > 0) sum += problemSolution.alphaList[i] * calculateSingleKernel(trainingSet.trainingArray[i], untrainedUnit, problemSolution); else sum -= problemSolution.alphaList[i] * calculateSingleKernel(trainingSet.trainingArray[i], untrainedUnit, problemSolution); } return sum + problemSolution.b; }
/// <summary>Trains current SVM with cross-validation, adjusting kernel parameter lambda and box parameter C. Returns best performance so far</summary> /// <param name="CrossValidationSetPercent">Percentage of training examples that should be used as cross validation set</param> /// <param name="lambdaSet">Values of lambda to try</param> /// <param name="CSet">Values of c to try</param> public float TrainWithCrossValidation(float CrossValidationSetPercent, float[] lambdaSet, float[] CSet) { if (alphaList == null || alphaList.Count != TrainingSet.getN) { //Problem changed, previous values dont make sense initializeWithZeros(); CrossValParams = null; } #region Constructs cross validation set TrainingSet CrossValidationSet = new TrainingSet(); int nCrossSet = (int)(CrossValidationSetPercent * (float)this.TrainingSet.getN); Random rnd = new Random(); for (int i = 0; i < nCrossSet; i++) { int ind = rnd.Next(0, this.TrainingSet.trainingArray.Count - 1); TrainingUnit u = this.TrainingSet.trainingArray[ind]; this.TrainingSet.trainingArray.Remove(u); CrossValidationSet.addTrainingUnit(u); } #endregion #region Loops through lambdas and Cs and finds maximum crossvalidation foreach (float _lambda in lambdaSet) { this.ProblemCfg.lambda = _lambda; this.initializeWithZeros(); PreComputeKernels(); foreach (float _c in CSet) { this.ProblemCfg.c = _c; //ProblemSolver.solveSMOStartingFromPreviousSolution(this); ProblemSolver.solveSMOStartingFromZero(this); float performance = this.GetHitRate(CrossValidationSet); if (CrossValParams == null) { CrossValParams = new float[3]; } if (performance > CrossValParams[0]) { CrossValParams[0] = performance; CrossValParams[1] = _lambda; CrossValParams[2] = _c; } } } #endregion #region Trains with best parameters so far this.ProblemCfg.lambda = CrossValParams[1]; this.ProblemCfg.c = CrossValParams[2]; this.Train(); #endregion return(CrossValParams[0]); }