public static float calculateSingleKernel(TrainingUnit xi, TrainingUnit xj,SVM ProblemSolution)
{
ProblemConfig problemConfig = ProblemSolution.ProblemCfg;
// Vectors size check
//if (xi.getDimension() != xj.getDimension()) return 0;
// Linear: u'*v (inner product)
if (problemConfig.kernelType == ProblemConfig.KernelType.Linear)
{
float sum = 0;
for (int i = 0; i < xi.getDimension(); i++)
{
sum += xi.xVector[i] * xj.xVector[i];
}
return sum;
}
// Radial basis function: exp(-gamma*|u-v|^2)
if (problemConfig.kernelType == ProblemConfig.KernelType.RBF)
{
// Gamma is, by choice, 1 / (number of features).
float sum = 0, temp;
for (int i = 0; i < xi.getDimension(); i++)
{
temp = xi.xVector[i] - xj.xVector[i];
sum += temp * temp;
}
return (float)Math.Exp(-ProblemSolution.ProblemCfg.lambda * sum);
}
return 0;
}
public static float calculateSingleKernel(TrainingUnit xi, TrainingUnit xj, SVM ProblemSolution)
{
ProblemConfig problemConfig = ProblemSolution.ProblemCfg;
// Vectors size check
//if (xi.getDimension() != xj.getDimension()) return 0;
// Linear: u'*v (inner product)
if (problemConfig.kernelType == ProblemConfig.KernelType.Linear)
{
float sum = 0;
for (int i = 0; i < xi.getDimension(); i++)
{
sum += xi.xVector[i] * xj.xVector[i];
}
return(sum);
}
// Radial basis function: exp(-gamma*|u-v|^2)
if (problemConfig.kernelType == ProblemConfig.KernelType.RBF)
{
// Gamma is, by choice, 1 / (number of features).
float sum = 0, temp;
for (int i = 0; i < xi.getDimension(); i++)
{
temp = xi.xVector[i] - xj.xVector[i];
sum += temp * temp;
}
return((float)Math.Exp(-ProblemSolution.ProblemCfg.lambda * sum));
}
return(0);
}
/// <summary>Creates a new multiclass SVM using desired outputs from training set. Classifications -1.0f are negative for all sets</summary>
/// <param name="TSet">Training set</param>
/// <param name="SVMCfg">Configuration parameters</param>
private void initMultiSVM(TrainingSet TSet, ProblemConfig SVMCfg)
{
//Determines how many different classifications are there
Classifications = new List <float>();
foreach (TrainingUnit tu in TSet.trainingArray)
{
if (Classifications.IndexOf(tu.y) < 0 && tu.y != -1.0f)
{
Classifications.Add(tu.y);
}
}
//For each different possible classification, create a different SVM
SVMs = new List <SVM>();
foreach (float c in Classifications)
{
SVM svm = new SVM();
svm.TrainingSet = new TrainingSet();
svm.ProblemCfg = SVMCfg.Clone();
SVMs.Add(svm);
foreach (TrainingUnit tu in TSet.trainingArray)
{
TrainingUnit newTu = tu.Clone();
newTu.y = tu.y == c ? 1 : -1;
svm.TrainingSet.addTrainingUnit(newTu);
}
//Train svm
svm.PreCalibrateCfg(0.8f / (float)Math.Sqrt(svm.TrainingSet.getN), 0.3f / (float)Math.Sqrt(svm.TrainingSet.getN));
svm.Train();
svm.RemoveNonSupportVectors();
}
}
/// <summary>Classifies a sample within a given category even if all SVMs predict it doesn`t belong to any.</summary>
/// <param name="Sample">Sample to classify</param>
/// <param name="maxVal">Maximum classification value found</param>
public float Classify(TrainingUnit Sample, out float maxVal)
{
sample = Sample;
if (ClassificationValues == null)
{
ClassificationValues = new float[SVMs.Count];
}
for (int i = 0; i < SVMs.Count; i++)
{
Classify(i);
}
//Finds maximum value
maxVal = ClassificationValues[0];
float classification = Classifications[0];
for (int i = 1; i < ClassificationValues.Length; i++)
{
if (ClassificationValues[i] > maxVal)
{
maxVal = ClassificationValues[i];
classification = Classifications[i];
}
}
return(classification);
}
/// <summary>Classifies a training unit with a float. The bigger, the more positive the sample. Values greater than zero
/// are assumed to be positive samples</summary>
/// <param name="Sample">Sample to be classified</param>
public float ClassificationValue(TrainingUnit Sample)
{
if (OpenCLTemplate.CLCalc.CLAcceleration == OpenCLTemplate.CLCalc.CLAccelerationType.UsingCL)
{
return (CLpredictOutput(this, Sample));
}
else return (ProblemSolver.predictOutput(this, Sample));
}
/// <summary>Classifies a training unit with a float. The bigger, the more positive the sample. Values greater than zero
/// are assumed to be positive samples</summary>
/// <param name="Sample">Sample to be classified</param>
public float ClassificationValue(TrainingUnit Sample)
{
if (OpenCLTemplate.CLCalc.CLAcceleration == OpenCLTemplate.CLCalc.CLAccelerationType.UsingCL)
{
return(CLpredictOutput(this, Sample));
}
else
{
return(ProblemSolver.predictOutput(this, Sample));
}
}
/// <summary>Attempts to classify a sample within a given category. Returns -1 if no classification was achieved.</summary>
public float ClassifyWithRejection(TrainingUnit Sample)
{
float maxVal;
float resp = Classify(Sample, out maxVal);
if (maxVal >= 0)
{
return(resp);
}
else
{
return(-1.0f);
}
}
/// <summary>Extracts a cross validation set from a given set</summary>
/// <param name="Set">Set to extract cross validation from</param>
/// <param name="CrossValidationSetPercent">Percentage of elements to extract</param>
public static TrainingSet GetCrossValidationSet(TrainingSet Set, float CrossValidationSetPercent)
{
TrainingSet CrossValidationSet = new TrainingSet();
int nCrossSet = (int)(CrossValidationSetPercent * (float)Set.getN);
Random rnd = new Random();
for (int i = 0; i < nCrossSet; i++)
{
int ind = rnd.Next(0, Set.trainingArray.Count - 1);
TrainingUnit u = Set.trainingArray[ind];
Set.trainingArray.Remove(u);
CrossValidationSet.addTrainingUnit(u);
}
return(CrossValidationSet);
}
/// <summary>
/// Predicts the output of a single entry, given a previous problem, solution and correspondent training set
/// </summary>
/// <param name="problemSolution">Correspondent problem solution</param>
/// <param name="untrainedUnit">Input features from which the output will be predicted</param>
/// <returns>The y classification (true/false = positive/negative)</returns>
public static float CLpredictOutput(SVM problemSolution, TrainingUnit untrainedUnit)
{
TrainingSet trainingSet = problemSolution.TrainingSet;
ProblemConfig problemConfig = problemSolution.ProblemCfg;
#region Compute kernel
float[] K = new float[problemSolution.TrainingSet.getN];
CLCalc.Program.MemoryObject[] args = new CLCalc.Program.MemoryObject[]
{
problemSolution.CLTrainingFeatures,
problemSolution.CLXVecLen,
problemSolution.CLSample,
problemSolution.CLKernelValues,
problemSolution.CLLambda
};
for (int j = 0; j < untrainedUnit.xVector.Length; j++)
{
problemSolution.HostSample[j] = untrainedUnit.xVector[j];
}
problemSolution.CLSample.WriteToDevice(problemSolution.HostSample);
lock (CLResource)
{
kernelComputeKernelRBF.Execute(args, problemSolution.TrainingSet.getN);
problemSolution.CLKernelValues.ReadFromDeviceTo(K);
}
#endregion
// F(x) = sum + b
// sum = summation of alpha_i * y_i * kernel(untrained unit, i) for all i in the training set
float sum = 0;
for (int i = 0; i < trainingSet.getN; i++)
{
if (trainingSet.trainingArray[i].y > 0)
{
sum += problemSolution.alphaList[i] * K[i];
}
else
{
sum -= problemSolution.alphaList[i] * K[i];
}
}
return(sum + problemSolution.b);
}
/// <summary>Adds a new training unit to the set</summary>
/// <param name="newTrainingUnit">New training unit to add</param>
public void addTrainingUnit(TrainingUnit newTrainingUnit)
{
if (p != 0)
{
if (p == newTrainingUnit.getDimension())
{
trainingArray.Add(newTrainingUnit);
}
else
{
// Invalid entry, not equal in size to the others training units
// Do nothing
}
}
else // The first training set is being added
{
p = newTrainingUnit.getDimension();
trainingArray.Add(newTrainingUnit);
}
}
/// <summary>Adds a new training unit to the set</summary>
/// <param name="newTrainingUnit">New training unit to add</param>
public void addTrainingUnit(TrainingUnit newTrainingUnit)
{
if (p != 0)
{
if (p == newTrainingUnit.getDimension())
{
trainingArray.Add(newTrainingUnit);
}
else
{
// Invalid entry, not equal in size to the others training units
// Do nothing
}
}
else // The first training set is being added
{
p = newTrainingUnit.getDimension();
trainingArray.Add(newTrainingUnit);
}
}
/// <summary>Classifies a sample within a given category even if all SVMs predict it doesn`t belong to any.</summary>
/// <param name="Sample">Sample to classify</param>
/// <param name="maxVal">Maximum classification value found</param>
public float Classify(TrainingUnit Sample, out float maxVal)
{
sample = Sample;
if (ClassificationValues == null) ClassificationValues = new float[SVMs.Count];
for (int i = 0; i < SVMs.Count; i++) Classify(i);
//Finds maximum value
maxVal = ClassificationValues[0];
float classification = Classifications[0];
for (int i = 1; i < ClassificationValues.Length; i++)
{
if (ClassificationValues[i] > maxVal)
{
maxVal = ClassificationValues[i];
classification = Classifications[i];
}
}
return classification;
}
/// <summary>Adds a new self training example</summary>
public void AddSelfTraining(int[] sbFrames, int faceIndex, Bitmap bmp)
{
if (SelfTSet == null)
{
SelfTSet = new TrainingSet();
}
float[] subF = new float[(sbFrames.Length / 3) * 364];
ExtractFeatures(sbFrames, subF, bmp);
for (int i = 0; i < sbFrames.Length / 3; i++)
{
float[] x = new float[364];
for (int k = 0; k < 364; k++)
{
x[k] = subF[k + i * 364];
}
TrainingUnit tu = new TrainingUnit(x, i == faceIndex ? 1.0f : -1.0f);
SelfTSet.addTrainingUnit(tu);
}
}
/// <summary>
/// Predicts the output of a single entry, given a previous problem, solution and correspondent training set
/// </summary>
/// <param name="problemSolution">Correspondent problem solution</param>
/// <param name="untrainedUnit">Input features from which the output will be predicted</param>
/// <returns>The y classification (true/false = positive/negative)</returns>
public static float predictOutput(SVM problemSolution, TrainingUnit untrainedUnit)
{
TrainingSet trainingSet = problemSolution.TrainingSet;
ProblemConfig problemConfig = problemSolution.ProblemCfg;
// F(x) = sum + b
// sum = summation of alpha_i * y_i * kernel(untrained unit, i) for all i in the training set
float sum = 0;
for (int i = 0; i < trainingSet.getN; i++)
{
if (trainingSet.trainingArray[i].y > 0)
{
sum += problemSolution.alphaList[i] * calculateSingleKernel(trainingSet.trainingArray[i], untrainedUnit, problemSolution);
}
else
{
sum -= problemSolution.alphaList[i] * calculateSingleKernel(trainingSet.trainingArray[i], untrainedUnit, problemSolution);
}
}
return(sum + problemSolution.b);
}
/// <summary>Attempts to pre-calibrate configuration parameters.
/// Finds an alpha that enhances similarities between positive examples
/// and reduces similarities between positive and negative examples.
/// Assumes that decreasing lambda increases kernel match.
/// </summary>
/// <param name="tolPositive">Positive kernels average should be greater than tolPositive</param>
/// <param name="tolNegative">Negative kernels average should be lesser than tolNegative</param>
public void PreCalibrateCfg(float tolPositive, float tolNegative)
{
#region Checks if there are positive and negative examples
bool posSamples = false; bool negSamples = false;
for (int i = 0; i < TrainingSet.trainingArray.Count; i++)
{
if (TrainingSet.trainingArray[i].y > 0)
{
posSamples = true;
}
if (TrainingSet.trainingArray[i].y < 0)
{
negSamples = true;
}
if (posSamples && negSamples)
{
i = TrainingSet.trainingArray.Count;
}
}
if ((!posSamples) || (!negSamples))
{
throw new Exception("Training set must contain positive and negative samples");
}
#endregion
Random rnd = new Random();
int nSet = (int)(20 * Math.Log(TrainingSet.getN, 2));
TrainingSet PositiveExamples1 = new TrainingSet();
TrainingSet PositiveExamples2 = new TrainingSet();
TrainingSet NegativeExamples = new TrainingSet();
//Kernel average for positive and negative samples
float positiveAvg = 0, negativeAvg = 0;
float invN = 1 / (float)nSet;
int count = 0;
float bestLambda = ProblemCfg.lambda;
float maxPosNegAvg = -1.0f;
while ((positiveAvg <= tolPositive || negativeAvg >= tolNegative) && count < nSet)
{
//Populates training sets
PositiveExamples1.trainingArray.Clear();
PositiveExamples2.trainingArray.Clear();
NegativeExamples.trainingArray.Clear();
while (PositiveExamples1.getN < nSet || PositiveExamples2.getN < nSet || NegativeExamples.getN < nSet)
{
TrainingUnit tu = TrainingSet.trainingArray[rnd.Next(TrainingSet.trainingArray.Count - 1)];
if (tu.y > 0 && PositiveExamples1.getN < nSet)
{
PositiveExamples1.addTrainingUnit(tu);
}
else if (tu.y > 0 && PositiveExamples2.getN < nSet)
{
PositiveExamples2.addTrainingUnit(tu);
}
if (tu.y < 0 && NegativeExamples.getN < nSet)
{
NegativeExamples.addTrainingUnit(tu);
}
}
count++;
positiveAvg = 0;
negativeAvg = 0;
for (int i = 0; i < nSet; i++)
{
positiveAvg += ProblemSolver.calculateSingleKernel(PositiveExamples1.trainingArray[i], PositiveExamples2.trainingArray[i], this);
negativeAvg += ProblemSolver.calculateSingleKernel(PositiveExamples1.trainingArray[i], NegativeExamples.trainingArray[i], this);
}
positiveAvg *= invN;
negativeAvg *= invN;
if (maxPosNegAvg < positiveAvg - negativeAvg)
{
bestLambda = ProblemCfg.lambda;
maxPosNegAvg = positiveAvg - negativeAvg;
}
//Desired: positiveAvg=1, negativeAvg = 0
if (positiveAvg <= tolPositive)
{
this.ProblemCfg.lambda *= 0.15f;
}
else if (negativeAvg >= tolNegative)
{
this.ProblemCfg.lambda *= 1.2f;
}
}
ProblemCfg.lambda = bestLambda;
}
/// <summary>Attempts to classify a sample within a given category. Returns -1 if no classification was achieved.</summary>
public float ClassifyWithRejection(TrainingUnit Sample)
{
float maxVal;
float resp = Classify(Sample, out maxVal);
if (maxVal >= 0) return resp;
else return -1.0f;
}
/// <summary>Adds a new self training example</summary>
public void AddSelfTraining(int[] sbFrames, int faceIndex, Bitmap bmp)
{
if (SelfTSet == null) SelfTSet = new TrainingSet();
float[] subF = new float[(sbFrames.Length / 3) * 364];
ExtractFeatures(sbFrames, subF, bmp);
for (int i = 0; i < sbFrames.Length / 3; i++)
{
float[] x = new float[364];
for (int k = 0; k < 364; k++) x[k] = subF[k + i * 364];
TrainingUnit tu = new TrainingUnit(x, i == faceIndex ? 1.0f : -1.0f);
SelfTSet.addTrainingUnit(tu);
}
}
/// <summary>Classifies a training unit as positive or negative (true or false)</summary>
/// <param name="Sample">Sample to be classified</param>
public bool Classify(TrainingUnit Sample)
{
if (OpenCLTemplate.CLCalc.CLAcceleration == OpenCLTemplate.CLCalc.CLAccelerationType.UsingCL)
{
return (CLpredictOutput(this, Sample)>=0);
}
else return (ProblemSolver.predictOutput(this, Sample)>=0);
}
/// <summary>
/// Predicts the output of a single entry, given a previous problem, solution and correspondent training set
/// </summary>
/// <param name="problemSolution">Correspondent problem solution</param>
/// <param name="untrainedUnit">Input features from which the output will be predicted</param>
/// <returns>The y classification (true/false = positive/negative)</returns>
public static float CLpredictOutput(SVM problemSolution, TrainingUnit untrainedUnit)
{
TrainingSet trainingSet = problemSolution.TrainingSet;
ProblemConfig problemConfig = problemSolution.ProblemCfg;
#region Compute kernel
float[] K = new float[problemSolution.TrainingSet.getN];
CLCalc.Program.MemoryObject[] args = new CLCalc.Program.MemoryObject[]
{
problemSolution.CLTrainingFeatures,
problemSolution.CLXVecLen,
problemSolution.CLSample,
problemSolution.CLKernelValues,
problemSolution.CLLambda
};
for (int j = 0; j < untrainedUnit.xVector.Length; j++)
problemSolution.HostSample[j] = untrainedUnit.xVector[j];
problemSolution.CLSample.WriteToDevice(problemSolution.HostSample);
lock (CLResource)
{
kernelComputeKernelRBF.Execute(args, problemSolution.TrainingSet.getN);
problemSolution.CLKernelValues.ReadFromDeviceTo(K);
}
#endregion
// F(x) = sum + b
// sum = summation of alpha_i * y_i * kernel(untrained unit, i) for all i in the training set
float sum = 0;
for (int i = 0; i < trainingSet.getN; i++)
{
if (trainingSet.trainingArray[i].y > 0)
sum += problemSolution.alphaList[i] * K[i];
else
sum -= problemSolution.alphaList[i] * K[i];
}
return sum + problemSolution.b;
}
/// <summary>
/// Predicts the output of a single entry, given a previous problem, solution and correspondent training set
/// </summary>
/// <param name="problemSolution">Correspondent problem solution</param>
/// <param name="untrainedUnit">Input features from which the output will be predicted</param>
/// <returns>The y classification (true/false = positive/negative)</returns>
public static float predictOutput(SVM problemSolution, TrainingUnit untrainedUnit)
{
TrainingSet trainingSet = problemSolution.TrainingSet;
ProblemConfig problemConfig = problemSolution.ProblemCfg;
// F(x) = sum + b
// sum = summation of alpha_i * y_i * kernel(untrained unit, i) for all i in the training set
float sum = 0;
for (int i = 0; i < trainingSet.getN; i++)
{
if (trainingSet.trainingArray[i].y > 0)
sum += problemSolution.alphaList[i] * calculateSingleKernel(trainingSet.trainingArray[i], untrainedUnit, problemSolution);
else
sum -= problemSolution.alphaList[i] * calculateSingleKernel(trainingSet.trainingArray[i], untrainedUnit, problemSolution);
}
return sum + problemSolution.b;
}
/// <summary>Trains current SVM with cross-validation, adjusting kernel parameter lambda and box parameter C. Returns best performance so far</summary>
/// <param name="CrossValidationSetPercent">Percentage of training examples that should be used as cross validation set</param>
/// <param name="lambdaSet">Values of lambda to try</param>
/// <param name="CSet">Values of c to try</param>
public float TrainWithCrossValidation(float CrossValidationSetPercent, float[] lambdaSet, float[] CSet)
{
if (alphaList == null || alphaList.Count != TrainingSet.getN)
{
//Problem changed, previous values dont make sense
initializeWithZeros();
CrossValParams = null;
}
#region Constructs cross validation set
TrainingSet CrossValidationSet = new TrainingSet();
int nCrossSet = (int)(CrossValidationSetPercent * (float)this.TrainingSet.getN);
Random rnd = new Random();
for (int i = 0; i < nCrossSet; i++)
{
int ind = rnd.Next(0, this.TrainingSet.trainingArray.Count - 1);
TrainingUnit u = this.TrainingSet.trainingArray[ind];
this.TrainingSet.trainingArray.Remove(u);
CrossValidationSet.addTrainingUnit(u);
}
#endregion
#region Loops through lambdas and Cs and finds maximum crossvalidation
foreach (float _lambda in lambdaSet)
{
this.ProblemCfg.lambda = _lambda;
this.initializeWithZeros();
PreComputeKernels();
foreach (float _c in CSet)
{
this.ProblemCfg.c = _c;
//ProblemSolver.solveSMOStartingFromPreviousSolution(this);
ProblemSolver.solveSMOStartingFromZero(this);
float performance = this.GetHitRate(CrossValidationSet);
if (CrossValParams == null)
{
CrossValParams = new float[3];
}
if (performance > CrossValParams[0])
{
CrossValParams[0] = performance;
CrossValParams[1] = _lambda;
CrossValParams[2] = _c;
}
}
}
#endregion
#region Trains with best parameters so far
this.ProblemCfg.lambda = CrossValParams[1];
this.ProblemCfg.c = CrossValParams[2];
this.Train();
#endregion
return(CrossValParams[0]);
}