/// <summary>Line search algorithm</summary>
/// <param name="f">Function to be searched</param>
/// <param name="x">Current point</param>
/// <param name="deltaX">Search direction</param>
/// <param name="gradF">Gradient of F</param>
/// <param name="temp">Temporary vector to store current direction</param>
/// <param name="HessF">Hessian of F</param>
private static void BackTrack(C2Function f, floatLinalg.floatVector x, floatLinalg.floatVector deltaX,
floatLinalg.floatVector gradF, ref floatLinalg.floatVector temp, ref floatLinalg.floatSymPosDefMatrix HessF)
{
float t = 1;
float alphagradFdeltaX = floatLinalg.BLAS.Dot(gradF, deltaX, ref temp) * Config.alpha;
floatLinalg.floatVector dum = null;
float valF = f(x, false, ref dum, ref HessF);
//temp holds f(x+t*deltax)
floatLinalg.BLAS.LinearCombination(1, x, t, deltaX, ref temp);
float valFupdt = f(temp, false, ref dum, ref HessF);
while ((t > 0) && (float.IsNaN(valFupdt) || valFupdt > valF + t * alphagradFdeltaX))
{
floatLinalg.BLAS.LinearCombination(1, x, t, deltaX, ref temp);
valFupdt = f(temp, false, ref dum, ref HessF);
t *= Config.beta;
}
if (t > 0)
{
//Consolidates deltaX
floatLinalg.BLAS.LinearCombination(1, temp, -1, x, ref deltaX);
//Updates x
floatLinalg.BLAS.CopyVector(temp, x);
}
}
/// <summary>Computes the unconstrained minimum of function F in x0.Length dimensions</summary>
/// <param name="CLx">Solution x and initial guess</param>
/// <param name="F">Function to be minimized</param>
/// <param name="iters">Number of iterations</param>
/// <param name="HessF">Holder of hessian of F</param>
/// <param name="gradF">Holder of gradient of F</param>
/// <param name="temp">Temporary vector of dimension n</param>
/// <param name="deltaX">X search direction</param>
public static float[] Solve(floatLinalg.floatVector CLx, C2Function F, out int iters,
floatLinalg.floatSymPosDefMatrix HessF, floatLinalg.floatVector gradF, floatLinalg.floatVector temp, floatLinalg.floatVector deltaX)
{
return Solve(CLx, F, out iters, HessF, gradF, temp, deltaX, null, null, null, null, null, null, null, null, null, null, null);
}
/// <summary>Computes the unconstrained minimum of function F in x0.Length dimensions</summary>
/// <param name="CLx">Solution x and initial guess</param>
/// <param name="F">Function to be minimized</param>
/// <param name="iters">Number of iterations</param>
/// <param name="HessF">Holder of hessian of F</param>
/// <param name="gradF">Holder of gradient of F</param>
/// <param name="temp">Temporary vector of dimension n</param>
/// <param name="deltaX">X search direction</param>
/// <param name="A">Equality constraint matrix</param>
/// <param name="b">Equality constraint rhs</param>
/// <param name="AinvHAt">Holder of A*inv(H)*A'</param>
/// <param name="tempAinvMatrix">Temporary matrix for computation of inv(H)*A'</param>
/// <param name="temp2">Temporary vector of length n</param>
/// <param name="AinvHrhs">Temporary vector</param>
/// <param name="rPri">Primal residual</param>
/// <param name="tempC">Temporary vector of length C</param>
/// <param name="CLDeltaNu">Search direction in Nu</param>
/// <param name="Atnu">A times nu</param>
/// <param name="onesC">Vector of ones of dimension C</param>
public static float[] Solve(floatLinalg.floatVector CLx, C2Function F, out int iters,
floatLinalg.floatSymPosDefMatrix HessF, floatLinalg.floatVector gradF, floatLinalg.floatVector temp, floatLinalg.floatVector deltaX,
floatLinalg.floatMatrix A, floatLinalg.floatVector b, floatLinalg.floatSymPosDefMatrix AinvHAt, floatLinalg.floatMatrix tempAinvMatrix,
floatLinalg.floatVector temp2, floatLinalg.floatVector AinvHrhs, floatLinalg.floatVector rPri, floatLinalg.floatVector tempC, floatLinalg.floatVector CLDeltaNu,
floatLinalg.floatVector Atnu, floatLinalg.floatVector onesC)
{
float lambda = 1E30f; //, lambdaPrev = 2E30f;
iters = 0;
while (lambda > Config.eps && iters < Config.MaxIterations)
{
//Computes gradient and hessian
F(CLx, true, ref gradF, ref HessF);
HessF.IsCholeskyFactorized = false;
//Search direction is minus gradient
floatLinalg.BLAS.LinearCombination(0, CLx, -1, gradF, ref temp);
if (A == null)
{
//Finds Newton step without equality constraints
deltaX = HessF.LinearSolve(temp, false, ref deltaX);
//HessF.CLcholDec.ReadFromDeviceTo(HessF.cholDec);
//HessF.Write("C:\\oct\\PP.txt");
//temp.Write("C:\\oct\\q.txt");
//deltaX.ReadFromDevice();
}
else
{
//Solves KKT system [Hpd A'; A 0] = [temp; rPri]. Remember that rPri was computed as -Ax+b
//primal residual, -rPri
floatLinalg.BLAS.MatrVecProdSumVec(A, CLx, -1, b, 1, ref rPri);
//A inv(H) A'
floatLinalg.BLAS.ComputeAinvHTranspA(A, HessF, ref AinvHAt, ref tempAinvMatrix, false);
//A inv(H) temp
HessF.LinearSolve(temp, false, ref temp2);
floatLinalg.BLAS.MatrVecProd(A, temp2, 1, ref AinvHrhs);
//Ainv(H)temp - rpri
floatLinalg.BLAS.LinearCombination(1, AinvHrhs, -1, rPri, ref tempC);
//Solves for deltaNu
AinvHAt.LinearSolve(tempC, false, ref CLDeltaNu);
//Procceeds to deltaX = invH * (deltaX - AtDeltanu)
floatLinalg.BLAS.MatrTraspVecMult(A, new floatLinalg.floatDiag(onesC), CLDeltaNu, ref Atnu);
floatLinalg.BLAS.LinearCombination(-1, Atnu, 1, temp, ref temp2);
HessF.LinearSolve(temp2, false, ref deltaX);
}
//Finds step that leads to decrement
BackTrack(F, CLx, deltaX, gradF, ref temp, ref HessF);
////////DEBUG
//if (UseOpenCLIfAvailable && CLCalc.CLAcceleration == CLCalc.CLAccelerationType.UsingCL)
//{
// HessF.CLValues.ReadFromDeviceTo(HessF.Values);
// HessF.CLcholDec.ReadFromDeviceTo(HessF.cholDec);
//}
////CLx.CLValues.ReadFromDeviceTo(CLx.Values);
////deltaX.CLValues.ReadFromDeviceTo(deltaX.Values);
//gradF.ReadFromDevice();
//deltaX.ReadFromDevice();
//CLx.ReadFromDevice();
lambda = (float)Math.Sqrt(-floatLinalg.BLAS.Dot(gradF, deltaX, ref temp));
iters++;
}
if (floatLinalg.UseOpenCLIfAvailable && CLCalc.CLAcceleration == CLCalc.CLAccelerationType.UsingCL) CLx.CLValues.ReadFromDeviceTo(CLx.Values);
return CLx.Values;
}
private void PrimalDualLineSearch(floatLinalg.floatVector x, floatLinalg.floatVector fx, floatLinalg.floatVector lambda, floatLinalg.floatVector nu,
floatLinalg.floatVector dx, floatLinalg.floatVector dlambda, floatLinalg.floatVector dnu,
floatLinalg.floatVector xPlus, floatLinalg.floatVector lambdaPlus, floatLinalg.floatVector nuPlus, ComputeRestrictions F, ComputePDResidualNorm r, ref float t,
out float surrogDualityGap)
{
float BETA = 0.6f;
float ALPHA = 0.02f;
//min(1, -lambda./dlambda)
float s = 1;
lambda.ReadFromDevice();
dlambda.ReadFromDevice();
for (int i = 0; i < lambda.Values.Length; i++)
{
if (dlambda.Values[i] < 0) s = Math.Min(s, -lambda.Values[i] / dlambda.Values[i]);
}
s *= 0.99f;
//fx can't have positive entries
floatLinalg.BLAS.LinearCombination(1, x, s, dx, ref xPlus);
floatLinalg.BLAS.LinearCombination(1, lambda, s, dlambda, ref lambdaPlus);
if (nu != null) floatLinalg.BLAS.LinearCombination(1, nu, s, dnu, ref nuPlus);
bool fxHasPositiveEntry = true;
while (fxHasPositiveEntry && s > 0)
{
F(xPlus, ref fx);
fxHasPositiveEntry = fx.HasPositiveEntries();
if (fx.HasPositiveEntries())
{
s *= BETA;
floatLinalg.BLAS.LinearCombination(1, x, s, dx, ref xPlus);
floatLinalg.BLAS.LinearCombination(1, lambda, s, dlambda, ref lambdaPlus);
if (nu != null) floatLinalg.BLAS.LinearCombination(1, nu, s, dnu, ref nuPlus);
}
}
//Save residuals at this point if necessary
SolutionLog.LogResiduals = true;
//Function decrease requirement
float NormResPrev = r(x, lambda, nu, t);
SolutionLog.LogResiduals = false;
float NormPlus = r(xPlus, lambdaPlus, nuPlus, t);
while (NormPlus > (1.0f - ALPHA * s) * NormResPrev)
{
s *= BETA;
floatLinalg.BLAS.LinearCombination(1, x, s, dx, ref xPlus);
floatLinalg.BLAS.LinearCombination(1, lambda, s, dlambda, ref lambdaPlus);
if (nu != null) floatLinalg.BLAS.LinearCombination(1, nu, s, dnu, ref nuPlus);
NormPlus = r(xPlus, lambdaPlus, nuPlus, t);
}
if (!float.IsNaN(NormPlus))
{
floatLinalg.BLAS.CopyVector(xPlus, x);
floatLinalg.BLAS.CopyVector(lambdaPlus, lambda);
if (nu != null) floatLinalg.BLAS.CopyVector(nuPlus, nu);
F(xPlus, ref fx);
surrogDualityGap = -floatLinalg.BLAS.Dot(fx, lambda, ref tempM);
}
else
{
//Can't improve anymore
surrogDualityGap = 0.0f;
}
t = lambda.Values.Length * 10.0f / surrogDualityGap;
if (SolutionLog.KeepLog)
{
SolutionLog.StepSizes.Add(s);
}
//ReadAll();
}
/// <summary>Solves a quadratic programming problem 1/2 x'Px + q'x subject to Mx less or equal d and Ax = b</summary>
/// <param name="x0">Start point x0</param>
/// <param name="P">Positive semidefinite quadratic objective matrix P</param>
/// <param name="q">Linear objective q</param>
/// <param name="M">Ineq constraint matrix M</param>
/// <param name="d">Ineq constraint right hand side d</param>
/// <param name="A">Constraint matrix A</param>
/// <param name="b">Constraint right hand side b</param>
/// <param name="sf">Stop function. The system won't check feasibility if this function is not null. Return true when you want the optimization to stop based on x, lambda and nu</param>
private float[] SolveBarrier(float[] x0, floatLinalg.floatSymPosDefMatrix P,
float[] q, float[,] M, float[] d, float[,] A, float[] b, StopFunc sf)
{
//Number of primal vars
int n = x0.Length;
//Number of dual vars
int m = M.GetLength(0);
//Constraint variables
int c = A == null ? 0 : A.GetLength(0);
if (P != null && P.getN != n) throw new Exception("Incompatible matrix P dimensions");
if (M.GetLength(0) != m || M.GetLength(1) != n) throw new Exception("Incompatible matrix M dimensions");
if (q.Length != n) throw new Exception("Incompatible vector q dimensions");
if (d.Length != m) throw new Exception("Incompatible vector d dimensions");
CLP = P;
if (CLP != null && !CLP.IsMatrixInClMemoryUpdated)
{
CLP.CLValues.WriteToDevice(CLP.Values);
CLP.IsMatrixInClMemoryUpdated = true;
}
if (c > 0)
{
if (b.Length != c) throw new Exception("Incompatible vector b dimensions");
if (A.GetLength(0) != c || A.GetLength(1) != n) throw new Exception("Incompatible matrix A dimensions");
}
CLM = new floatLinalg.floatMatrix(M);
CLx = new floatLinalg.floatVector(x0);
return null;
}
private void CompRestric(floatLinalg.floatVector x, ref floatLinalg.floatVector fx)
{
floatLinalg.BLAS.MatrVecProdSumVec(CLM, x, 1, CLd, -1, ref fx);
}
/// <summary>Computes the norm of the residuals in a given point</summary>
/// <param name="x">Primal vars</param>
/// <param name="lambda">Dual vars</param>
/// <param name="nu">Equality constraint vars</param>
/// <param name="t">Solution quuality parameter t</param>
private float PDResidual(floatLinalg.floatVector x, floatLinalg.floatVector lambda, floatLinalg.floatVector nu, float t)
{
float resSquared = 0;
//computes rDual = -(Px+q+M'*lambda+A'nu)
if (CLP != null) floatLinalg.BLAS.SymPosDefSymMatrVecMultiply(CLP, x, ref CLPx); //P
floatLinalg.BLAS.MatrTraspVecMult(CLM, new floatLinalg.floatDiag(onesM), lambda, ref temp); //M'*lambda
floatLinalg.BLAS.LinearCombination(1, CLq, 1, temp, ref temp2); // q + M'*lambda
if (CLP != null) floatLinalg.BLAS.LinearCombination(1, CLPx, 1, temp2, ref temp); //Px + q + M'*lambda
else floatLinalg.BLAS.CopyVector(temp2, temp);
float tempR;
if (nu != null) //there are equality constraints
{
floatLinalg.BLAS.MatrTraspVecMult(CLA, new floatLinalg.floatDiag(onesC), CLnu, ref Atnu);
floatLinalg.BLAS.LinearCombination(1, temp, 1, Atnu, ref rDual); //2Px + q + M'*lambda + A*nu
tempR = rDual.NormSquared(ref temp2);
resSquared += tempR;
}
else
{
tempR = temp.NormSquared(ref temp2);
resSquared += tempR;
}
if (SolutionLog.KeepLog && SolutionLog.LogResiduals)
SolutionLog.DualResiduals.Add(tempR);
//Centralization residue -diag(lambda)f(x) - 1/t*ones(m)
//Computes Mx - d = f(x)
floatLinalg.BLAS.MatrVecProdSumVec(CLM, CLx, 1, CLd, -1, ref Mxd);
floatLinalg.BLAS.DiagVecProd(new floatLinalg.floatDiag(lambda), Mxd, -1, ref tempM);
floatLinalg.BLAS.LinearCombination(1, tempM, -1 / t, onesM, ref rCent);
tempR = rCent.NormSquared(ref tempM);
resSquared += tempR;
if (SolutionLog.KeepLog && SolutionLog.LogResiduals)
SolutionLog.CentResiduals.Add(tempR);
if (nu != null)
{
//primal residual, rPri
floatLinalg.BLAS.MatrVecProdSumVec(CLA, CLx, 1, CLb, -1, ref rPri);
resSquared += rPri.NormSquared(ref tempC);
}
return (float)Math.Sqrt(resSquared);
}
/// <summary>Solves a quadratic programming problem 1/2 x'Px + q'x subject to Mx less or equal d and Ax = b</summary>
/// <param name="x0">Start point x0</param>
/// <param name="lambda0">Start dual point lambda0</param>
/// <param name="nu0">Start nus (equality constraints)</param>
/// <param name="P">Positive semidefinite quadratic objective matrix P</param>
/// <param name="q">Linear objective q</param>
/// <param name="M">Ineq constraint matrix M</param>
/// <param name="d">Ineq constraint right hand side d</param>
/// <param name="A">Constraint matrix A</param>
/// <param name="b">Constraint right hand side b</param>
/// <param name="sf">Stop function. The system won't check feasibility if this function is not null. Return true when you want the optimization to stop based on x, lambda and nu</param>
public float[] SolvePrimalDual(float[] x0, float[] lambda0, float[] nu0, floatLinalg.floatSymPosDefMatrix P,
float[] q, float[,] M, float[] d, float[,] A, float[] b, StopFunc sf)
{
//Number of primal vars
int n = x0.Length;
//Number of dual vars
int m = lambda0.Length;
//Constraint variables
int c = nu0 == null ? 0 : nu0.Length;
if (P!=null && P.getN != n) throw new Exception("Incompatible matrix P dimensions");
if (M.GetLength(0) != m || M.GetLength(1) != n) throw new Exception("Incompatible matrix M dimensions");
if (q.Length != n) throw new Exception("Incompatible vector q dimensions");
if (d.Length != m) throw new Exception("Incompatible vector d dimensions");
if (nu0 == null && A != null) throw new Exception("Equality constraint matrix was given. Please also give initial values for laplace multipliers nu0");
CLP = P;
if (CLP != null && !CLP.IsMatrixInClMemoryUpdated)
{
CLP.CLValues.WriteToDevice(CLP.Values);
CLP.IsMatrixInClMemoryUpdated = true;
}
if (c > 0)
{
if (b.Length != c) throw new Exception("Incompatible vector b dimensions");
if (A.GetLength(0) != c || A.GetLength(1) != n) throw new Exception("Incompatible matrix A dimensions");
}
CLM = new floatLinalg.floatMatrix(M);
CLx = new floatLinalg.floatVector(x0);
CLxPlus = new floatLinalg.floatVector(x0);
CLlambda = new floatLinalg.floatVector(lambda0);
CLlambdaPlus = new floatLinalg.floatVector(lambda0);
CLDeltaX = new floatLinalg.floatVector(x0);
CLDeltaLambda = new floatLinalg.floatVector(lambda0);
CLq = new floatLinalg.floatVector(q);
CLd = new floatLinalg.floatVector(d);
//Auxiliary vars
zerosN = new floatLinalg.floatVector(new float[n]);
float[] fOnes = new float[m];
for (int i = 0; i < fOnes.Length; i++) fOnes[i] = 1;
onesM = new floatLinalg.floatVector(fOnes);
Hpd = new floatLinalg.floatSymPosDefMatrix(n);
MtDzM = new floatLinalg.floatSymPosDefMatrix(n);
temp = new floatLinalg.floatVector(new float[n]);
temp2 = new floatLinalg.floatVector(new float[n]);
rhsXpd = new floatLinalg.floatVector(new float[n]);
Mxd = new floatLinalg.floatVector(new float[m]);
z2 = new floatLinalg.floatVector(new float[m]);
z = new floatLinalg.floatVector(new float[m]);
rCent = new floatLinalg.floatVector(new float[m]);
tempM = new floatLinalg.floatVector(new float[m]);
tempM2 = new floatLinalg.floatVector(new float[m]);
CLPx = new floatLinalg.floatVector(new float[n]);
//Equality constraint variables
if (c > 0)
{
CLA = new floatLinalg.floatMatrix(A);
CLb = new floatLinalg.floatVector(b);
CLnu = new floatLinalg.floatVector(nu0);
CLDeltaNu = new floatLinalg.floatVector(nu0);
rPri = new floatLinalg.floatVector(new float[c]);
Atnu = new floatLinalg.floatVector(new float[n]);
tempC = new floatLinalg.floatVector(new float[c]);
CLnuPlus = new floatLinalg.floatVector(new float[c]);
AinvHAt = new floatLinalg.floatSymPosDefMatrix(c);
AinvHrhs = new floatLinalg.floatVector(new float[c]);
tempAinvMatrix = new floatLinalg.floatMatrix(new float[n, c]);
fOnes = new float[c];
for (int i = 0; i < fOnes.Length; i++) fOnes[i] = 1;
onesC = new floatLinalg.floatVector(fOnes);
}
else
{
CLA = null;
CLb = null;
CLnu = null;
CLnuPlus = null;
}
//Checks feasibility
bool feasible = true;
float[] xfeas;
if (sf == null)
{
xfeas = CheckFeasibility(x0, M, d, A, b);
feasible = (xfeas != null);
if (floatLinalg.UseOpenCLIfAvailable && CLCalc.CLAcceleration == CLCalc.CLAccelerationType.UsingCL) CLx.CLValues.WriteToDevice(xfeas);
else for (int i = 0; i < xfeas.Length; i++) CLx.Values[i] = xfeas[i];
}
if (SolutionLog.KeepLog)
{
CLx.ReadFromDevice();
SolutionLog.PtSequence.Add((float[])CLx.Values.Clone());
}
if (feasible)
{
float surrogDualityGap = 100;
float t = 0.1f;
int MAXITER = 60;
int curIter = 0;
while (surrogDualityGap > 5e-5f && curIter < MAXITER &&
(sf == null || (sf != null && !sf(CLx, CLlambda, CLnu)))
)
{
if (curIter == (MAXITER >> 1))
{
//DEBUG, taking too long to converge
for (int i = 0; i < CLlambda.Values.Length; i++) CLlambda.Values[i] = 0.1f;
if (floatLinalg.UseOpenCLIfAvailable && CLCalc.CLAcceleration == CLCalc.CLAccelerationType.UsingCL) CLlambda.CLValues.WriteToDevice(CLlambda.Values);
t *= 0.1f;
}
//PDResidual(CLx, CLlambda, CLnu, t);
ComputePrimalDualSearchDir(t);
PrimalDualLineSearch(CLx, Mxd, CLlambda, CLnu, CLDeltaX, CLDeltaLambda,
CLDeltaNu, CLxPlus, CLlambdaPlus, CLnuPlus, CompRestric, PDResidual, ref t, out surrogDualityGap);
curIter++;
if (SolutionLog.KeepLog)
{
SolutionLog.SurrDualityGaps.Add(surrogDualityGap);
CLx.ReadFromDevice();
SolutionLog.PtSequence.Add((float[])CLx.Values.Clone());
}
}
SolutionLog.Iterations = curIter;
//ReadAll();
//Copies lambdas and nus
CLlambda.ReadFromDevice();
for (int i = 0; i < m; i++) lambda0[i] = CLlambda.Values[i];
if (CLnu != null)
{
CLnu.ReadFromDevice();
for (int i = 0; i < c; i++) nu0[i] = CLnu.Values[i];
}
CLx.ReadFromDevice();
return CLx.Values;
}
else return null; //Not feasible
}
static bool feasStopFunc(floatLinalg.floatVector x, floatLinalg.floatVector lambda, floatLinalg.floatVector nu)
{
float[] lastElem = new float[1];
if (floatLinalg.UseOpenCLIfAvailable && CLCalc.CLAcceleration == CLCalc.CLAccelerationType.UsingCL)
{
if (x.CLn == null) x.CLn = new CLCalc.Program.Variable(new int[] { x.Values.Length });
kernelgetLast.Execute(new CLCalc.Program.MemoryObject[] { x.CLValues, x.CLn, x.CLCoef }, 1);
x.CLCoef.ReadFromDeviceTo(lastElem);
}
else
{
lastElem[0] = x.Values[x.Values.Length - 1];
}
if (SolutionLog.KeepLog) SolutionLog.EvolutionOfT.Add(lastElem[0]);
//Stops if last element is less than zero
return lastElem[0] < 0;
}
/// <summary>Computes cost function for logistic regression at a given x</summary>
/// <param name="CLTheta">Current function point</param>
/// <param name="ComputeGradHess">Compute gradient and hessian?</param>
/// <param name="Grad">Gradientof F, if requested. Should not be computed if not requested</param>
/// <param name="Hess">Hessian of F, if requested. Should not be computed if not requested</param>
public float RegularizedLogistReg(floatLinalg.floatVector CLTheta, bool ComputeGradHess, ref floatLinalg.floatVector Grad, ref floatLinalg.floatSymPosDefMatrix Hess)
{
//Computes x*Theta
floatLinalg.BLAS.MatrVecProd(CLX, CLTheta, 1, ref XTheta);
floatLinalg.BLAS.CopyVector(CLTheta, tempX);
//Computes cost and gradient/hessian terms
if (floatLinalg.UseOpenCLIfAvailable && CLCalc.CLAcceleration == CLCalc.CLAccelerationType.UsingCL)
{
kernelComputeLogistRegParams.Execute(new CLCalc.Program.Variable[] { XTheta.CLValues, y.CLValues, z1.CLValues, z2.CLValues, cost.CLValues }, XTheta.Length);
//Regularization
kernelpNorm.Execute(new CLCalc.Program.Variable[] { tempX.CLValues, CLq.CLValues, lambda.CLValues }, tempX.Length);
}
else
{
for (int i = 0; i < z1.Length; i++)
{
float eMz = (float)Math.Exp(-XTheta.Values[i]);
float hTheta = 1.0f / (1.0f + eMz);
z1.Values[i] = hTheta - y.Values[i];
z2.Values[i] = eMz * hTheta * hTheta;
cost.Values[i] = y.Values[i] == 0 ? -(float)Math.Log(1 - hTheta) : -(float)Math.Log(hTheta);
}
//Regularization cost
for (int i = 0; i < CLTheta.Length; i++)
{
tempX.Values[i] = (float)Math.Pow(Math.Abs(tempX.Values[i]), CLq.Values[0]) * lambda.Values[i];
}
}
if (ComputeGradHess)
{
//Regularization
if (floatLinalg.UseOpenCLIfAvailable && CLCalc.CLAcceleration == CLCalc.CLAccelerationType.UsingCL)
{
kerneldpNorm.Execute(new CLCalc.Program.Variable[] { CLTheta.CLValues, CLq.CLValues, lambda.CLValues, tempdX.CLValues, tempd2X.CLValues }, CLTheta.Length);
}
else
{
for (int i = 0; i < CLTheta.Values.Length; i++)
{
float temp = CLTheta.Values[i];
tempdX.Values[i] = (float)Math.Pow(Math.Abs(temp), CLq.Values[0] - 1.0f) * lambda.Values[i] * Math.Sign(temp) * CLq.Values[0];
tempd2X.Values[i] = (float)Math.Pow(Math.Abs(temp), CLq.Values[0] - 2.0f) * lambda.Values[i] * CLq.Values[0] * (CLq.Values[0] - 1.0f);
}
}
floatLinalg.BLAS.MatrTraspVecMult(CLX, new floatLinalg.floatDiag(ones), z1, ref CLtempGrad);
floatLinalg.BLAS.LinearCombination(1, CLtempGrad, 1, tempdX, ref CLGrad);
floatLinalg.BLAS.MatrTranspMatrProd(CLX, new floatLinalg.floatDiag(z2), tempd2X, ref Hess);
}
return cost.Sum() + tempX.Sum();
}
/// <summary>Checks if it's possible to satisfy Mx less than d and Ax = b. Returns a feasible point in x0 if so</summary>
/// <param name="x0">Initial guess</param>
/// <param name="M">Inequality constraint matrix</param>
/// <param name="d">Inequality rhs</param>
/// <param name="A">Equality constr matrix</param>
/// <param name="b">Equality rhs</param>
/// <param name="Mxd">Temporary vector to hold M*x - d</param>
public static bool CheckFeasibility(floatLinalg.floatVector x0, floatLinalg.floatMatrix M, floatLinalg.floatVector d, floatLinalg.floatMatrix A,
floatLinalg.floatVector b, floatLinalg.floatVector Mxd)
{
SolutionLog.Clear();
//Computes Mx - d
floatLinalg.BLAS.MatrVecProdSumVec(M, x0, 1, d, -1, ref Mxd);
//Problem is already feasible
if (!Mxd.HasPositiveEntries())
{
return true;
}
//If problem is not feasible, computes largest term
Mxd.ReadFromDevice();
float max = 1.0f;
for (int i = 0; i < Mxd.Values.Length; i++) if (max < Mxd.Values[i]) max = Mxd.Values[i];
//Augments max to create a feasible point for the feasibility problem
max *= 1.1f;
//Constructs the instance of the feasibility problem.
//Augments M and A by one. M will contain the -1s in the extra row; A has to contain an extra zero in the last additional column
if (CLCalc.CLAcceleration==CLCalc.CLAccelerationType.UsingCL) M.CLValues.ReadFromDeviceTo(M.Values);
float[,] Mfeas = new float[M.Rows, M.Cols + 1];
for (int i = 0; i < M.Rows; i++)
{
for (int j = 0; j < M.Cols; j++)
{
Mfeas[i, j] = M[i, j];
}
Mfeas[i, M.Cols] = -1;
}
float[,] Afeas = null;
if (A != null)
{
Afeas = new float[A.Rows, A.Cols + 1];
for (int i = 0; i < A.Rows; i++)
{
for (int j = 0; j < A.Cols; j++)
{
Afeas[i, j] = A[i, j];
}
Afeas[i, M.Cols] = 0;
}
}
//Feasibility problem feasible point
float[] xFeas = new float[x0.Length + 1];
x0.ReadFromDevice();
for (int i = 0; i < x0.Length; i++) xFeas[i] = x0.Values[i];
xFeas[x0.Length] = max;
float[] lambda = new float[M.Rows];
for (int i = 0; i < lambda.Length; i++) lambda[i] = 0.1f;
float[] nu = null;
if (A != null) nu = new float[A.Rows];
//Objective is to minimize last variable
float[] q = new float[x0.Length + 1];
//for (int i = 0; i < x0.Length; i++) q[i] = 0.001f;
q[x0.Length] = 10 * max * (float)lambda.Length * (float)lambda.Length;
QuadraticProgramming qp = new QuadraticProgramming();
d.ReadFromDevice();
float[] bVals = null;
if (A != null)
{
b.ReadFromDevice();
bVals = b.Values;
}
float[] xF = qp.SolvePrimalDual(xFeas, lambda, nu, null, q, Mfeas, d.Values, Afeas, bVals, feasStopFunc);
if (xF[xF.Length - 1] < 0) //feasible
{
for (int i = 0; i < x0.Length; i++) x0.Values[i] = xF[i];
if (floatLinalg.UseOpenCLIfAvailable && CLCalc.CLAccelerationType.UsingCL == CLCalc.CLAcceleration)
{
x0.CLValues.WriteToDevice(x0.Values);
}
return true;
}
else //infeasible
{
return false;
}
}
/// <summary>Gets hit rate of a given set</summary>
/// <param name="Samples">Samples to rate. One sample per row</param>
/// <param name="Labels">Correct labels</param>
public float GetHitRate(floatLinalg.floatMatrix Samples, float[] Labels)
{
floatLinalg.floatMatrix Values = null;
float[] MaxVals;
float[] TestLbls = Classify(Samples, ref Values, out MaxVals);
float total = 0;
float correct = 0;
for (int i = 0; i < Samples.Rows; i++)
{
total += 1;
if ((Labels[i] == TestLbls[i] && MaxVals[i] > 0) || (Labels[i] == 0 && MaxVals[i] <= 0))
{
correct += 1;
}
}
return correct / total;
}
/// <summary>Classifies a Samples matrix, one sample per row. Note: the interceptor x[0] = 1 has to be included</summary>
/// <param name="Samples">Samples matrix, [n x p+1], where p = original x dimension</param>
/// <param name="Values">Classification values</param>
/// <param name="maxVals">Maximum classification values.
/// Note: in the classical approach one would need to compute the 1/(1+exp(-Values[i])) to get the logistic rating</param>
public float[] Classify(floatLinalg.floatMatrix Samples, ref floatLinalg.floatMatrix Values, out float[] maxVals)
{
int n = Samples.Rows;
int p = CLM.Cols - 1;
if (Samples.Cols != p + 1) throw new Exception("Incompatible Samples dimensions");
floatLinalg.BLAS.MatrTranspMatrProd(CLM, Samples, ref Values);
if (floatLinalg.UseOpenCLIfAvailable && CLCalc.CLAcceleration == CLCalc.CLAccelerationType.UsingCL) Values.CLValues.ReadFromDeviceTo(Values.Values);
//Values dimensions: nSamples x nCategories
maxVals = new float[n];
int[] indMax = new int[n];
for (int j = 0; j < n; j++) maxVals[j] = Values.Values[j];
for (int i = 1; i < Categories.Count; i++)
{
for (int j = 0; j < n; j++)
{
if (maxVals[j] < Values.Values[j + n * i])
{
maxVals[j] = Values.Values[j + n * i];
indMax[j] = i;
}
}
}
float[] categories = new float[n];
for (int i = 0; i < categories.Length; i++) categories[i] = Categories[indMax[i]];
return categories;
}
/// <summary>Creates a new logistic regression classifier from already trained coefficients.
/// Note: GetInternalHitRate will not work! (because there are no internal samples)</summary>
/// <param name="CLM">Coefficients, including interceptor coefficient</param>
/// <param name="Categories">Classification categories</param>
public LogisticRegression(floatLinalg.floatMatrix CLM, List<float> Categories)
{
int p = CLM.Cols - 1;
int nCategs = CLM.Rows;
if (Categories.Count != nCategs) throw new Exception("Number of categories and classification coefficients dimension not compatible");
this.CLM = CLM;
this.Categories = Categories;
}
/// <summary>Computes objective function, gradient and Hessian for p-norm minimization with q-norm regularization</summary>
public float F(floatLinalg.floatVector x, bool ComputeGradHess, ref floatLinalg.floatVector gradF, ref floatLinalg.floatSymPosDefMatrix H)
{
float resp;
//Computes Ax-b
floatLinalg.BLAS.MatrVecProdSumVec(A, x, 1, b, -1, ref Axmb);
floatLinalg.BLAS.CopyVector(Axmb, tempAxmb);
floatLinalg.BLAS.CopyVector(x, tempX);
if (floatLinalg.UseOpenCLIfAvailable && CLCalc.CLAcceleration == CLCalc.CLAccelerationType.UsingCL)
{
kernelpNorm.Execute(new CLCalc.Program.Variable[] { tempX.CLValues, CLq.CLValues, lambda.CLValues }, tempX.Length);
kernelpNorm.Execute(new CLCalc.Program.Variable[] { tempAxmb.CLValues, CLp.CLValues, w.CLValues }, tempAxmb.Length);
}
else
{
for (int i = 0; i < A.Rows; i++)
{
tempAxmb.Values[i] = (float)Math.Pow(Math.Abs(tempAxmb.Values[i]), CLp.Values[0]) * w.Values[i];
}
for (int i = 0; i < A.Cols; i++)
{
tempX.Values[i] = (float)Math.Pow(Math.Abs(tempX.Values[i]), CLq.Values[0]) * lambda.Values[i];
}
}
resp = tempX.Sum() + tempAxmb.Sum();
if (ComputeGradHess)
{
//Atz
if (floatLinalg.UseOpenCLIfAvailable && CLCalc.CLAcceleration == CLCalc.CLAccelerationType.UsingCL)
kerneldpNorm.Execute(new CLCalc.Program.Variable[] { Axmb.CLValues, CLp.CLValues, w.CLValues, dtempAtz.CLValues, d2tempAtz.CLValues }, tempAxmb.Length);
else
{
for (int i = 0; i < Axmb.Values.Length; i++)
{
float temp = Axmb.Values[i];
dtempAtz.Values[i] = (float)Math.Pow(Math.Abs(temp), CLp.Values[0] - 1.0f) * w.Values[i] * Math.Sign(temp) * CLp.Values[0];
d2tempAtz.Values[i] = (float)Math.Pow(Math.Abs(temp), CLp.Values[0] - 2.0f) * w.Values[i] * CLp.Values[0] * (CLp.Values[0] - 1.0f);
}
}
floatLinalg.BLAS.MatrTraspVecMult(A, Identity, dtempAtz, ref tempAtz);
//x
if (floatLinalg.UseOpenCLIfAvailable && CLCalc.CLAcceleration == CLCalc.CLAccelerationType.UsingCL)
kerneldpNorm.Execute(new CLCalc.Program.Variable[] { x.CLValues, CLq.CLValues, lambda.CLValues, tempdX.CLValues, tempd2x.CLValues }, x.Length);
else
{
for (int i = 0; i < x.Values.Length; i++)
{
float temp = x.Values[i];
tempdX.Values[i] = (float)Math.Pow(Math.Abs(temp), CLq.Values[0] - 1.0f) * lambda.Values[i] * Math.Sign(temp) * CLq.Values[0];
tempd2x.Values[i] = (float)Math.Pow(Math.Abs(temp), CLq.Values[0] - 2.0f) * lambda.Values[i] * CLq.Values[0] * (CLq.Values[0] - 1.0f);
}
}
//Accumulates gradients
floatLinalg.BLAS.LinearCombination(1, tempAtz, 1, tempdX, ref gradF);
//d2tempAtz.ReadFromDevice();
//tempd2x.ReadFromDevice();
//Hessian
//A.Write("C:\\oct\\A.txt");
//d2tempAtz.Write("C:\\oct\\z2.txt");
floatLinalg.BLAS.MatrTranspMatrProd(A, Diagd2Atz, tempd2x, ref H);
}
return resp;
}
public PNormMinClass(floatLinalg.floatMatrix A, floatLinalg.floatVector b, floatLinalg.floatVector w,
floatLinalg.floatVector lambda, floatLinalg.floatVector CLp, floatLinalg.floatVector CLq)
{
this.A = A;
this.b = b;
this.w = w;
this.lambda = lambda;
this.CLp = CLp;
this.CLq = CLq;
//Temporary buffers
tempX = new floatLinalg.floatVector(new float[A.Cols]);
tempdX = new floatLinalg.floatVector(new float[A.Cols]);
tempAxmb = new floatLinalg.floatVector(new float[A.Rows]);
Axmb = new floatLinalg.floatVector(new float[A.Rows]);
Diagw = new floatLinalg.floatDiag(w);
Diaglambda = new floatLinalg.floatDiag(lambda);
tempdX = new floatLinalg.floatVector(new float[lambda.Length]);
tempd2x = new floatLinalg.floatVector(new float[lambda.Length]);
tempAtz = new floatLinalg.floatVector(new float[lambda.Length]);
dtempAtz = new floatLinalg.floatVector(new float[w.Length]);
d2tempAtz = new floatLinalg.floatVector(new float[w.Length]);
Diagd2Atz = new floatLinalg.floatDiag(d2tempAtz);
float[] id = new float[A.Rows];
for (int i = 0; i < A.Rows; i++) id[i] = 1;
Identity = new floatLinalg.floatDiag(id);
}
