/// <summary>Computes the unconstrained minimum of function F in x0.Length dimensions subject to equality constraint Ax=b</summary>
/// <param name="x0">Start point. Problem dimension is assumed to be x0.Length</param>
/// <param name="F">Function to minimize</param>
/// <param name="A">Equality constraint matrix</param>
/// <param name="b">Equality constraint rhs</param>
/// <param name="iters">Number of iterations used to solve the problem</param>
public static float[] Solve(float[] x0, C2Function F, float[,] A, float[] b, out int iters)
{
floatLinalg.floatVector CLx = new floatLinalg.floatVector(x0);
//Creates Hessian and Gradient
floatLinalg.floatSymPosDefMatrix HessF = new floatLinalg.floatSymPosDefMatrix(x0.Length);
floatLinalg.floatVector gradF = new floatLinalg.floatVector(x0);
//Line search variables
floatLinalg.floatVector temp = new floatLinalg.floatVector(x0);
floatLinalg.floatVector deltaX = new floatLinalg.floatVector(x0);
//Equality constraint vars
if (A != null)
{
int c = A.GetLength(0);
int n = x0.Length;
floatLinalg.floatMatrix CLA = new floatLinalg.floatMatrix(A);
floatLinalg.floatVector CLb = new floatLinalg.floatVector(b);
floatLinalg.floatVector CLDeltaNu = new floatLinalg.floatVector(b);
floatLinalg.floatVector rPri = new floatLinalg.floatVector(new float[c]);
floatLinalg.floatVector Atnu = new floatLinalg.floatVector(new float[n]);
floatLinalg.floatVector tempC = new floatLinalg.floatVector(new float[c]);
floatLinalg.floatVector temp2 = new floatLinalg.floatVector(x0);
floatLinalg.floatSymPosDefMatrix AinvHAt = new floatLinalg.floatSymPosDefMatrix(c);
floatLinalg.floatVector AinvHrhs = new floatLinalg.floatVector(new float[c]);
floatLinalg.floatMatrix tempAinvMatrix = new floatLinalg.floatMatrix(new float[n, c]);
float[] fOnes = new float[c];
for (int i = 0; i < fOnes.Length; i++) fOnes[i] = 1;
floatLinalg.floatVector onesC = new floatLinalg.floatVector(fOnes);
return Solve(CLx, F, out iters, HessF, gradF, temp, deltaX, CLA, CLb, AinvHAt, tempAinvMatrix, temp2, AinvHrhs, rPri, tempC, CLDeltaNu, Atnu, onesC);
}
else return Solve(CLx, F, out iters, HessF, gradF, temp, deltaX, null, null, null, null, null, null, null, null, null, null, null);
}
/// <summary>Solves a quadratic programming problem 1/2 x'Px + q'x subject to Mx less or equal d and Ax = b</summary>
/// <param name="x0">Start point x0</param>
/// <param name="lambda0">Start dual point lambda0</param>
/// <param name="nu0">Start nus (equality constraints)</param>
/// <param name="P">Positive semidefinite quadratic objective matrix P</param>
/// <param name="q">Linear objective q</param>
/// <param name="M">Ineq constraint matrix M</param>
/// <param name="d">Ineq constraint right hand side d</param>
/// <param name="A">Constraint matrix A</param>
/// <param name="b">Constraint right hand side b</param>
/// <param name="sf">Stop function. The system won't check feasibility if this function is not null. Return true when you want the optimization to stop based on x, lambda and nu</param>
public float[] SolvePrimalDual(float[] x0, float[] lambda0, float[] nu0, floatLinalg.floatSymPosDefMatrix P,
float[] q, float[,] M, float[] d, float[,] A, float[] b, StopFunc sf)
{
//Number of primal vars
int n = x0.Length;
//Number of dual vars
int m = lambda0.Length;
//Constraint variables
int c = nu0 == null ? 0 : nu0.Length;
if (P!=null && P.getN != n) throw new Exception("Incompatible matrix P dimensions");
if (M.GetLength(0) != m || M.GetLength(1) != n) throw new Exception("Incompatible matrix M dimensions");
if (q.Length != n) throw new Exception("Incompatible vector q dimensions");
if (d.Length != m) throw new Exception("Incompatible vector d dimensions");
if (nu0 == null && A != null) throw new Exception("Equality constraint matrix was given. Please also give initial values for laplace multipliers nu0");
CLP = P;
if (CLP != null && !CLP.IsMatrixInClMemoryUpdated)
{
CLP.CLValues.WriteToDevice(CLP.Values);
CLP.IsMatrixInClMemoryUpdated = true;
}
if (c > 0)
{
if (b.Length != c) throw new Exception("Incompatible vector b dimensions");
if (A.GetLength(0) != c || A.GetLength(1) != n) throw new Exception("Incompatible matrix A dimensions");
}
CLM = new floatLinalg.floatMatrix(M);
CLx = new floatLinalg.floatVector(x0);
CLxPlus = new floatLinalg.floatVector(x0);
CLlambda = new floatLinalg.floatVector(lambda0);
CLlambdaPlus = new floatLinalg.floatVector(lambda0);
CLDeltaX = new floatLinalg.floatVector(x0);
CLDeltaLambda = new floatLinalg.floatVector(lambda0);
CLq = new floatLinalg.floatVector(q);
CLd = new floatLinalg.floatVector(d);
//Auxiliary vars
zerosN = new floatLinalg.floatVector(new float[n]);
float[] fOnes = new float[m];
for (int i = 0; i < fOnes.Length; i++) fOnes[i] = 1;
onesM = new floatLinalg.floatVector(fOnes);
Hpd = new floatLinalg.floatSymPosDefMatrix(n);
MtDzM = new floatLinalg.floatSymPosDefMatrix(n);
temp = new floatLinalg.floatVector(new float[n]);
temp2 = new floatLinalg.floatVector(new float[n]);
rhsXpd = new floatLinalg.floatVector(new float[n]);
Mxd = new floatLinalg.floatVector(new float[m]);
z2 = new floatLinalg.floatVector(new float[m]);
z = new floatLinalg.floatVector(new float[m]);
rCent = new floatLinalg.floatVector(new float[m]);
tempM = new floatLinalg.floatVector(new float[m]);
tempM2 = new floatLinalg.floatVector(new float[m]);
CLPx = new floatLinalg.floatVector(new float[n]);
//Equality constraint variables
if (c > 0)
{
CLA = new floatLinalg.floatMatrix(A);
CLb = new floatLinalg.floatVector(b);
CLnu = new floatLinalg.floatVector(nu0);
CLDeltaNu = new floatLinalg.floatVector(nu0);
rPri = new floatLinalg.floatVector(new float[c]);
Atnu = new floatLinalg.floatVector(new float[n]);
tempC = new floatLinalg.floatVector(new float[c]);
CLnuPlus = new floatLinalg.floatVector(new float[c]);
AinvHAt = new floatLinalg.floatSymPosDefMatrix(c);
AinvHrhs = new floatLinalg.floatVector(new float[c]);
tempAinvMatrix = new floatLinalg.floatMatrix(new float[n, c]);
fOnes = new float[c];
for (int i = 0; i < fOnes.Length; i++) fOnes[i] = 1;
onesC = new floatLinalg.floatVector(fOnes);
}
else
{
CLA = null;
CLb = null;
CLnu = null;
CLnuPlus = null;
}
//Checks feasibility
bool feasible = true;
float[] xfeas;
if (sf == null)
{
xfeas = CheckFeasibility(x0, M, d, A, b);
feasible = (xfeas != null);
if (floatLinalg.UseOpenCLIfAvailable && CLCalc.CLAcceleration == CLCalc.CLAccelerationType.UsingCL) CLx.CLValues.WriteToDevice(xfeas);
else for (int i = 0; i < xfeas.Length; i++) CLx.Values[i] = xfeas[i];
}
if (SolutionLog.KeepLog)
{
CLx.ReadFromDevice();
SolutionLog.PtSequence.Add((float[])CLx.Values.Clone());
}
if (feasible)
{
float surrogDualityGap = 100;
float t = 0.1f;
int MAXITER = 60;
int curIter = 0;
while (surrogDualityGap > 5e-5f && curIter < MAXITER &&
(sf == null || (sf != null && !sf(CLx, CLlambda, CLnu)))
)
{
if (curIter == (MAXITER >> 1))
{
//DEBUG, taking too long to converge
for (int i = 0; i < CLlambda.Values.Length; i++) CLlambda.Values[i] = 0.1f;
if (floatLinalg.UseOpenCLIfAvailable && CLCalc.CLAcceleration == CLCalc.CLAccelerationType.UsingCL) CLlambda.CLValues.WriteToDevice(CLlambda.Values);
t *= 0.1f;
}
//PDResidual(CLx, CLlambda, CLnu, t);
ComputePrimalDualSearchDir(t);
PrimalDualLineSearch(CLx, Mxd, CLlambda, CLnu, CLDeltaX, CLDeltaLambda,
CLDeltaNu, CLxPlus, CLlambdaPlus, CLnuPlus, CompRestric, PDResidual, ref t, out surrogDualityGap);
curIter++;
if (SolutionLog.KeepLog)
{
SolutionLog.SurrDualityGaps.Add(surrogDualityGap);
CLx.ReadFromDevice();
SolutionLog.PtSequence.Add((float[])CLx.Values.Clone());
}
}
SolutionLog.Iterations = curIter;
//ReadAll();
//Copies lambdas and nus
CLlambda.ReadFromDevice();
for (int i = 0; i < m; i++) lambda0[i] = CLlambda.Values[i];
if (CLnu != null)
{
CLnu.ReadFromDevice();
for (int i = 0; i < c; i++) nu0[i] = CLnu.Values[i];
}
CLx.ReadFromDevice();
return CLx.Values;
}
else return null; //Not feasible
}
/// <summary>Solves a quadratic programming problem 1/2 x'Px + q'x subject to Mx less or equal d and Ax = b</summary>
/// <param name="x0">Start point x0</param>
/// <param name="P">Positive semidefinite quadratic objective matrix P</param>
/// <param name="q">Linear objective q</param>
/// <param name="M">Ineq constraint matrix M</param>
/// <param name="d">Ineq constraint right hand side d</param>
/// <param name="A">Constraint matrix A</param>
/// <param name="b">Constraint right hand side b</param>
/// <param name="sf">Stop function. The system won't check feasibility if this function is not null. Return true when you want the optimization to stop based on x, lambda and nu</param>
private float[] SolveBarrier(float[] x0, floatLinalg.floatSymPosDefMatrix P,
float[] q, float[,] M, float[] d, float[,] A, float[] b, StopFunc sf)
{
//Number of primal vars
int n = x0.Length;
//Number of dual vars
int m = M.GetLength(0);
//Constraint variables
int c = A == null ? 0 : A.GetLength(0);
if (P != null && P.getN != n) throw new Exception("Incompatible matrix P dimensions");
if (M.GetLength(0) != m || M.GetLength(1) != n) throw new Exception("Incompatible matrix M dimensions");
if (q.Length != n) throw new Exception("Incompatible vector q dimensions");
if (d.Length != m) throw new Exception("Incompatible vector d dimensions");
CLP = P;
if (CLP != null && !CLP.IsMatrixInClMemoryUpdated)
{
CLP.CLValues.WriteToDevice(CLP.Values);
CLP.IsMatrixInClMemoryUpdated = true;
}
if (c > 0)
{
if (b.Length != c) throw new Exception("Incompatible vector b dimensions");
if (A.GetLength(0) != c || A.GetLength(1) != n) throw new Exception("Incompatible matrix A dimensions");
}
CLM = new floatLinalg.floatMatrix(M);
CLx = new floatLinalg.floatVector(x0);
return null;
}
private void Init(float[,] Samples, float[] Classifications, float[] RegularizationWeights, float regularizationQ)
{
int n = Samples.GetLength(0);
int p = Samples.GetLength(1);
if (Classifications.Length != n) throw new Exception("Incompatible Classifications vector dimension");
if (RegularizationWeights.Length != p) throw new Exception("Incompatible RegularizationWeights vector dimension");
if (regularizationQ <= 1) throw new Exception("Regularization term Q should be > 1");
//Includes intercept term
float[,] X = new float[n, p + 1];
for (int i = 0; i < n; i++)
{
X[i, 0] = 1;
for (int j = 0; j < p; j++)
{
X[i, j + 1] = Samples[i, j];
}
}
//Regularization terms
float[] regTerms = new float[p + 1];
regTerms[0] = 0;
for (int i = 0; i < RegularizationWeights.Length; i++) regTerms[i + 1] = RegularizationWeights[i];
//Retrieves categories
for (int i = 0; i < Classifications.Length; i++)
{
if (Classifications[i] > 0 && Categories.IndexOf(Classifications[i]) < 0)
Categories.Add(Classifications[i]);
}
if (Categories.Count == 0) throw new Exception("At least one category should be given");
//Creates samples matrix and vectors
CLX = new floatLinalg.floatMatrix(X);
y = new floatLinalg.floatVector(Classifications);
//Don't regularize first term
float[] newLambs = new float[p + 1];
newLambs[0] = 0;
for (int i = 0; i < p; i++) newLambs[i + 1] = RegularizationWeights[i];
lambda = new floatLinalg.floatVector(newLambs);
//Initial guess
float[] t0 = new float[p + 1];
for (int i = 0; i < t0.Length; i++) t0[i] = 1E-3f;
//t0[0] = 1; t0[1] = 0.2f; t0[2] = 0.3f;
CLTheta = new floatLinalg.floatVector(t0);
CLGrad = new floatLinalg.floatVector(new float[p + 1]);
CLtempGrad = new floatLinalg.floatVector(new float[p + 1]);
CLq = new floatLinalg.floatVector(new float[] { regularizationQ });
tempX = new floatLinalg.floatVector(new float[p + 1]);
tempdX = new floatLinalg.floatVector(new float[p + 1]);
tempd2X = new floatLinalg.floatVector(new float[p + 1]);
temp = new floatLinalg.floatVector(new float[p + 1]);
CLDeltaTheta = new floatLinalg.floatVector(new float[p + 1]);
Hess = new floatLinalg.floatSymPosDefMatrix(p + 1);
XTheta = new floatLinalg.floatVector(new float[n]);
z1 = new floatLinalg.floatVector(new float[n]);
z2 = new floatLinalg.floatVector(new float[n]);
cost = new floatLinalg.floatVector(new float[n]);
float[] vones = new float[n];
for (int i = 0; i < n; i++) vones[i] = 1;
ones = new floatLinalg.floatVector(vones);
_classifs = Classifications;
//Trains classifier
Train();
}
/// <summary>Checks if it's possible to satisfy Mx less than d and Ax = b. Returns a feasible point if so</summary>
/// <param name="x0">Initial guess</param>
/// <param name="M">Inequality constraint matrix</param>
/// <param name="d">Inequality rhs</param>
/// <param name="A">Equality constr matrix</param>
/// <param name="b">Equality rhs</param>
public static float[] CheckFeasibility(float[] x0, float[,] M, float[] d, float[,] A, float[] b)
{
//Computes Mx-d
floatLinalg.floatMatrix CLM = new floatLinalg.floatMatrix(M);
floatLinalg.floatVector CLd = new floatLinalg.floatVector(d);
floatLinalg.floatVector CLx = new floatLinalg.floatVector(x0);
floatLinalg.floatMatrix CLA = null;
floatLinalg.floatVector CLb = null;
if (A != null)
{
CLA = new floatLinalg.floatMatrix(M);
CLb = new floatLinalg.floatVector(d);
}
floatLinalg.floatVector Mxd = new floatLinalg.floatVector(new float[M.GetLength(0)]);
if (CheckFeasibility(CLx, CLM, CLd, CLA, CLb, Mxd))
{
CLx.ReadFromDevice();
return (float[])CLx.Values.Clone();
}
else return null;
}
/// <summary>Trains this classifier using the samples Matrix and classifications vector</summary>
public void Train()
{
int n = CLX.Rows;
int p = CLX.Cols - 1;
//Builds the matrix M that will be used in the classification, M[Categories.Count, p+1]
float[,] M = new float[Categories.Count, p + 1];
float[] x0 = new float[p + 1];
for (int i = 0; i < p + 1; i++) x0[i] = 0.1f;
//Classification part
for (int i = 0; i < Categories.Count; i++)
{
//Composes y for this classifier
for (int j = 0; j < n; j++)
{
y.Values[j] = _classifs[j] == Categories[i] ? 1 : 0;
}
if (floatLinalg.UseOpenCLIfAvailable && CLCalc.CLAcceleration == CLCalc.CLAccelerationType.UsingCL) y.CLValues.WriteToDevice(y.Values);
int iters;
if (floatLinalg.UseOpenCLIfAvailable && CLCalc.CLAcceleration == CLCalc.CLAccelerationType.UsingCL) CLTheta.CLValues.WriteToDevice(x0);
else for (int k = 0; k < CLTheta.Length; k++) CLTheta.Values[k] = x0[k];
float[] beta = floatOptimization.UnconstrainedMinimization.Solve(CLTheta, RegularizedLogistReg, out iters, Hess, CLGrad, temp, CLDeltaTheta);
//Copies result to classification matrix M
for (int j = 0; j < beta.Length; j++) M[i, j] = beta[j];
}
//Creates classification matrix
if (CLM == null || CLM.Rows != M.GetLength(0) || CLM.Cols != M.GetLength(1)) CLM = new floatLinalg.floatMatrix(M);
else CLM.SetValues(M);
}
/// <summary>Creates a new logistic regression classifier from already trained coefficients.
/// Note: GetInternalHitRate will not work! (because there are no internal samples)</summary>
/// <param name="CLM">Coefficients, including interceptor coefficient</param>
/// <param name="Categories">Classification categories</param>
public LogisticRegression(floatLinalg.floatMatrix CLM, List<float> Categories)
{
int p = CLM.Cols - 1;
int nCategs = CLM.Rows;
if (Categories.Count != nCategs) throw new Exception("Number of categories and classification coefficients dimension not compatible");
this.CLM = CLM;
this.Categories = Categories;
}
public PNormMinClass(floatLinalg.floatMatrix A, floatLinalg.floatVector b, floatLinalg.floatVector w,
floatLinalg.floatVector lambda, floatLinalg.floatVector CLp, floatLinalg.floatVector CLq)
{
this.A = A;
this.b = b;
this.w = w;
this.lambda = lambda;
this.CLp = CLp;
this.CLq = CLq;
//Temporary buffers
tempX = new floatLinalg.floatVector(new float[A.Cols]);
tempdX = new floatLinalg.floatVector(new float[A.Cols]);
tempAxmb = new floatLinalg.floatVector(new float[A.Rows]);
Axmb = new floatLinalg.floatVector(new float[A.Rows]);
Diagw = new floatLinalg.floatDiag(w);
Diaglambda = new floatLinalg.floatDiag(lambda);
tempdX = new floatLinalg.floatVector(new float[lambda.Length]);
tempd2x = new floatLinalg.floatVector(new float[lambda.Length]);
tempAtz = new floatLinalg.floatVector(new float[lambda.Length]);
dtempAtz = new floatLinalg.floatVector(new float[w.Length]);
d2tempAtz = new floatLinalg.floatVector(new float[w.Length]);
Diagd2Atz = new floatLinalg.floatDiag(d2tempAtz);
float[] id = new float[A.Rows];
for (int i = 0; i < A.Rows; i++) id[i] = 1;
Identity = new floatLinalg.floatDiag(id);
}
/// <summary>Computes the p-norm minimization of Ax - b with weights w, using q-norm regularization with weights lambda on x</summary>
/// <param name="x0">Start point</param>
/// <param name="A">Dependent variables</param>
/// <param name="b">Independent variables measurements</param>
/// <param name="W">Weights of each equation</param>
/// <param name="lambda">Regularization term of each component of x</param>
/// <param name="p">Ax - b minimization exponent</param>
/// <param name="q">x regularization exponent</param>
/// <param name="AeqConstr">Equality constraint matrix AeqConstr * x = bConstr</param>
/// <param name="bEqConstr">Equality constraint right hand side</param>
public static float[] PNormMinimization(float[] x0, float[,] A, float[] b, float[] W, float[] lambda, float p, float q, float[,] AeqConstr, float[] bEqConstr)
{
//Dimensionality check
if (A.GetLength(1) != x0.Length) throw new Exception("Number of columns of A should be x.Length");
if (A.GetLength(0) != b.Length) throw new Exception("Number of rows of A should be b.Length");
if (A.GetLength(0) != W.Length) throw new Exception("Number of weights should be equal to b.Length");
if (A.GetLength(1) != lambda.Length) throw new Exception("Number of lambda weights should be equal to x.Length");
if (p < 1 || q < 1) throw new Exception("p and q values should be greater than 1");
if (AeqConstr != null && AeqConstr.GetLength(1) != x0.Length) throw new Exception("Number of columns of AeqConstr should be x.Length");
if (AeqConstr != null && bEqConstr.Length != AeqConstr.GetLength(0)) throw new Exception("bEqConstr.Length not compatible with AeqConstr");
float[] resp = null;
floatLinalg.floatMatrix CLA = new floatLinalg.floatMatrix(A);
//floatLinalg.floatVector CLx = new floatLinalg.floatVector(x0);
floatLinalg.floatVector CLb = new floatLinalg.floatVector(b);
floatLinalg.floatVector CLW = new floatLinalg.floatVector(W);
floatLinalg.floatVector CLlambda = new floatLinalg.floatVector(lambda);
floatLinalg.floatVector CLp = new floatLinalg.floatVector(new float[] { p });
floatLinalg.floatVector CLq = new floatLinalg.floatVector(new float[] { q });
PNormMinClass pnm = new PNormMinClass(CLA, CLb, CLW, CLlambda, CLp, CLq);
floatLinalg.floatVector x = new floatLinalg.floatVector(x0);
floatLinalg.floatVector g = new floatLinalg.floatVector(x0);
floatLinalg.floatSymPosDefMatrix h = new floatLinalg.floatSymPosDefMatrix(x0.Length);
//pnm.F(x, true, ref g, ref h);
int iters;
resp = UnconstrainedMinimization.Solve(x0, pnm.F, AeqConstr, bEqConstr, out iters);
return resp;
}
/// <summary>Computes least squares fitting of Ax = b weighted by W and returns x</summary>
/// <param name="A">Dependent variables measurements</param>
/// <param name="b">Independent variables measurements</param>
/// <param name="W">Weights</param>
/// <param name="lambda">Regularization term</param>
public static float[] LeastSquares(float[,] A, float[] b, float[] W, float lambda)
{
floatLinalg.floatMatrix CLA = new floatLinalg.floatMatrix(A);
floatLinalg.floatVector CLb = new floatLinalg.floatVector(b);
if (W != null && W.Length != CLA.Rows) throw new Exception("Incompatible Weight dimensions");
floatLinalg.floatDiag CLW;
if (W == null)
{
float[] ww = new float[CLA.Rows];
for (int i = 0; i < ww.Length; i++) ww[i] = 1;
CLW = new floatLinalg.floatDiag(ww);
}
else CLW = new floatLinalg.floatDiag(W);
float[] lambdas = new float[CLA.Cols];
for (int i = 0; i < lambdas.Length; i++) lambdas[i] = lambda;
floatLinalg.floatVector CLlambda = new floatLinalg.floatVector(lambdas);
floatLinalg.floatSymPosDefMatrix AtA = null;
AtA = floatLinalg.BLAS.MatrTranspMatrProd(CLA, CLW, CLlambda, ref AtA);
//CLA.CLValues.ReadFromDeviceTo(CLA.Values);
//AtA.CLValues.ReadFromDeviceTo(AtA.Values);
floatLinalg.floatVector Atb = null;
Atb = floatLinalg.BLAS.MatrTraspVecMult(CLA, CLW, CLb, ref Atb);
//Atb.CLValues.WriteToDevice(Atb.Values);
GC.Collect();
floatLinalg.floatVector resp = null;
AtA.LinearSolve(Atb, true, ref resp);
if (floatLinalg.UseOpenCLIfAvailable && CLCalc.CLAcceleration == CLCalc.CLAccelerationType.UsingCL) resp.CLValues.ReadFromDeviceTo(resp.Values);
return resp.Values;
}