/// <summary> Main algorithm. Descriptions see "Practical Optimization"
///
/// </summary>
/// <param name="initX">initial point of x, assuming no value's on the bound!
/// </param>
/// <param name="constraints">the bound constraints of each variable
/// constraints[0] is the lower bounds and
/// constraints[1] is the upper bounds
/// </param>
/// <returns> the solution of x, null if number of iterations not enough
/// </returns>
/// <exception cref="Exception">if an error occurs
/// </exception>
public virtual double[] findArgmin(double[] initX, double[][] constraints)
{
int l = initX.Length;
// Initially all variables are free, all bounds are constraints of
// non-working-set constraints
bool[] isFixed = new bool[l];
double[][] nwsBounds = new double[2][];
for (int i = 0; i < 2; i++)
{
nwsBounds[i] = new double[l];
}
// Record indice of fixed variables, simply for efficiency
DynamicIntArray wsBdsIndx = new DynamicIntArray(this, constraints.Length);
// Vectors used to record the variable indices to be freed
DynamicIntArray toFree = null, oldToFree = null;
// Initial value of obj. function, gradient and inverse of the Hessian
m_f = objectiveFunction(initX);
if (System.Double.IsNaN(m_f))
throw new System.Exception("Objective function value is NaN!");
double sum = 0;
double[] grad = evaluateGradient(initX), oldGrad, oldX, deltaGrad = new double[l], deltaX = new double[l], direct = new double[l], x = new double[l];
Matrix L = new Matrix(l, l); // Lower triangle of Cholesky factor
double[] D = new double[l]; // Diagonal of Cholesky factor
for (int i = 0; i < l; i++)
{
L.setRow(i, new double[l]);
L.setXmlElement(i, i, 1.0);
D[i] = 1.0;
direct[i] = - grad[i];
sum += grad[i] * grad[i];
x[i] = initX[i];
nwsBounds[0][i] = constraints[0][i];
nwsBounds[1][i] = constraints[1][i];
isFixed[i] = false;
}
double stpmax = m_STPMX * System.Math.Max(System.Math.Sqrt(sum), l);
iterates:
for (int step = 0; step < m_MAXITS; step++)
{
if (m_Debug)
System.Console.Error.WriteLine("\nIteration # " + step + ":");
// Try at most one feasible newton step, i.e. 0<lamda<=alpha
oldX = x;
oldGrad = grad;
// Also update grad
if (m_Debug)
System.Console.Error.WriteLine("Line search ... ");
m_IsZeroStep = false;
x = lnsrch(x, grad, direct, stpmax, isFixed, nwsBounds, wsBdsIndx);
if (m_Debug)
System.Console.Error.WriteLine("Line search finished.");
if (m_IsZeroStep)
{
// Zero step, simply delete rows/cols of D and L
for (int f = 0; f < wsBdsIndx.size(); f++)
{
int idx = wsBdsIndx.elementAt(f);
L.setRow(idx, new double[l]);
L.setColumn(idx, new double[l]);
D[idx] = 0.0;
}
grad = evaluateGradient(x);
step--;
}
else
{
// Check converge on x
bool finish = false;
double test = 0.0;
for (int h = 0; h < l; h++)
{
deltaX[h] = x[h] - oldX[h];
double tmp = System.Math.Abs(deltaX[h]) / System.Math.Max(System.Math.Abs(x[h]), 1.0);
if (tmp > test)
test = tmp;
}
if (test < m_Zero)
{
if (m_Debug)
System.Console.Error.WriteLine("\nDeltaX converge: " + test);
finish = true;
}
// Check zero gradient
grad = evaluateGradient(x);
test = 0.0;
double denom = 0.0, dxSq = 0.0, dgSq = 0.0, newlyBounded = 0.0;
for (int g = 0; g < l; g++)
{
if (!isFixed[g])
{
deltaGrad[g] = grad[g] - oldGrad[g];
// Calculate the denominators
denom += deltaX[g] * deltaGrad[g];
dxSq += deltaX[g] * deltaX[g];
dgSq += deltaGrad[g] * deltaGrad[g];
}
// Only newly bounded variables will be non-zero
else
newlyBounded += deltaX[g] * (grad[g] - oldGrad[g]);
// Note: CANNOT use projected gradient for testing
// convergence because of newly bounded variables
double tmp = System.Math.Abs(grad[g]) * System.Math.Max(System.Math.Abs(direct[g]), 1.0) / System.Math.Max(System.Math.Abs(m_f), 1.0);
if (tmp > test)
test = tmp;
}
if (test < m_Zero)
{
if (m_Debug)
System.Console.Error.WriteLine("Gradient converge: " + test);
finish = true;
}
// dg'*dx could be < 0 using inexact lnsrch
if (m_Debug)
System.Console.Error.WriteLine("dg'*dx=" + (denom + newlyBounded));
// dg'*dx = 0
if (System.Math.Abs(denom + newlyBounded) < m_Zero)
finish = true;
int size = wsBdsIndx.size();
bool isUpdate = true; // Whether to update BFGS formula
// Converge: check whether release any current constraints
if (finish)
{
if (m_Debug)
System.Console.Error.WriteLine("Test any release possible ...");
if (toFree != null)
oldToFree = (DynamicIntArray) toFree.copy();
toFree = new DynamicIntArray(this, wsBdsIndx.size());
for (int m = size - 1; m >= 0; m--)
{
int index = wsBdsIndx.elementAt(m);
double[] hessian = evaluateHessian(x, index);
double deltaL = 0.0;
if (hessian != null)
{
for (int mm = 0; mm < hessian.Length; mm++)
if (!isFixed[mm])
// Free variable
deltaL += hessian[mm] * direct[mm];
}
// First and second order Lagrangian multiplier estimate
// If user didn't provide Hessian, use first-order only
double L1, L2;
if (x[index] >= constraints[1][index])
// Upper bound
L1 = - grad[index];
else if (x[index] <= constraints[0][index])
// Lower bound
L1 = grad[index];
else
throw new System.Exception("x[" + index + "] not fixed on the" + " bounds where it should have been!");
// L2 = L1 + deltaL
L2 = L1 + deltaL;
if (m_Debug)
System.Console.Error.WriteLine("Variable " + index + ": Lagrangian=" + L1 + "|" + L2);
//Check validity of Lagrangian multiplier estimate
bool isConverge = (2.0 * System.Math.Abs(deltaL)) < System.Math.Min(System.Math.Abs(L1), System.Math.Abs(L2));
if ((L1 * L2 > 0.0) && isConverge)
{
//Same sign and converge: valid
if (L2 < 0.0)
{
// Negative Lagrangian: feasible
toFree.addElement(index);
wsBdsIndx.removeElementAt(m);
finish = false; // Not optimal, cannot finish
}
}
// Although hardly happen, better check it
// If the first-order Lagrangian multiplier estimate is wrong,
// avoid zigzagging
if ((hessian == null) && (toFree != null) && toFree.equal(oldToFree))
finish = true;
}
if (finish)
{
// Min. found
if (m_Debug)
System.Console.Error.WriteLine("Minimum found.");
m_f = objectiveFunction(x);
if (System.Double.IsNaN(m_f))
throw new System.Exception("Objective function value is NaN!");
return x;
}
// Free some variables
for (int mmm = 0; mmm < toFree.size(); mmm++)
{
int freeIndx = toFree.elementAt(mmm);
isFixed[freeIndx] = false; // Free this variable
if (x[freeIndx] <= constraints[0][freeIndx])
{
// Lower bound
nwsBounds[0][freeIndx] = constraints[0][freeIndx];
if (m_Debug)
System.Console.Error.WriteLine("Free variable " + freeIndx + " from bound " + nwsBounds[0][freeIndx]);
}
else
{
// Upper bound
nwsBounds[1][freeIndx] = constraints[1][freeIndx];
if (m_Debug)
System.Console.Error.WriteLine("Free variable " + freeIndx + " from bound " + nwsBounds[1][freeIndx]);
}
L.setXmlElement(freeIndx, freeIndx, 1.0);
D[freeIndx] = 1.0;
isUpdate = false;
}
}
if (denom < System.Math.Max(m_Zero * System.Math.Sqrt(dxSq) * System.Math.Sqrt(dgSq), m_Zero))
{
if (m_Debug)
System.Console.Error.WriteLine("dg'*dx negative!");
isUpdate = false; // Do not update
}
// If Hessian will be positive definite, update it
if (isUpdate)
{
// modify once: dg*dg'/(dg'*dx)
double coeff = 1.0 / denom; // 1/(dg'*dx)
updateCholeskyFactor(L, D, deltaGrad, coeff, isFixed);
// modify twice: g*g'/(g'*p)
coeff = 1.0 / m_Slope; // 1/(g'*p)
updateCholeskyFactor(L, D, oldGrad, coeff, isFixed);
}
}
// Find new direction
Matrix LD = new Matrix(l, l); // L*D
double[] b = new double[l];
for (int k = 0; k < l; k++)
{
if (!isFixed[k])
b[k] = - grad[k];
else
b[k] = 0.0;
for (int j = k; j < l; j++)
{
// Lower triangle
if (!isFixed[j] && !isFixed[k])
LD.setXmlElement(j, k, L.getXmlElement(j, k) * D[k]);
}
}
// Solve (LD)*y = -g, where y=L'*direct
double[] LDIR = solveTriangle(LD, b, true, isFixed);
LD = null;
for (int m = 0; m < LDIR.Length; m++)
{
if (System.Double.IsNaN(LDIR[m]))
throw new System.Exception("L*direct[" + m + "] is NaN!" + "|-g=" + b[m] + "|" + isFixed[m] + "|diag=" + D[m]);
}
// Solve L'*direct = y
direct = solveTriangle(L, LDIR, false, isFixed);
for (int m = 0; m < direct.Length; m++)
{
if (System.Double.IsNaN(direct[m]))
throw new System.Exception("direct is NaN!");
}
//System.gc();
}
if (m_Debug)
System.Console.Error.WriteLine("Cannot find minimum" + " -- too many interations!");
m_X = x;
return null;
}
