public static int newton
(Func <vector, vector> f, ref vector x, double eps = 1e-3, double dx = 1e-7)
{
vector fx = f(x), z, fz;
int nsteps = 0;
while (++nsteps < 999)
{
matrix J = jacobian(f, x, fx);
qrdecomposition qrJ = new qrdecomposition(J);
vector Dx = qrJ.solve(-fx);
double s = 1;
do // backtracking linesearch
{
z = x + Dx * s;
fz = f(z);
if (fz.norm() < (1 - s / 2) * fx.norm())
{
break;
}
if (s < 1.0 / 32)
{
break;
}
}while((s /= 2) > 1.0 / 64);
x = z;
fx = fz;
if (fx.norm() < eps)
{
break;
}
}
return(nsteps);
}//broyden
// The following method performs the inverse iteration method on a real symmetric matrix A
public static int inverse_iteration(matrix A, ref double s, ref vector v, double tau = 1e-6, double eps = 1e-6, int n_max = 999, int updates = 999)
{
int n = 0; int m = 0;
matrix As; matrix I = new matrix(A.size1, A.size1); I.set_identity();
v = v / v.norm();
As = A - s * I;
qr As_QR = new qr(As);
double abs = 0; double rel = 0;
while (converge(v, A, s, tau, eps, ref abs, ref rel) && n < n_max)
{
v = As_QR.solve(v);
v = v / v.norm();
s = v.dot(A * v);
if (m > updates) // Update QR decomposition if Rayleigh updates are used (if updates<999)
{
m = 0;
s = v.dot(A * v) / (v.dot(v));
As = A - s * I;
As_QR = new qr(As);
}
n++; m++;
}
s = v.dot(A * v) / (v.norm() * v.norm());
v = v / v.norm();
return(n);
}
public static vector newton(Func<vector,vector> f,
vector x, double eps=1e-3, double dx=1e-7)
{
double n=x.size;
vector fx=f(x),z,fz;
while(true){
matrix J=jacobian(f,x,fx);
var qrJ= new gs(J);
matrix B= qrJ.inverse();
vector Delta_x=-B*fx; //the newton step
double lambda=1;
while(true){
z=x+lambda*Delta_x;
fz=f(z);
if(fz.norm()<(1-lambda/2)*fx.norm()) break; //stop if the step is good
else if(lambda<1.0/32) break; //stop if minimum stepsize is reached
else lambda/=2; //backtrack by making a half step
}
x=z;
fx=fz;
if(fx.norm()<eps) break; //stop if tolerance is reached
else if(x.norm()<dx) break;
}
return x;
}// method Newton
public static vector newton(Func<vector, vector> f, vector x, double eps = 1e-3, double dx = 1e-7){
int n = x.size;
matrix J = new matrix(n,n);
vector root = x.copy();
vector f_r = f(root);
vector z,fz;
while(f_r.norm()>eps){
for(int i = 0; i<n; i++){
root[i] = root[i]+dx;
vector df = f(root)-f_r;
for(int j=0; j<n; j++){
J[i,j] = df[j]/dx;
} // for
root[i] -= dx;
} // for
QRdecompositionGS qr = new QRdecompositionGS(J);
vector dr = qr.solve(-1*f_r);
double lambda = 1.0;
z = root+lambda*dr;
fz = f(z);
while(fz.norm()>(1-lambda/2)*f_r.norm()){
double lambda_min = 1/64.0;
if(lambda>lambda_min){
lambda = lambda/2;
fz = f(root+lambda*dr);
} // if
else{Error.WriteLine($"Bad step: lambda = {lambda_min}");
break;
} // else
} // while
root += lambda*dr;
f_r = f(root);
} // while
return root;
} // newton
private static int limit = 5; // maximum number of times we halve the step size
/* find the roots of an equation.
* f: the equation
* x: the starting point
* epsilon: accuracy goal
* dx: the length-scale used to calculate the Jacobian
*/
public static vector newton(Func <vector, vector> f, vector x, double epsilon = 1e-3, double dx = 1e-7)
{
vector fx = f(x), s, fs;
while (fx.norm() > epsilon)
{
matrix J = jacobian(f, x, dx), R = new matrix(J.size2, J.size2);
qr_decomp.gs.decomp(J, R);
matrix B = qr_decomp.gs.inverse(J, R); // B = J^-1
vector Dx = -B *f(x);
double min = 1.0 / Pow(2, limit), l = 1.0; // l is lambda
do // backtracking
{
s = x + l * Dx; // current step
fs = f(s);
if (fs.norm() < (1 - l / 2) * fx.norm())
{
break; // accept the step
}
l /= 2; // halve the step size
} while (l > min);
x = s;
fx = fs;
}
return(x);
}
public static vector newton(Func<vector,vector> f, vector x, double eps=1e-3, double dx=1e-7){
int n=x.size;
vector fx=f(x);
vector z,fz;
while(true){
matrix J=jacobian(f,x,fx); // define the Jacobian matrix with partial derivatives
qrdecomposition qrJ=new qrdecomposition(J); // use QR-decomposition
matrix B=qrJ.inverse(); // construct the inverse of our QR-decomposition
vector Dx=-B*fx; // solve the Newton's algorithm in matrix form (using eq. 5)
double s=1;
while(true){ // backtracking linesearch
z=x+Dx*s; // creating left-hand side of eq. 8
fz=f(z); // creating left-hand side of eq. 8
if(fz.norm()<(1-s/2)*fx.norm()){ // stop if step is good
break;
}
if(s<1.0/32){ // stop if minimum step-size is reached
break;
}
s/=2; // backtrack using half the step-size
}
x=z;
fx=fz;
if(fx.norm()<eps)break; // until converged
}
return x;
}//newton
public static vector newton(Func <vector, vector> f, vector x_start, double eps = 1e-3, double dx = 1e-7)
{
vector x = x_start.copy();
int n = x.size;
matrix J = new matrix(n, n);
vector fx = new vector(n);
vector df = new vector(n);
vector y = new vector(n);
vector fy = new vector(n);
vector Dx = new vector(n);
bool bool1 = true;
do
{
bool bool2 = true;
fx = f(x);
for (int j = 0; j < n; j++)
{
x[j] += dx;
df = f(x) - fx;
for (int i = 0; i < n; i++)
{
J[i, j] = df[i] / dx;
}
x[j] -= dx;
}
qrDecomp.qr_givens_decomp(J);
Dx = qrDecomp.qr_givens_solve(J, fx * (-1));
double s = 2;
do
{
s /= 2;
y = x + Dx * s;
fy = f(y);
if (fy.norm() < (1 - s / 2) * fx.norm())
{
bool2 = false;
}
if (s < 0.02)
{
bool2 = false;
}
}while(bool2);
x = y;
fx = fy;
if (Dx.norm() < dx)
{
bool1 = false;
}
if (fx.norm() < eps)
{
bool1 = false;
}
}while(bool1);
return(x);
}
public static vector newton
(
Func <vector, vector> f, //takes the input vector x and retrun f(x)
vector xstart, //Starting point
double eps = 1e-3, //The accuracy goal, ||f(x)||< epsilon
double dx = 1e-6 //finite difference
)
{
vector x = xstart.copy();
int n = xstart.size;
vector fx = f(x);
vector y;
vector fy;
do
{
matrix J = new matrix(n, n);
for (int j = 0; j < n; j++)
{
x[j] += dx;
vector df = f(x) - fx;
for (int i = 0; i < n; i++)
{
J[i, j] = df[i] / dx;
}
x[j] -= dx;
}
var JQR = new qrdecompositionGS(J);
matrix B = JQR.inverse();
vector Dx = -B * fx;
double s = 1;
do
{
y = x + Dx * s;
fy = f(y);
if (fy.norm() < (1 - s / 2) * fx.norm())
{
break;
}
if (s < 1.0 / 32)
{
break;
}
s /= 2;
} while (true);
x = y;
fx = fy;
if (fx.norm() < eps)
{
break;
}
} while (true);
return(x);
}
}//broyden
public static int broyden
(Func <vector, vector> f, ref vector x, double eps = 1e-3)
{
vector fx = f(x), z, fz;
matrix J = jacobian(f, x, fx);
var qrJ = new qrdecomposition(J);
matrix B = qrJ.inverse();
int nsteps = 0;
while (++nsteps < 999)
{
vector Dx = -B * fx;
double s = 1;
while (true)
{
z = x + Dx * s;
fz = f(z);
if (fz.norm() < (1 - s / 2) * fx.norm())
{
break;
}
if (s < 1.0 / 32)
{
J = jacobian(f, x, fx);
qrJ = new qrdecomposition(J);
B = qrJ.inverse();
break;
}
s /= 2;
}
vector dx = z - x;
vector df = fz - fx;
if (dx.dot(df) > 1e-9)
{
vector c = (dx - B * df) / dx.dot(df);
B.update(c, dx);
}
//vector c=(dx-B*df)/(df%df); B.update(c,df);
//vector c=(dx-B*df)/(dx%(B*df)); B.update(c,B*df);
x = z;
fx = fz;
if (fx.norm() < eps)
{
break;
}
}
return(nsteps);
}//broyden
static void Main()
{
Func <vector, double> f1 = (z) => (1 - z[0]) * (1 - z[0]) + 100 * (z[1] - z[0] * z[0]) * (z[1] - z[0] * z[0]); // Rosenbrock's valley function (a=1,b=100)
double eps = 1e-4;
vector p = new vector("2 3"); // starting point
Write("SR1: Rosenbrock's valley function\n");
p.print("start point:");
int nsteps = qnewton.sr1(f1, ref p, eps);
WriteLine($"nsteps={nsteps}");
p.print("minimum: ");
Write($"f(x_min) = {(float)f1(p)}\n");
vector g = qnewton.gradient(f1, p);
Write($"|gradient| goal = {(float)eps}\n");
Write($"|gradient| actual = {(float)g.norm()}\n");
if (g.norm() < eps)
{
WriteLine("test passed");
}
else
{
WriteLine("test failed");
}
// Minimum at (1,1)
Func <vector, double> f2 = (z) => (z[0] * z[0] + z[1] - 11) * (z[0] * z[0] + z[1] - 11) + (z[0] + z[1] * z[1] - 7) * (z[0] + z[1] * z[1] - 7); // Himmelblau's function
eps = 1e-4;
p = new vector("0 1"); // starting point
Write("\nSR1: Himmelblau's function\n");
p.print("start point:");
nsteps = qnewton.sr1(f2, ref p, eps);
WriteLine($"nsteps={nsteps}");
p.print("minimum: ");
Write($"f(x_min) = {(float)f2(p)}\n");
g = qnewton.gradient(f, p);
Write($"|gradient| goal = {(float)eps}\n");
Write($"|gradient| actual = {(float)g.norm()}\n");
if (g.norm() < eps)
{
WriteLine("test passed");
}
else
{
WriteLine("test failed");
}
// Minima at (3,2), (-2.805118,3.131312), (-3.779310,-3.283186), and (3.584428,-1.848126)
}
public static void generate_errors(qr As_QR, ref matrix A, ref matrix I, ref List <double> errors, ref List <double> errors_s, double s, int updates, double e_J, vector v_0, double tau = 1e-6, double eps = 1e-6, double n_max = 999)
{
int n = 0; int m = 0;
matrix As;
vector u; vector v;
u = v_0 / v_0.norm();
double abs = 0; double rel = 0;
while (converge(u, A, s, tau, eps, ref abs, ref rel) && n < n_max)
{
v = As_QR.solve(u);
u = v / v.norm();
s = u.dot(A * u);
if (m > updates)
{
m = 0;
s = u.dot(A * u) / (u.dot(u));
As = A - s * I;
As_QR = new qr(As);
}
n++; m++; errors.Add(rel); errors_s.Add(Abs(s - e_J));
}
errors.Add(rel);
errors_s.Add(Abs(s - e_J));
s = u.dot(A * u) / (u.norm() * u.norm());
}
public static Tuple <vector, int> qnewton(Func <vector, double> f, vector x0, double acc = 1e-3, double eps = 1.0 / 4194304)
{
vector x = x0.copy();
double fx = f(x);
matrix B = new matrix(x.size, x.size);
B.set_unity();
int n = 0;
vector gx = gradient(f, x, eps);
while (n < 1000)
{
vector Dx = -B * gx;
n++;
if (gx.norm() < acc || Dx.norm() < eps * x.norm())
{
break;
}
double lam = 1.0;
// Commence backtracking
vector z;
double fz;
do
{
z = x + Dx * lam;
fz = f(z);
if (fz < fx)
{
break; // Defines a good step
}
if (lam < eps) // Here we just don't care
{
B.set_unity();
break;
}
lam /= 2;
}while(true);
vector s = z - x;
vector gz = gradient(f, z, eps);
vector y = gz - gx;
vector u = s - B * y;
double uTy = u.dot(y);
// Updating B
if (Abs(uTy) > 1e-6)
{
for (int i = 0; i < B.size1; i++)
{
for (int j = 0; j < B.size2; j++)
{
B[i, j] += u[i] * u[j] * (1 / uTy);
}
}
}
x = z;
gx = gz;
fx = fz;
}
return(Tuple.Create(x, n));
}
public static vector newton(Func <vector, vector> f, vector x, double eps = 1e-3, double dx = 1e-7)
{
matrix J = jacobian(f, x, dx);
qr Jqr = new qr(J);
vector deltax = Jqr.solve(-1.0 * f(x));
double a = 1;
// while((f(x) + a*deltax).norm() < (1-a/2)*f(x).norm() && a>1/64){a = a/2;}
while ((f(x + a * deltax).norm() > (1 - a / 2.0) * f(x).norm()) && a > 1.0 / 64)
{
a = a / 2.0;
}
x += a * deltax;
if (deltax.norm() < dx)
{
return(x);
}
else if (f(x).norm() < eps)
{
return(x);
}
else
{
return(newton(f, x, eps, dx));
}
}
public static vector newton(Func <vector, vector> f, vector x0, double eps = 1e-6, double dx = 1e-7)
{
// Prepare a bunch of stuff
int n = x0.size;
vector x = x0.copy();
vector fx = new vector(n);
vector df = new vector(n);
vector y = new vector(n);
vector fy = new vector(n);
vector Dx = new vector(n);
matrix J = new matrix(n, n);
matrix R = J.copy();
bool conin; // Condition for inner loop
bool conout; // Condition for outer loop
do
{
conin = true;
fx = f(x);
for (int i = 0; i < n; i++)
{
x[i] += dx;
df = f(x) - fx;
for (int j = 0; j < n; j++)
{
J[j, i] = df[j] / dx;
}
x[i] -= dx;
}
matrix.givens_qr(J);
Dx = matrix.givens_qr_solve(J, (-1) * fx);
double lam = 1.0; // Commence the backtracking
do
{
y = x + Dx * lam;
fy = f(y);
lam /= 2;
conin = (fy.norm() > (1 - lam / 2) * fx.norm() && lam > 1.0 / 128);
}while(conin);
x = y;
fx = fy;
conout = (Dx.norm() > dx && fx.norm() > eps);
}while(conout);
return(x);
}
public static int qnewton(Func <vector, double> f, ref vector x)
{
double eps = 1e-3;
// Hessian Matrix
int n = x.size;
matrix B = new matrix(n, n);
B.set_unity();
// Starting values
double fx = f(x);
vector gx = gradient(f, x);
vector delta_x;
int counts = 0;
do
{
counts++;
delta_x = -B * gx;
// Back-tracking linesearch
double fz, l = 1.0;
while (true)
{
fz = f(x + l * delta_x);
if (fz < fx)
{
break;
}
if (l < EPS)
{
B.set_unity();
break;
}
l = l / 2;
}
vector s = l * delta_x;
vector gz = gradient(f, x + s);
vector y = gz - gx;
vector u = s - B * y;
double udoty = u.dot(y);
if (Abs(udoty) > 10e-6)
{
B = B + outer(u, u) * 1 / udoty;
}
x += s;
gx = gz;
fx = fz;
} while(gx.norm() > eps && delta_x.norm() > EPS * x.norm());
return(counts);
}
public static int qnewton
(
Func <vector, double> f, /* objective function */
ref vector xstart, /* starting point */
double eps /* accuracy goal, on exit |gradient| should be <eps */
)
{
double fx = f(xstart);
vector grad_fx = gradient(f, xstart);
matrix A = matrix.id(xstart.size);
int n_steps = 0;
while (n_steps < 999)
{
n_steps++;
var Dx = -A * grad_fx;
if (Dx.norm() < EPS * xstart.norm())
{
break;
}
if (grad_fx.norm() < eps)
{
break;
}
double lam = 1.0;
vector y;
double fy;
while (true)
{
y = xstart + Dx * lam;
fy = f(y);
if (fy < fx)
{
break;
}
if (lam < EPS)
{
A.setid();
break;
}
lam /= 2;
}
vector s = y - xstart;
vector grad_fy = gradient(f, y);
vector z = grad_fy - grad_fx;
vector u = s - A * z;
double uTz = u.dot(z);
if (Abs(uTz) > 1e-6)
{
A.update(u, u, 1.0 / uTz);
}
xstart = y;
grad_fx = grad_fy;
fx = fy;
}
return(n_steps);
}
static vector driver()
{
double t = a;
vector yt = ya;
vector yh, dy;
double e, tol, hOld;
int s, nSteps = 0;
if (ts != null)
{
ts.Clear();
ts.Add(t);
}
if (ys != null)
{
ys.Clear();
ys.Add(yt);
}
while (t < b)
{
s = 0;
if (nSteps > nMax)
{
Error.Write("To many steps taken! \n");
return(yt);
}
do
{
if (t + h > b)
{
h = b - t;
}
vector[] trialStep = rkstep12(t, yt, h);
yh = trialStep[0];
dy = trialStep[1];
tol = (acc + err * yt.norm()) * Sqrt(h / (b - a));
e = dy.norm();
hOld = h;
h = h * Pow(tol / e, 0.25) * 0.95;
s++;
}while(e > tol);
Error.Write("Step taken with step size: {0} \n Number of bad steps: {1} \n", hOld, s - 1);
t += h;
yt = yh;
if (ts != null)
{
ts.Add(t);
}
if (ys != null)
{
ys.Add(yt);
}
nSteps++;
}
return(yt);
}//driver
public static (vector, int) qnewton(
Func <vector, double> f, /* objective function */
vector xstart, /* starting point */
double acc = 1e-3, /* accuracy */
double eps = 1e-7 /* small number epsilon */
)
{
vector x = xstart.copy();
double fx = f(x);
vector gx = gradient(f, x);
matrix B = matrix.id(x.size);
int nsteps = 0;
while (nsteps < 999)
{
nsteps++;
vector Dx = -B * gx;
if (Dx.norm() < eps * x.norm())
{
break;
}
if (gx.norm() < acc)
{
break;
}
vector z;
double fz, lambda = 1;
while (true) // backtracking linesearch
{
z = x + Dx * lambda;
fz = f(z);
if (fz < fx)
{
break;
}
if (lambda < eps)
{
B.setid();
break;
}
lambda /= 2;
}
vector s = z - x;
vector gz = gradient(f, z);
vector y = gz - gx;
vector u = s - B * y;
double uTy = u % y;
if (Abs(uTy) > 1e-6)
{
B.update(u, u, 1 / uTy); // rank-1 update
}
x = z;
gx = gz;
fx = fz;
}
return(x, nsteps);
}
public static vector newton(Func <vector, vector> f, vector x, double eps = 1e-3, double dx = 1e-7)
{
int n = x.size;
vector fx = f(x), z, fz;
while (true)
{
matrix J = jacobi(f, fx, x);
matrix R = new matrix(n, n);
QR.qr_gs_decomp(J, R);
matrix B = QR.qr_gs_inverse(J, R);
//B.print("inverse B");
vector del_x = -B * fx;
double lambda = 1;
while (true)
{
z = x + lambda * del_x;
fz = f(z);
if (fz.norm() < (1 - lambda / 2) * fx.norm())
{
break;
}
else if (lambda < 1.0 / 64)
{
break;
}
else
{
lambda /= 2;
}
}
x = z;
fx = fz;
if (fx.norm() < eps)
{
break;
}
else if (x.norm() < dx)
{
break;
}
}
return(x);
}
}//Method: gradient
public static vector sr1(Func <vector, double> f, vector x, double acc = 1e-6)
{
double fx = f(x);
vector gx = gradient(f, x);
matrix B = matrix.id(x.size);
int nsteps = 0;
while (nsteps < 999)
{
nsteps++;
vector delta_x = -B * gx;
if (delta_x.norm() < eps * x.norm())
{
Error.WriteLine($"broyden: |delta_x|<eps*|x|");
break;
}
if (gx.norm() < acc)
{
Error.WriteLine($"broyden: |gx|<acc");
break;
}
double fz, lambda = 1, alpha = 1e-4;
vector z, s;
while (true)
{
s = delta_x * lambda;
z = x + s;
fz = f(z);
if (fz < fx + alpha * s.dot(gx))
{
break;
} //Stop if step is good
if (lambda < eps)
{
B.setid(); break;
} //If step is too small: reset B and stop
lambda /= 2;
}
vector gz = gradient(f, z);
vector y = gz - gx;
vector u = s - B * y;
if (Abs(u.dot(y)) > 1e-6)
{
B += matrix.outer(u, u) / (u.dot(y));
}
x = z;
gx = gz;
fx = fz;
}
WriteLine($"nsteps={nsteps}");
return(x);
} //Method: sr1
public static vector qnewton(Func <vector, double> f, vector xstart, double eps)
{
double fx = f(xstart);
vector gx = gradient(f, xstart, eps);
int m = xstart.size;
matrix B = new matrix(m, m);
for (int i = 0; i < m; i++)
{
B[i, i] = 1;
}
//B.print("B");
int n = 0;
while (n < 500)
{
vector Dx = -B * gx;
n++;
if (gx.norm() < eps)
{
break;
}
double lambda = 1.0;
while (f(xstart + lambda * Dx) > fx)
{
if (lambda < eps)
{
for (int i = 0; i < m; i++)
{
B[i, i] = 1;
}
break;
}
lambda /= 2.0;
}
vector s = lambda * Dx;
vector gz = gradient(f, xstart + s, eps);
vector y = gz - gx;
vector u = s - B * y;
if (Abs(u % y) > 1e-6)
{
for (int i = 0; i < m; i++)
{
for (int j = 0; j < m; j++)
{
B[i, j] = B[i, j] + u[i] * u[j] * 1.0 / (u % y);
}
}
}
xstart = xstart + s;
gx = gz;
fx = f(xstart);
}
return(xstart);
}
public static (vector, int) qNewton(
Func <vector, double> f, // Function phi
vector x0, // Starting point
double eps // Desired accuracy
)
{
int steps = 0;
vector x = x0.copy();
matrix B = resetB(x.size);
vector dfdx = gradient(f, x);
while (dfdx.norm() > eps)
{
steps++;
// Calc dx
vector dx = -1 * B * dfdx;
// Backtracking search
double lambda = 1;
matrix dxT = new matrix(1, dx.size);
for (int i = 0; i < dx.size; i++)
{
dxT[0, i] = dx[i];
}
double armijo = (dxT * dfdx)[0]; // Armijo condition
while (f(x + lambda * dx) > f(x) + lambda * armijo / 10000)
{
if (lambda < 1.0 / 64)
{
B = resetB(x.size);
break;
}
lambda /= 2;
}
// SR1 update of B
vector y = gradient(f, x + lambda * dx) - dfdx;
vector u = lambda * dx - B * y;
matrix uT = new matrix(1, u.size);
for (int i = 0; i < u.size; i++)
{
uT[0, i] = u[i];
}
double uTy = (uT * y)[0];
if (Abs(uTy) > 1e-6)
{
B.update(u, u, 1 / uTy);
}
// Update x, dfdx
x = x + lambda * dx;
dfdx = gradient(f, x);
}
return(x, steps);
}
public static int sr1
(Func <vector, double> f, ref vector x, double acc = 1e-3)
{
double fx = f(x);
vector gx = gradient(f, x);
matrix B = matrix.id(x.size);
int nsteps = 0;
while (nsteps < 999)
{
nsteps++;
vector Dx = -B * gx;
if (Dx.norm() < EPS * x.norm())
{
Error.Write($"broyden: |Dx|<EPS*|x|\n");
break;
}
if (gx.norm() < acc)
{
Error.Write($"broyden: |gx|<acc\n");
break;
}
vector z;
double fz, lambda = 1;
while (true) // backtracking linesearch
{
z = x + Dx * lambda;
fz = f(z);
if (fz < fx)
{
break; // good step
}
if (lambda < EPS)
{
B.setid();
break; // accept anyway
}
lambda /= 2;
}
vector s = z - x;
vector gz = gradient(f, z);
vector y = gz - gx;
vector u = s - B * y;
double uTy = u % y;
if (Abs(uTy) > 1e-6)
{
B.update(u, u, 1 / uTy); // SR1 update
}
x = z;
gx = gz;
fx = fz;
}
return(nsteps);
} //SR1
//function to find roots of f. x is the start guess..
public static vector newton(Func <vector, vector> f, vector x, double eps = 1e-3, double dx = 1e-7)
{
bool seed2 = true;
while (seed2)
{
//seed2=false;
//First the jacobian matrix J is calculated:
vector fx = f(x);
matrix J = new matrix(x.size, x.size);
for (int k = 0; k < x.size; k++) // for-loop over vector x
{
x[k] = x[k] + dx; //change x
vector df = (f(x) - fx) / dx; // the differnce in f(x)
for (int i = 0; i < x.size; i++)
{
J[i, k] = df[i]; // calculate the matrix element
} //end for
x[k] = x[k] - dx; // reset x
} // end for
// solving J*DX=-f(x)
gram_s GJ = new gram_s(J);
vector Dx = GJ.solve((-1) * fx); // using my own method to solve the equation ...
//calculating the step size s
double s = 2;
bool seed1 = true;
while (seed1)
{
s = s / 2;
if ((f(x + s * Dx).norm() < (1 - s / 2) * f(x).norm()) && (s < 1.0 / 64))
{
seed1 = false;
} //end if
} //end while
x = x + s * Dx; //calculate the new x!
//if((Dx.norm()<dx) && (f(x).norm() < eps)){
if (Dx.norm() < dx)
{
seed2 = false;
} //end if
else if (f(x).norm() < eps)
{
seed2 = false;
} //end else if
//seed2=false;
}//end while
return(x);
} //end newton...
public static vector makeRandUnitVector(int n)
{
var rand = new System.Random();
vector v = new vector(n);
for (int i = 0; i < n; i++)
{
v[i] = rand.NextDouble();
}
return(v / v.norm());
}
public static vector newton
(Func <vector, vector> f, vector x, double eps = 1e-3, double dx = 1e-7)
{
int n = x.size;
vector fx = f(x), z, fz;
while (true)
{
matrix J = jacobian(f, x, fx);
qrdecomposition qrJ = new qrdecomposition(J);
matrix B = qrJ.inverse();
vector Dx = -B * fx;
double s = 1;
while (true) // backtracking linesearch
{
z = x + Dx * s;
fz = f(z);
if (fz.norm() < (1 - s / 2) * fx.norm())
{
break;
}
if (s < 1.0 / 32)
{
break;
}
s /= 2;
}
x = z;
fx = fz;
if (fx.norm() < eps)
{
break;
}
}
return(x);
} //broyden
}//constructor
public void gkl(matrix A)
{
int m = A.size1;
int n = A.size2;
double alpha, beta;
beta = 0;
vector v = new vector(n);
vector u = new vector(m);
for (int i = 0; i < n; i++)
{
v[i] = 1.0 / Sqrt(n);
}
for (int i = 0; i < m; i++)
{
u[i] = 0;
}
U = new matrix(m, n);
V = new matrix(n, n);
B = new matrix(n, n);
for (int k = 0; k < n; k++)
{
for (int i = 0; i < n; i++)
{
V[i, k] = v[i];
}
u = (A * v) - beta * u;
alpha = u.norm();
u /= alpha;
for (int i = 0; i < m; i++)
{
U[i, k] = u[i];
}
v = (A.transpose() * u) - alpha * v;
beta = v.norm();
v /= beta;
B[k, k] = alpha;
if (k < n - 1)
{
B[k, k + 1] = beta;
}
}
}//gkl
public static (vector, int) qnewton(Func <vector, double> f, vector x0, double eps)
{
int n = 0; // Number of steps
vector x = x0.copy(); // Position vector
vector s; // Position vector
double fx = f(x); // Function value
double fxs; // Function value
vector gx = gradient(f, x); // Gradient vector
vector gxs; // Gradient vector
double a = 1e-4; // Alpha in Armijo condition
matrix B = matrix.id(x.size); // Inverse Hessian, initially set to the identity matrix
while (eps < gx.norm()) // The accuracy goal
{
n++;
vector Dx = -B * gx; // Equation 6
double lambda = 1;
s = lambda * Dx; // Equation 8
fxs = f(x + s);
while (!(fxs < fx + a * s.dot(gx))) // Backtracking
{
s = lambda * Dx; // Equation 8
fxs = f(x + s);
if (lambda < 1.0 / Pow(2, 5))
{
B = matrix.id(x.size); // Reset B if lambda becomes to small
break;
}
lambda /= 2; // Halve the step size
}
gxs = gradient(f, x + s);
vector y = gxs - gx; // Statement after eq 12
vector u = s - B * y; // Statement after eq 12
double uty = u.dot(y); // Denominatior of eq 18
if (Abs(uty) > eps) // Condition for eq 18
{
B.update(u, u, 1 / uty); // SR1 update
}
// Prepare for next iteration
x = x + s;
gx = gxs;
fx = fxs;
}
return(x, n); // Return the vector x, and the number of steps taken
}
}//gradient
public static int sr1(Func<vector,double> f, ref vector x, double acc=1e-3){ //symmetric-rank-1 update
double fx=f(x);
vector gx=gradient(f,x); //the gradient of f
matrix B=matrix.id(x.size); //identity matrix, B
int nsteps=0;
while(nsteps<999){
nsteps++;
vector Dx=-B*gx; //the gradient of the objective function at x (eq. 6), i.e., the Newton's step
if(Dx.norm()<EPS*x.norm()){
Error.Write($"broyden: |Dx|<EPS*|x|\n"); //stop if the denominator is too small?
break;
}
if(gx.norm()<acc){
Error.Write($"broyden: |gx|<acc\n"); //stop if the denominator is too small?
break;
}
vector z;
double fz;
double lambda=1;
while(true){ //backtracking linesearch
z=x+Dx*lambda; //x+s, where s=λ*∆x (i.e, lambda*Dx)
fz=f(z); //φ(x+s)
if(fz<fx){ //(eq. 9)
break; //good step
}
if(lambda<EPS){
B.setid();
break; //accept anyway
}
lambda/=2; //backtrack using half the step-size
}
vector s=z-x;
vector gz=gradient(f,z); //∇φ(x+s) (eq. 10)
vector y=gz-gx; //y = ∇φ(x+s)- ∇φ(x)
vector u=s-B*y; //u = s-B*y
double uTy=u%y; //demoninator of eq. 18, u^T*y
if(Abs(uTy)>1e-6){ //we make sure that the denominator is not too small
B.update(u,u,1/uTy); //SR1 update (eq. 18)
}
x=z; //
gx=gz; //
fx=fz; //
}
return nsteps;
}//SR1
public static vector qnewton(Func<vector, double> f, vector xstart, double acc=1e-7){
int nsteps = 0, n = xstart.size;
vector x = xstart;
matrix B = new matrix(n,n);
B.set_identity();
vector g = gradient(f,x);
vector dx = -B*g;
// Lineseach
while(nsteps<999){
if(g.norm() < acc){
Error.WriteLine("Norm of Gradient < accuracy goal");
break;
} // if
if(dx.norm() < eps*x.norm()){
Error.WriteLine("|dx| < eps*|x|");
break;
} // if
double fx = f(x), lambda = 1;
vector increm = lambda*dx;
while(f(x+increm) > fx){
lambda /= 2;
increm = lambda*dx;
if(lambda < eps){
B.set_identity();
break;
} // if
} // while
nsteps++;
vector y = gradient(f, x+increm);
vector dy = y - g;
vector u = increm-B*dy;
double uTdy = u%dy;
if(Abs(increm.dot(dy)) > eps){
B.update(u,u,1/uTdy);
}// if
x += increm;
fx = f(x);
g = y;
dx = -B*g;
} // while
return x;
} // qnewton
