public static int Main(string[] args)
{
int n = 0;
int m = 0;
minlbfgs.minlbfgsstate state = new minlbfgs.minlbfgsstate();
minlbfgs.minlbfgsreport rep = new minlbfgs.minlbfgsreport();
double[] s = new double[0];
double x = 0;
double y = 0;
//
// Function minimized:
// F = exp(x-1) + exp(1-x) + (y-x)^2
// N = 2 - task dimension
// M = 1 - build tank-1 model
//
n = 2;
m = 1;
s = new double[2];
s[0] = AP.Math.RandomReal() - 0.5;
s[1] = AP.Math.RandomReal() - 0.5;
minlbfgs.minlbfgscreate(n, m, ref s, ref state);
minlbfgs.minlbfgssetcond(ref state, 0.0, 0.0, 0.0001, 0);
while (minlbfgs.minlbfgsiteration(ref state))
{
if (state.needfg)
{
x = state.x[0];
y = state.x[1];
state.f = Math.Exp(x - 1) + Math.Exp(1 - x) + AP.Math.Sqr(y - x);
state.g[0] = Math.Exp(x - 1) - Math.Exp(1 - x) + 2 * (x - y);
state.g[1] = 2 * (y - x);
}
}
minlbfgs.minlbfgsresults(ref state, ref s, ref rep);
//
// output results
//
System.Console.WriteLine();
System.Console.WriteLine();
System.Console.Write("F = exp(x-1) + exp(1-x) + (y-x)^2");
System.Console.WriteLine();
System.Console.Write("X = ");
System.Console.Write("{0,4:F2}", s[0]);
System.Console.Write(" (should be 1.00)");
System.Console.WriteLine();
System.Console.Write("Y = ");
System.Console.Write("{0,4:F2}", s[1]);
System.Console.Write(" (should be 1.00)");
System.Console.WriteLine();
System.Console.WriteLine();
System.Console.WriteLine();
return(0);
}
public static int Main(string[] args)
{
int n = 0;
int m = 0;
minlbfgs.minlbfgsstate state = new minlbfgs.minlbfgsstate();
minlbfgs.minlbfgsreport rep = new minlbfgs.minlbfgsreport();
double[] s = new double[0];
double x = 0;
double y = 0;
//
// Function minimized:
// F = exp(x-1) + exp(1-x) + (y-x)^2
// N = 2 - task dimension
// M = 1 - build tank-1 model
//
n = 2;
m = 1;
s = new double[2];
s[0] = AP.Math.RandomReal()-0.5;
s[1] = AP.Math.RandomReal()-0.5;
minlbfgs.minlbfgscreate(n, m, ref s, ref state);
minlbfgs.minlbfgssetcond(ref state, 0.0, 0.0, 0.0001, 0);
while( minlbfgs.minlbfgsiteration(ref state) )
{
if( state.needfg )
{
x = state.x[0];
y = state.x[1];
state.f = Math.Exp(x-1)+Math.Exp(1-x)+AP.Math.Sqr(y-x);
state.g[0] = Math.Exp(x-1)-Math.Exp(1-x)+2*(x-y);
state.g[1] = 2*(y-x);
}
}
minlbfgs.minlbfgsresults(ref state, ref s, ref rep);
//
// output results
//
System.Console.WriteLine();
System.Console.WriteLine();
System.Console.Write("F = exp(x-1) + exp(1-x) + (y-x)^2");
System.Console.WriteLine();
System.Console.Write("X = ");
System.Console.Write("{0,4:F2}",s[0]);
System.Console.Write(" (should be 1.00)");
System.Console.WriteLine();
System.Console.Write("Y = ");
System.Console.Write("{0,4:F2}",s[1]);
System.Console.Write(" (should be 1.00)");
System.Console.WriteLine();
System.Console.WriteLine();
System.Console.WriteLine();
return 0;
}
public override void init()
{
s = new double[0];
bndl = new double[0];
bndu = new double[0];
hasbndl = new bool[0];
hasbndu = new bool[0];
cleic = new double[0,0];
x = new double[0];
fi = new double[0];
j = new double[0,0];
rstate = new rcommstate();
rstateaul = new rcommstate();
scaledbndl = new double[0];
scaledbndu = new double[0];
scaledcleic = new double[0,0];
xc = new double[0];
xstart = new double[0];
xbase = new double[0];
fbase = new double[0];
dfbase = new double[0];
fm2 = new double[0];
fm1 = new double[0];
fp1 = new double[0];
fp2 = new double[0];
dfm1 = new double[0];
dfp1 = new double[0];
bufd = new double[0];
bufc = new double[0];
bufw = new double[0,0];
xk = new double[0];
xk1 = new double[0];
gk = new double[0];
gk1 = new double[0];
auloptimizer = new minlbfgs.minlbfgsstate();
aulreport = new minlbfgs.minlbfgsreport();
nubc = new double[0];
nulc = new double[0];
nunlc = new double[0];
}
public override void init()
{
x = new double[0];
fi = new double[0];
j = new double[0,0];
h = new double[0,0];
g = new double[0];
xbase = new double[0];
fibase = new double[0];
gbase = new double[0];
quadraticmodel = new double[0,0];
bndl = new double[0];
bndu = new double[0];
havebndl = new bool[0];
havebndu = new bool[0];
s = new double[0];
xdir = new double[0];
deltax = new double[0];
deltaf = new double[0];
rstate = new rcommstate();
choleskybuf = new double[0];
tmp0 = new double[0];
fm1 = new double[0];
fp1 = new double[0];
fc1 = new double[0];
gm1 = new double[0];
gp1 = new double[0];
gc1 = new double[0];
internalstate = new minlbfgs.minlbfgsstate();
internalrep = new minlbfgs.minlbfgsreport();
qpstate = new minqp.minqpstate();
qprep = new minqp.minqpreport();
}
public minlbfgsreport(minlbfgs.minlbfgsreport obj)
{
_innerobj = obj;
}
public minlbfgsreport()
{
_innerobj = new minlbfgs.minlbfgsreport();
}
/*************************************************************************
Neural network training using early stopping (base algorithm - L-BFGS with
regularization).
INPUT PARAMETERS:
Network - neural network with initialized geometry
TrnXY - training set
TrnSize - training set size, TrnSize>0
ValXY - validation set
ValSize - validation set size, ValSize>0
Decay - weight decay constant, >=0.001
Decay term 'Decay*||Weights||^2' is added to error
function.
If you don't know what Decay to choose, use 0.001.
Restarts - number of restarts, either:
* strictly positive number - algorithm make specified
number of restarts from random position.
* -1, in which case algorithm makes exactly one run
from the initial state of the network (no randomization).
If you don't know what Restarts to choose, choose one
one the following:
* -1 (deterministic start)
* +1 (one random restart)
* +5 (moderate amount of random restarts)
OUTPUT PARAMETERS:
Network - trained neural network.
Info - return code:
* -2, if there is a point with class number
outside of [0..NOut-1].
* -1, if wrong parameters specified
(NPoints<0, Restarts<1, ...).
* 2, task has been solved, stopping criterion met -
sufficiently small step size. Not expected (we
use EARLY stopping) but possible and not an
error.
* 6, task has been solved, stopping criterion met -
increasing of validation set error.
Rep - training report
NOTE:
Algorithm stops if validation set error increases for a long enough or
step size is small enought (there are task where validation set may
decrease for eternity). In any case solution returned corresponds to the
minimum of validation set error.
-- ALGLIB --
Copyright 10.03.2009 by Bochkanov Sergey
*************************************************************************/
public static void mlptraines(mlpbase.multilayerperceptron network,
double[,] trnxy,
int trnsize,
double[,] valxy,
int valsize,
double decay,
int restarts,
ref int info,
mlpreport rep)
{
int i = 0;
int pass = 0;
int nin = 0;
int nout = 0;
int wcount = 0;
double[] w = new double[0];
double[] wbest = new double[0];
double e = 0;
double v = 0;
double ebest = 0;
double[] wfinal = new double[0];
double efinal = 0;
int itcnt = 0;
int itbest = 0;
minlbfgs.minlbfgsreport internalrep = new minlbfgs.minlbfgsreport();
minlbfgs.minlbfgsstate state = new minlbfgs.minlbfgsstate();
double wstep = 0;
bool needrandomization = new bool();
int i_ = 0;
info = 0;
wstep = 0.001;
//
// Test inputs, parse flags, read network geometry
//
if( ((trnsize<=0 || valsize<=0) || (restarts<1 && restarts!=-1)) || (double)(decay)<(double)(0) )
{
info = -1;
return;
}
if( restarts==-1 )
{
needrandomization = false;
restarts = 1;
}
else
{
needrandomization = true;
}
mlpbase.mlpproperties(network, ref nin, ref nout, ref wcount);
if( mlpbase.mlpissoftmax(network) )
{
for(i=0; i<=trnsize-1; i++)
{
if( (int)Math.Round(trnxy[i,nin])<0 || (int)Math.Round(trnxy[i,nin])>=nout )
{
info = -2;
return;
}
}
for(i=0; i<=valsize-1; i++)
{
if( (int)Math.Round(valxy[i,nin])<0 || (int)Math.Round(valxy[i,nin])>=nout )
{
info = -2;
return;
}
}
}
info = 2;
//
// Prepare
//
mlpbase.mlpinitpreprocessor(network, trnxy, trnsize);
w = new double[wcount-1+1];
wbest = new double[wcount-1+1];
wfinal = new double[wcount-1+1];
efinal = math.maxrealnumber;
for(i=0; i<=wcount-1; i++)
{
wfinal[i] = 0;
}
//
// Multiple starts
//
rep.ncholesky = 0;
rep.nhess = 0;
rep.ngrad = 0;
for(pass=1; pass<=restarts; pass++)
{
//
// Process
//
if( needrandomization )
{
mlpbase.mlprandomize(network);
}
ebest = mlpbase.mlperror(network, valxy, valsize);
for(i_=0; i_<=wcount-1;i_++)
{
wbest[i_] = network.weights[i_];
}
itbest = 0;
itcnt = 0;
for(i_=0; i_<=wcount-1;i_++)
{
w[i_] = network.weights[i_];
}
minlbfgs.minlbfgscreate(wcount, Math.Min(wcount, 10), w, state);
minlbfgs.minlbfgssetcond(state, 0.0, 0.0, wstep, 0);
minlbfgs.minlbfgssetxrep(state, true);
while( minlbfgs.minlbfgsiteration(state) )
{
//
// Calculate gradient
//
if( state.needfg )
{
for(i_=0; i_<=wcount-1;i_++)
{
network.weights[i_] = state.x[i_];
}
mlpbase.mlpgradnbatch(network, trnxy, trnsize, ref state.f, ref state.g);
v = 0.0;
for(i_=0; i_<=wcount-1;i_++)
{
v += network.weights[i_]*network.weights[i_];
}
state.f = state.f+0.5*decay*v;
for(i_=0; i_<=wcount-1;i_++)
{
state.g[i_] = state.g[i_] + decay*network.weights[i_];
}
rep.ngrad = rep.ngrad+1;
}
//
// Validation set
//
if( state.xupdated )
{
for(i_=0; i_<=wcount-1;i_++)
{
network.weights[i_] = state.x[i_];
}
e = mlpbase.mlperror(network, valxy, valsize);
if( (double)(e)<(double)(ebest) )
{
ebest = e;
for(i_=0; i_<=wcount-1;i_++)
{
wbest[i_] = network.weights[i_];
}
itbest = itcnt;
}
if( itcnt>30 && (double)(itcnt)>(double)(1.5*itbest) )
{
info = 6;
break;
}
itcnt = itcnt+1;
}
}
minlbfgs.minlbfgsresults(state, ref w, internalrep);
//
// Compare with final answer
//
if( (double)(ebest)<(double)(efinal) )
{
for(i_=0; i_<=wcount-1;i_++)
{
wfinal[i_] = wbest[i_];
}
efinal = ebest;
}
}
//
// The best network
//
for(i_=0; i_<=wcount-1;i_++)
{
network.weights[i_] = wfinal[i_];
}
}
/*************************************************************************
Neural network training using L-BFGS algorithm with regularization.
Subroutine trains neural network with restarts from random positions.
Algorithm is well suited for problems of any dimensionality (memory
requirements and step complexity are linear by weights number).
INPUT PARAMETERS:
Network - neural network with initialized geometry
XY - training set
NPoints - training set size
Decay - weight decay constant, >=0.001
Decay term 'Decay*||Weights||^2' is added to error
function.
If you don't know what Decay to choose, use 0.001.
Restarts - number of restarts from random position, >0.
If you don't know what Restarts to choose, use 2.
WStep - stopping criterion. Algorithm stops if step size is
less than WStep. Recommended value - 0.01. Zero step
size means stopping after MaxIts iterations.
MaxIts - stopping criterion. Algorithm stops after MaxIts
iterations (NOT gradient calculations). Zero MaxIts
means stopping when step is sufficiently small.
OUTPUT PARAMETERS:
Network - trained neural network.
Info - return code:
* -8, if both WStep=0 and MaxIts=0
* -2, if there is a point with class number
outside of [0..NOut-1].
* -1, if wrong parameters specified
(NPoints<0, Restarts<1).
* 2, if task has been solved.
Rep - training report
-- ALGLIB --
Copyright 09.12.2007 by Bochkanov Sergey
*************************************************************************/
public static void mlptrainlbfgs(mlpbase.multilayerperceptron network,
double[,] xy,
int npoints,
double decay,
int restarts,
double wstep,
int maxits,
ref int info,
mlpreport rep)
{
int i = 0;
int pass = 0;
int nin = 0;
int nout = 0;
int wcount = 0;
double[] w = new double[0];
double[] wbest = new double[0];
double e = 0;
double v = 0;
double ebest = 0;
minlbfgs.minlbfgsreport internalrep = new minlbfgs.minlbfgsreport();
minlbfgs.minlbfgsstate state = new minlbfgs.minlbfgsstate();
int i_ = 0;
info = 0;
//
// Test inputs, parse flags, read network geometry
//
if( (double)(wstep)==(double)(0) && maxits==0 )
{
info = -8;
return;
}
if( ((npoints<=0 || restarts<1) || (double)(wstep)<(double)(0)) || maxits<0 )
{
info = -1;
return;
}
mlpbase.mlpproperties(network, ref nin, ref nout, ref wcount);
if( mlpbase.mlpissoftmax(network) )
{
for(i=0; i<=npoints-1; i++)
{
if( (int)Math.Round(xy[i,nin])<0 || (int)Math.Round(xy[i,nin])>=nout )
{
info = -2;
return;
}
}
}
decay = Math.Max(decay, mindecay);
info = 2;
//
// Prepare
//
mlpbase.mlpinitpreprocessor(network, xy, npoints);
w = new double[wcount-1+1];
wbest = new double[wcount-1+1];
ebest = math.maxrealnumber;
//
// Multiple starts
//
rep.ncholesky = 0;
rep.nhess = 0;
rep.ngrad = 0;
for(pass=1; pass<=restarts; pass++)
{
//
// Process
//
mlpbase.mlprandomize(network);
for(i_=0; i_<=wcount-1;i_++)
{
w[i_] = network.weights[i_];
}
minlbfgs.minlbfgscreate(wcount, Math.Min(wcount, 10), w, state);
minlbfgs.minlbfgssetcond(state, 0.0, 0.0, wstep, maxits);
while( minlbfgs.minlbfgsiteration(state) )
{
for(i_=0; i_<=wcount-1;i_++)
{
network.weights[i_] = state.x[i_];
}
mlpbase.mlpgradnbatch(network, xy, npoints, ref state.f, ref state.g);
v = 0.0;
for(i_=0; i_<=wcount-1;i_++)
{
v += network.weights[i_]*network.weights[i_];
}
state.f = state.f+0.5*decay*v;
for(i_=0; i_<=wcount-1;i_++)
{
state.g[i_] = state.g[i_] + decay*network.weights[i_];
}
rep.ngrad = rep.ngrad+1;
}
minlbfgs.minlbfgsresults(state, ref w, internalrep);
for(i_=0; i_<=wcount-1;i_++)
{
network.weights[i_] = w[i_];
}
//
// Compare with best
//
v = 0.0;
for(i_=0; i_<=wcount-1;i_++)
{
v += network.weights[i_]*network.weights[i_];
}
e = mlpbase.mlperrorn(network, xy, npoints)+0.5*decay*v;
if( (double)(e)<(double)(ebest) )
{
for(i_=0; i_<=wcount-1;i_++)
{
wbest[i_] = network.weights[i_];
}
ebest = e;
}
}
//
// The best network
//
for(i_=0; i_<=wcount-1;i_++)
{
network.weights[i_] = wbest[i_];
}
}
/*************************************************************************
Neural network training using modified Levenberg-Marquardt with exact
Hessian calculation and regularization. Subroutine trains neural network
with restarts from random positions. Algorithm is well suited for small
and medium scale problems (hundreds of weights).
INPUT PARAMETERS:
Network - neural network with initialized geometry
XY - training set
NPoints - training set size
Decay - weight decay constant, >=0.001
Decay term 'Decay*||Weights||^2' is added to error
function.
If you don't know what Decay to choose, use 0.001.
Restarts - number of restarts from random position, >0.
If you don't know what Restarts to choose, use 2.
OUTPUT PARAMETERS:
Network - trained neural network.
Info - return code:
* -9, if internal matrix inverse subroutine failed
* -2, if there is a point with class number
outside of [0..NOut-1].
* -1, if wrong parameters specified
(NPoints<0, Restarts<1).
* 2, if task has been solved.
Rep - training report
-- ALGLIB --
Copyright 10.03.2009 by Bochkanov Sergey
*************************************************************************/
public static void mlptrainlm(mlpbase.multilayerperceptron network,
double[,] xy,
int npoints,
double decay,
int restarts,
ref int info,
mlpreport rep)
{
int nin = 0;
int nout = 0;
int wcount = 0;
double lmftol = 0;
double lmsteptol = 0;
int i = 0;
int k = 0;
double v = 0;
double e = 0;
double enew = 0;
double xnorm2 = 0;
double stepnorm = 0;
double[] g = new double[0];
double[] d = new double[0];
double[,] h = new double[0,0];
double[,] hmod = new double[0,0];
double[,] z = new double[0,0];
bool spd = new bool();
double nu = 0;
double lambdav = 0;
double lambdaup = 0;
double lambdadown = 0;
minlbfgs.minlbfgsreport internalrep = new minlbfgs.minlbfgsreport();
minlbfgs.minlbfgsstate state = new minlbfgs.minlbfgsstate();
double[] x = new double[0];
double[] y = new double[0];
double[] wbase = new double[0];
double[] wdir = new double[0];
double[] wt = new double[0];
double[] wx = new double[0];
int pass = 0;
double[] wbest = new double[0];
double ebest = 0;
int invinfo = 0;
matinv.matinvreport invrep = new matinv.matinvreport();
int solverinfo = 0;
densesolver.densesolverreport solverrep = new densesolver.densesolverreport();
int i_ = 0;
info = 0;
mlpbase.mlpproperties(network, ref nin, ref nout, ref wcount);
lambdaup = 10;
lambdadown = 0.3;
lmftol = 0.001;
lmsteptol = 0.001;
//
// Test for inputs
//
if( npoints<=0 || restarts<1 )
{
info = -1;
return;
}
if( mlpbase.mlpissoftmax(network) )
{
for(i=0; i<=npoints-1; i++)
{
if( (int)Math.Round(xy[i,nin])<0 || (int)Math.Round(xy[i,nin])>=nout )
{
info = -2;
return;
}
}
}
decay = Math.Max(decay, mindecay);
info = 2;
//
// Initialize data
//
rep.ngrad = 0;
rep.nhess = 0;
rep.ncholesky = 0;
//
// General case.
// Prepare task and network. Allocate space.
//
mlpbase.mlpinitpreprocessor(network, xy, npoints);
g = new double[wcount-1+1];
h = new double[wcount-1+1, wcount-1+1];
hmod = new double[wcount-1+1, wcount-1+1];
wbase = new double[wcount-1+1];
wdir = new double[wcount-1+1];
wbest = new double[wcount-1+1];
wt = new double[wcount-1+1];
wx = new double[wcount-1+1];
ebest = math.maxrealnumber;
//
// Multiple passes
//
for(pass=1; pass<=restarts; pass++)
{
//
// Initialize weights
//
mlpbase.mlprandomize(network);
//
// First stage of the hybrid algorithm: LBFGS
//
for(i_=0; i_<=wcount-1;i_++)
{
wbase[i_] = network.weights[i_];
}
minlbfgs.minlbfgscreate(wcount, Math.Min(wcount, 5), wbase, state);
minlbfgs.minlbfgssetcond(state, 0, 0, 0, Math.Max(25, wcount));
while( minlbfgs.minlbfgsiteration(state) )
{
//
// gradient
//
for(i_=0; i_<=wcount-1;i_++)
{
network.weights[i_] = state.x[i_];
}
mlpbase.mlpgradbatch(network, xy, npoints, ref state.f, ref state.g);
//
// weight decay
//
v = 0.0;
for(i_=0; i_<=wcount-1;i_++)
{
v += network.weights[i_]*network.weights[i_];
}
state.f = state.f+0.5*decay*v;
for(i_=0; i_<=wcount-1;i_++)
{
state.g[i_] = state.g[i_] + decay*network.weights[i_];
}
//
// next iteration
//
rep.ngrad = rep.ngrad+1;
}
minlbfgs.minlbfgsresults(state, ref wbase, internalrep);
for(i_=0; i_<=wcount-1;i_++)
{
network.weights[i_] = wbase[i_];
}
//
// Second stage of the hybrid algorithm: LM
//
// Initialize H with identity matrix,
// G with gradient,
// E with regularized error.
//
mlpbase.mlphessianbatch(network, xy, npoints, ref e, ref g, ref h);
v = 0.0;
for(i_=0; i_<=wcount-1;i_++)
{
v += network.weights[i_]*network.weights[i_];
}
e = e+0.5*decay*v;
for(i_=0; i_<=wcount-1;i_++)
{
g[i_] = g[i_] + decay*network.weights[i_];
}
for(k=0; k<=wcount-1; k++)
{
h[k,k] = h[k,k]+decay;
}
rep.nhess = rep.nhess+1;
lambdav = 0.001;
nu = 2;
while( true )
{
//
// 1. HMod = H+lambda*I
// 2. Try to solve (H+Lambda*I)*dx = -g.
// Increase lambda if left part is not positive definite.
//
for(i=0; i<=wcount-1; i++)
{
for(i_=0; i_<=wcount-1;i_++)
{
hmod[i,i_] = h[i,i_];
}
hmod[i,i] = hmod[i,i]+lambdav;
}
spd = trfac.spdmatrixcholesky(ref hmod, wcount, true);
rep.ncholesky = rep.ncholesky+1;
if( !spd )
{
lambdav = lambdav*lambdaup*nu;
nu = nu*2;
continue;
}
densesolver.spdmatrixcholeskysolve(hmod, wcount, true, g, ref solverinfo, solverrep, ref wdir);
if( solverinfo<0 )
{
lambdav = lambdav*lambdaup*nu;
nu = nu*2;
continue;
}
for(i_=0; i_<=wcount-1;i_++)
{
wdir[i_] = -1*wdir[i_];
}
//
// Lambda found.
// 1. Save old w in WBase
// 1. Test some stopping criterions
// 2. If error(w+wdir)>error(w), increase lambda
//
for(i_=0; i_<=wcount-1;i_++)
{
network.weights[i_] = network.weights[i_] + wdir[i_];
}
xnorm2 = 0.0;
for(i_=0; i_<=wcount-1;i_++)
{
xnorm2 += network.weights[i_]*network.weights[i_];
}
stepnorm = 0.0;
for(i_=0; i_<=wcount-1;i_++)
{
stepnorm += wdir[i_]*wdir[i_];
}
stepnorm = Math.Sqrt(stepnorm);
enew = mlpbase.mlperror(network, xy, npoints)+0.5*decay*xnorm2;
if( (double)(stepnorm)<(double)(lmsteptol*(1+Math.Sqrt(xnorm2))) )
{
break;
}
if( (double)(enew)>(double)(e) )
{
lambdav = lambdav*lambdaup*nu;
nu = nu*2;
continue;
}
//
// Optimize using inv(cholesky(H)) as preconditioner
//
matinv.rmatrixtrinverse(ref hmod, wcount, true, false, ref invinfo, invrep);
if( invinfo<=0 )
{
//
// if matrix can't be inverted then exit with errors
// TODO: make WCount steps in direction suggested by HMod
//
info = -9;
return;
}
for(i_=0; i_<=wcount-1;i_++)
{
wbase[i_] = network.weights[i_];
}
for(i=0; i<=wcount-1; i++)
{
wt[i] = 0;
}
minlbfgs.minlbfgscreatex(wcount, wcount, wt, 1, 0.0, state);
minlbfgs.minlbfgssetcond(state, 0, 0, 0, 5);
while( minlbfgs.minlbfgsiteration(state) )
{
//
// gradient
//
for(i=0; i<=wcount-1; i++)
{
v = 0.0;
for(i_=i; i_<=wcount-1;i_++)
{
v += state.x[i_]*hmod[i,i_];
}
network.weights[i] = wbase[i]+v;
}
mlpbase.mlpgradbatch(network, xy, npoints, ref state.f, ref g);
for(i=0; i<=wcount-1; i++)
{
state.g[i] = 0;
}
for(i=0; i<=wcount-1; i++)
{
v = g[i];
for(i_=i; i_<=wcount-1;i_++)
{
state.g[i_] = state.g[i_] + v*hmod[i,i_];
}
}
//
// weight decay
// grad(x'*x) = A'*(x0+A*t)
//
v = 0.0;
for(i_=0; i_<=wcount-1;i_++)
{
v += network.weights[i_]*network.weights[i_];
}
state.f = state.f+0.5*decay*v;
for(i=0; i<=wcount-1; i++)
{
v = decay*network.weights[i];
for(i_=i; i_<=wcount-1;i_++)
{
state.g[i_] = state.g[i_] + v*hmod[i,i_];
}
}
//
// next iteration
//
rep.ngrad = rep.ngrad+1;
}
minlbfgs.minlbfgsresults(state, ref wt, internalrep);
//
// Accept new position.
// Calculate Hessian
//
for(i=0; i<=wcount-1; i++)
{
v = 0.0;
for(i_=i; i_<=wcount-1;i_++)
{
v += wt[i_]*hmod[i,i_];
}
network.weights[i] = wbase[i]+v;
}
mlpbase.mlphessianbatch(network, xy, npoints, ref e, ref g, ref h);
v = 0.0;
for(i_=0; i_<=wcount-1;i_++)
{
v += network.weights[i_]*network.weights[i_];
}
e = e+0.5*decay*v;
for(i_=0; i_<=wcount-1;i_++)
{
g[i_] = g[i_] + decay*network.weights[i_];
}
for(k=0; k<=wcount-1; k++)
{
h[k,k] = h[k,k]+decay;
}
rep.nhess = rep.nhess+1;
//
// Update lambda
//
lambdav = lambdav*lambdadown;
nu = 2;
}
//
// update WBest
//
v = 0.0;
for(i_=0; i_<=wcount-1;i_++)
{
v += network.weights[i_]*network.weights[i_];
}
e = 0.5*decay*v+mlpbase.mlperror(network, xy, npoints);
if( (double)(e)<(double)(ebest) )
{
ebest = e;
for(i_=0; i_<=wcount-1;i_++)
{
wbest[i_] = network.weights[i_];
}
}
}
//
// copy WBest to output
//
for(i_=0; i_<=wcount-1;i_++)
{
network.weights[i_] = wbest[i_];
}
}
public minlmstate()
{
x = new double[0];
fi = new double[0];
j = new double[0,0];
h = new double[0,0];
g = new double[0];
internalstate = new minlbfgs.minlbfgsstate();
internalrep = new minlbfgs.minlbfgsreport();
xprec = new double[0];
xbase = new double[0];
xdir = new double[0];
gbase = new double[0];
xprev = new double[0];
rawmodel = new double[0,0];
model = new double[0,0];
work = new double[0];
rstate = new rcommstate();
solverrep = new densesolver.densesolverreport();
invrep = new matinv.matinvreport();
}
/*************************************************************************
This function tests other properties
On failure sets Err to True (leaves it unchanged otherwise)
*************************************************************************/
private static void testother(ref bool err)
{
int n = 0;
int m = 0;
double[] x = new double[0];
double[] a = new double[0];
double[] s = new double[0];
double[] h = new double[0];
double[] xlast = new double[0];
bool hasxlast = new bool();
double lastscaledstep = 0;
int i = 0;
minlbfgs.minlbfgsstate state = new minlbfgs.minlbfgsstate();
minlbfgs.minlbfgsreport rep = new minlbfgs.minlbfgsreport();
double fprev = 0;
double xprev = 0;
double v = 0;
double stpmax = 0;
double tmpeps = 0;
double epsg = 0;
int pkind = 0;
int ckind = 0;
int mkind = 0;
double vc = 0;
double vm = 0;
double diffstep = 0;
int dkind = 0;
bool wasf = new bool();
bool wasfg = new bool();
double r = 0;
int i_ = 0;
//
// Test reports (F should form monotone sequence)
//
n = 50;
m = 2;
x = new double[n];
xlast = new double[n];
for(i=0; i<=n-1; i++)
{
x[i] = 1;
}
minlbfgs.minlbfgscreate(n, m, x, state);
minlbfgs.minlbfgssetcond(state, 0, 0, 0, 100);
minlbfgs.minlbfgssetxrep(state, true);
fprev = math.maxrealnumber;
while( minlbfgs.minlbfgsiteration(state) )
{
if( state.needfg )
{
state.f = 0;
for(i=0; i<=n-1; i++)
{
state.f = state.f+math.sqr((1+i)*state.x[i]);
state.g[i] = 2*(1+i)*state.x[i];
}
}
if( state.xupdated )
{
err = err | (double)(state.f)>(double)(fprev);
if( (double)(fprev)==(double)(math.maxrealnumber) )
{
for(i=0; i<=n-1; i++)
{
err = err | (double)(state.x[i])!=(double)(x[i]);
}
}
fprev = state.f;
for(i_=0; i_<=n-1;i_++)
{
xlast[i_] = state.x[i_];
}
}
}
minlbfgs.minlbfgsresults(state, ref x, rep);
for(i=0; i<=n-1; i++)
{
err = err | (double)(x[i])!=(double)(xlast[i]);
}
//
// Test differentiation vs. analytic gradient
// (first one issues NeedF requests, second one issues NeedFG requests)
//
n = 50;
m = 5;
diffstep = 1.0E-6;
for(dkind=0; dkind<=1; dkind++)
{
x = new double[n];
xlast = new double[n];
for(i=0; i<=n-1; i++)
{
x[i] = 1;
}
if( dkind==0 )
{
minlbfgs.minlbfgscreate(n, m, x, state);
}
if( dkind==1 )
{
minlbfgs.minlbfgscreatef(n, m, x, diffstep, state);
}
minlbfgs.minlbfgssetcond(state, 0, 0, 0, n/2);
wasf = false;
wasfg = false;
while( minlbfgs.minlbfgsiteration(state) )
{
if( state.needf | state.needfg )
{
state.f = 0;
}
for(i=0; i<=n-1; i++)
{
if( state.needf | state.needfg )
{
state.f = state.f+math.sqr((1+i)*state.x[i]);
}
if( state.needfg )
{
state.g[i] = 2*(1+i)*state.x[i];
}
}
wasf = wasf | state.needf;
wasfg = wasfg | state.needfg;
}
minlbfgs.minlbfgsresults(state, ref x, rep);
if( dkind==0 )
{
err = (err | wasf) | !wasfg;
}
if( dkind==1 )
{
err = (err | !wasf) | wasfg;
}
}
//
// Test that numerical differentiation uses scaling.
//
// In order to test that we solve simple optimization
// problem: min(x^2) with initial x equal to 0.0.
//
// We choose random DiffStep and S, then we check that
// optimizer evaluates function at +-DiffStep*S only.
//
x = new double[1];
s = new double[1];
diffstep = math.randomreal()*1.0E-6;
s[0] = Math.Exp(math.randomreal()*4-2);
x[0] = 0;
minlbfgs.minlbfgscreatef(1, 1, x, diffstep, state);
minlbfgs.minlbfgssetcond(state, 1.0E-6, 0, 0, 0);
minlbfgs.minlbfgssetscale(state, s);
v = 0;
while( minlbfgs.minlbfgsiteration(state) )
{
state.f = math.sqr(state.x[0]);
v = Math.Max(v, Math.Abs(state.x[0]));
}
minlbfgs.minlbfgsresults(state, ref x, rep);
r = v/(s[0]*diffstep);
err = err | (double)(Math.Abs(Math.Log(r)))>(double)(Math.Log(1+1000*math.machineepsilon));
//
// test maximum step
//
n = 1;
m = 1;
x = new double[n];
x[0] = 100;
stpmax = 0.05+0.05*math.randomreal();
minlbfgs.minlbfgscreate(n, m, x, state);
minlbfgs.minlbfgssetcond(state, 1.0E-9, 0, 0, 0);
minlbfgs.minlbfgssetstpmax(state, stpmax);
minlbfgs.minlbfgssetxrep(state, true);
xprev = x[0];
while( minlbfgs.minlbfgsiteration(state) )
{
if( state.needfg )
{
state.f = Math.Exp(state.x[0])+Math.Exp(-state.x[0]);
state.g[0] = Math.Exp(state.x[0])-Math.Exp(-state.x[0]);
err = err | (double)(Math.Abs(state.x[0]-xprev))>(double)((1+Math.Sqrt(math.machineepsilon))*stpmax);
}
if( state.xupdated )
{
err = err | (double)(Math.Abs(state.x[0]-xprev))>(double)((1+Math.Sqrt(math.machineepsilon))*stpmax);
xprev = state.x[0];
}
}
//
// Test correctness of the scaling:
// * initial point is random point from [+1,+2]^N
// * f(x) = SUM(A[i]*x[i]^4), C[i] is random from [0.01,100]
// * we use random scaling matrix
// * we test different variants of the preconditioning:
// 0) unit preconditioner
// 1) random diagonal from [0.01,100]
// 2) scale preconditioner
// * we set stringent stopping conditions (we try EpsG and EpsX)
// * and we test that in the extremum stopping conditions are
// satisfied subject to the current scaling coefficients.
//
tmpeps = 1.0E-10;
m = 1;
for(n=1; n<=10; n++)
{
for(pkind=0; pkind<=2; pkind++)
{
x = new double[n];
xlast = new double[n];
a = new double[n];
s = new double[n];
h = new double[n];
for(i=0; i<=n-1; i++)
{
x[i] = math.randomreal()+1;
a[i] = Math.Exp(Math.Log(100)*(2*math.randomreal()-1));
s[i] = Math.Exp(Math.Log(100)*(2*math.randomreal()-1));
h[i] = Math.Exp(Math.Log(100)*(2*math.randomreal()-1));
}
minlbfgs.minlbfgscreate(n, m, x, state);
minlbfgs.minlbfgssetscale(state, s);
minlbfgs.minlbfgssetxrep(state, true);
if( pkind==1 )
{
minlbfgs.minlbfgssetprecdiag(state, h);
}
if( pkind==2 )
{
minlbfgs.minlbfgssetprecscale(state);
}
//
// Test gradient-based stopping condition
//
for(i=0; i<=n-1; i++)
{
x[i] = math.randomreal()+1;
}
minlbfgs.minlbfgssetcond(state, tmpeps, 0, 0, 0);
minlbfgs.minlbfgsrestartfrom(state, x);
while( minlbfgs.minlbfgsiteration(state) )
{
if( state.needfg )
{
state.f = 0;
for(i=0; i<=n-1; i++)
{
state.f = state.f+a[i]*Math.Pow(state.x[i], 4);
state.g[i] = 4*a[i]*Math.Pow(state.x[i], 3);
}
}
}
minlbfgs.minlbfgsresults(state, ref x, rep);
if( rep.terminationtype<=0 )
{
err = true;
return;
}
v = 0;
for(i=0; i<=n-1; i++)
{
v = v+math.sqr(s[i]*4*a[i]*Math.Pow(x[i], 3));
}
v = Math.Sqrt(v);
err = err | (double)(v)>(double)(tmpeps);
//
// Test step-based stopping condition
//
for(i=0; i<=n-1; i++)
{
x[i] = math.randomreal()+1;
}
hasxlast = false;
minlbfgs.minlbfgssetcond(state, 0, 0, tmpeps, 0);
minlbfgs.minlbfgsrestartfrom(state, x);
while( minlbfgs.minlbfgsiteration(state) )
{
if( state.needfg )
{
state.f = 0;
for(i=0; i<=n-1; i++)
{
state.f = state.f+a[i]*Math.Pow(state.x[i], 4);
state.g[i] = 4*a[i]*Math.Pow(state.x[i], 3);
}
}
if( state.xupdated )
{
if( hasxlast )
{
lastscaledstep = 0;
for(i=0; i<=n-1; i++)
{
lastscaledstep = lastscaledstep+math.sqr(state.x[i]-xlast[i])/math.sqr(s[i]);
}
lastscaledstep = Math.Sqrt(lastscaledstep);
}
else
{
lastscaledstep = 0;
}
for(i_=0; i_<=n-1;i_++)
{
xlast[i_] = state.x[i_];
}
hasxlast = true;
}
}
minlbfgs.minlbfgsresults(state, ref x, rep);
if( rep.terminationtype<=0 )
{
err = true;
return;
}
err = err | (double)(lastscaledstep)>(double)(tmpeps);
}
}
//
// Check correctness of the "trimming".
//
// Trimming is a technique which is used to help algorithm
// cope with unbounded functions. In order to check this
// technique we will try to solve following optimization
// problem:
//
// min f(x) subject to no constraints on X
// { 1/(1-x) + 1/(1+x) + c*x, if -0.999999<x<0.999999
// f(x) = {
// { M, if x<=-0.999999 or x>=0.999999
//
// where c is either 1.0 or 1.0E+6, M is either 1.0E8, 1.0E20 or +INF
// (we try different combinations)
//
for(ckind=0; ckind<=1; ckind++)
{
for(mkind=0; mkind<=2; mkind++)
{
//
// Choose c and M
//
if( ckind==0 )
{
vc = 1.0;
}
if( ckind==1 )
{
vc = 1.0E+6;
}
if( mkind==0 )
{
vm = 1.0E+8;
}
if( mkind==1 )
{
vm = 1.0E+20;
}
if( mkind==2 )
{
vm = Double.PositiveInfinity;
}
//
// Create optimizer, solve optimization problem
//
epsg = 1.0E-6*vc;
x = new double[1];
x[0] = 0.0;
minlbfgs.minlbfgscreate(1, 1, x, state);
minlbfgs.minlbfgssetcond(state, epsg, 0, 0, 0);
while( minlbfgs.minlbfgsiteration(state) )
{
if( state.needfg )
{
if( (double)(-0.999999)<(double)(state.x[0]) & (double)(state.x[0])<(double)(0.999999) )
{
state.f = 1/(1-state.x[0])+1/(1+state.x[0])+vc*state.x[0];
state.g[0] = 1/math.sqr(1-state.x[0])-1/math.sqr(1+state.x[0])+vc;
}
else
{
state.f = vm;
}
}
}
minlbfgs.minlbfgsresults(state, ref x, rep);
if( rep.terminationtype<=0 )
{
err = true;
return;
}
err = err | (double)(Math.Abs(1/math.sqr(1-x[0])-1/math.sqr(1+x[0])+vc))>(double)(epsg);
}
}
}
/*************************************************************************
This function tests preconditioning
On failure sets Err to True (leaves it unchanged otherwise)
*************************************************************************/
private static void testpreconditioning(ref bool err)
{
int pass = 0;
int n = 0;
int m = 0;
int i = 0;
int j = 0;
int k = 0;
int cntb1 = 0;
int cntb2 = 0;
int cntg1 = 0;
int cntg2 = 0;
double epsg = 0;
int pkind = 0;
minlbfgs.minlbfgsstate state = new minlbfgs.minlbfgsstate();
minlbfgs.minlbfgsreport rep = new minlbfgs.minlbfgsreport();
double[] x = new double[0];
double[] s = new double[0];
double[,] a = new double[0,0];
double[] diagh = new double[0];
m = 1;
k = 50;
epsg = 1.0E-10;
//
// Preconditioner test1.
//
// If
// * B1 is default preconditioner
// * B2 is Cholesky preconditioner with unit diagonal
// * G1 is Cholesky preconditioner based on exact Hessian with perturbations
// * G2 is diagonal precomditioner based on approximate diagonal of Hessian matrix
// then "bad" preconditioners (B1/B2/..) are worse than "good" ones (G1/G2/..).
// "Worse" means more iterations to converge.
//
// We test it using f(x) = sum( ((i*i+1)*x[i])^2, i=0..N-1) and L-BFGS
// optimizer with deliberately small M=1.
//
// N - problem size
// PKind - zero for upper triangular preconditioner, one for lower triangular.
// K - number of repeated passes (should be large enough to average out random factors)
//
for(n=10; n<=15; n++)
{
pkind = math.randominteger(2);
x = new double[n];
for(i=0; i<=n-1; i++)
{
x[i] = 0;
}
minlbfgs.minlbfgscreate(n, m, x, state);
//
// Test it with default preconditioner
//
minlbfgs.minlbfgssetprecdefault(state);
cntb1 = 0;
for(pass=0; pass<=k-1; pass++)
{
for(i=0; i<=n-1; i++)
{
x[i] = 2*math.randomreal()-1;
}
minlbfgs.minlbfgsrestartfrom(state, x);
while( minlbfgs.minlbfgsiteration(state) )
{
calciip2(state, n);
}
minlbfgs.minlbfgsresults(state, ref x, rep);
cntb1 = cntb1+rep.iterationscount;
err = err | rep.terminationtype<=0;
}
//
// Test it with unit preconditioner
//
a = new double[n, n];
for(i=0; i<=n-1; i++)
{
for(j=0; j<=n-1; j++)
{
if( i==j )
{
a[i,i] = 1;
}
else
{
a[i,j] = 0;
}
}
}
minlbfgs.minlbfgssetpreccholesky(state, a, pkind==0);
cntb2 = 0;
for(pass=0; pass<=k-1; pass++)
{
for(i=0; i<=n-1; i++)
{
x[i] = 2*math.randomreal()-1;
}
minlbfgs.minlbfgsrestartfrom(state, x);
while( minlbfgs.minlbfgsiteration(state) )
{
calciip2(state, n);
}
minlbfgs.minlbfgsresults(state, ref x, rep);
cntb2 = cntb2+rep.iterationscount;
err = err | rep.terminationtype<=0;
}
//
// Test it with perturbed Hessian preconditioner
//
a = new double[n, n];
for(i=0; i<=n-1; i++)
{
for(j=0; j<=n-1; j++)
{
if( i==j )
{
a[i,i] = (i*i+1)*(0.8+0.4*math.randomreal());
}
else
{
if( (pkind==0 & j>i) | (pkind==1 & j<i) )
{
a[i,j] = 0.1*math.randomreal()-0.05;
}
else
{
a[i,j] = Double.NaN;
}
}
}
}
minlbfgs.minlbfgssetpreccholesky(state, a, pkind==0);
cntg1 = 0;
for(pass=0; pass<=k-1; pass++)
{
for(i=0; i<=n-1; i++)
{
x[i] = 2*math.randomreal()-1;
}
minlbfgs.minlbfgsrestartfrom(state, x);
while( minlbfgs.minlbfgsiteration(state) )
{
calciip2(state, n);
}
minlbfgs.minlbfgsresults(state, ref x, rep);
cntg1 = cntg1+rep.iterationscount;
err = err | rep.terminationtype<=0;
}
//
// Test it with perturbed diagonal preconditioner
//
diagh = new double[n];
for(i=0; i<=n-1; i++)
{
diagh[i] = 2*math.sqr(i*i+1)*(0.8+0.4*math.randomreal());
}
minlbfgs.minlbfgssetprecdiag(state, diagh);
cntg2 = 0;
for(pass=0; pass<=k-1; pass++)
{
for(i=0; i<=n-1; i++)
{
x[i] = 2*math.randomreal()-1;
}
minlbfgs.minlbfgsrestartfrom(state, x);
while( minlbfgs.minlbfgsiteration(state) )
{
calciip2(state, n);
}
minlbfgs.minlbfgsresults(state, ref x, rep);
cntg2 = cntg2+rep.iterationscount;
err = err | rep.terminationtype<=0;
}
//
// Compare
//
err = err | cntb1<cntg1;
err = err | cntb2<cntg1;
err = err | cntb1<cntg2;
err = err | cntb2<cntg2;
}
//
// Preconditioner test 2.
//
// If
// * B2 is default preconditioner with non-unit scale S[i]=1/sqrt(h[i])
// * G2 is scale-based preconditioner with non-unit scale S[i]=1/sqrt(h[i])
// then B2 is worse than G2.
// "Worse" means more iterations to converge.
//
for(n=10; n<=15; n++)
{
x = new double[n];
for(i=0; i<=n-1; i++)
{
x[i] = 0;
}
minlbfgs.minlbfgscreate(n, m, x, state);
s = new double[n];
for(i=0; i<=n-1; i++)
{
s[i] = 1/Math.Sqrt(2*Math.Pow(i*i+1, 2)*(0.8+0.4*math.randomreal()));
}
minlbfgs.minlbfgssetprecdefault(state);
minlbfgs.minlbfgssetscale(state, s);
cntb2 = 0;
for(pass=0; pass<=k-1; pass++)
{
for(i=0; i<=n-1; i++)
{
x[i] = 2*math.randomreal()-1;
}
minlbfgs.minlbfgsrestartfrom(state, x);
while( minlbfgs.minlbfgsiteration(state) )
{
calciip2(state, n);
}
minlbfgs.minlbfgsresults(state, ref x, rep);
cntb2 = cntb2+rep.iterationscount;
err = err | rep.terminationtype<=0;
}
minlbfgs.minlbfgssetprecscale(state);
minlbfgs.minlbfgssetscale(state, s);
cntg2 = 0;
for(pass=0; pass<=k-1; pass++)
{
for(i=0; i<=n-1; i++)
{
x[i] = 2*math.randomreal()-1;
}
minlbfgs.minlbfgsrestartfrom(state, x);
while( minlbfgs.minlbfgsiteration(state) )
{
calciip2(state, n);
}
minlbfgs.minlbfgsresults(state, ref x, rep);
cntg2 = cntg2+rep.iterationscount;
err = err | rep.terminationtype<=0;
}
err = err | cntb2<cntg2;
}
}
public static bool testminlbfgs(bool silent)
{
bool result = new bool();
bool waserrors = new bool();
bool referror = new bool();
bool nonconverror = new bool();
bool eqerror = new bool();
bool converror = new bool();
bool crashtest = new bool();
bool othererrors = new bool();
bool restartserror = new bool();
bool precerror = new bool();
int n = 0;
int m = 0;
double[] x = new double[0];
double[] xe = new double[0];
double[] b = new double[0];
double[] xlast = new double[0];
int i = 0;
int j = 0;
double v = 0;
double[,] a = new double[0,0];
double[] diagh = new double[0];
int maxits = 0;
minlbfgs.minlbfgsstate state = new minlbfgs.minlbfgsstate();
minlbfgs.minlbfgsreport rep = new minlbfgs.minlbfgsreport();
double diffstep = 0;
int dkind = 0;
int i_ = 0;
waserrors = false;
precerror = false;
nonconverror = false;
restartserror = false;
eqerror = false;
converror = false;
crashtest = false;
othererrors = false;
testpreconditioning(ref precerror);
testother(ref othererrors);
//
// Reference problem
//
diffstep = 1.0E-6;
for(dkind=0; dkind<=1; dkind++)
{
x = new double[3];
n = 3;
m = 2;
x[0] = 100*math.randomreal()-50;
x[1] = 100*math.randomreal()-50;
x[2] = 100*math.randomreal()-50;
if( dkind==0 )
{
minlbfgs.minlbfgscreate(n, m, x, state);
}
if( dkind==1 )
{
minlbfgs.minlbfgscreatef(n, m, x, diffstep, state);
}
minlbfgs.minlbfgssetcond(state, 0, 0, 0, 0);
while( minlbfgs.minlbfgsiteration(state) )
{
if( state.needf | state.needfg )
{
state.f = math.sqr(state.x[0]-2)+math.sqr(state.x[1])+math.sqr(state.x[2]-state.x[0]);
}
if( state.needfg )
{
state.g[0] = 2*(state.x[0]-2)+2*(state.x[0]-state.x[2]);
state.g[1] = 2*state.x[1];
state.g[2] = 2*(state.x[2]-state.x[0]);
}
}
minlbfgs.minlbfgsresults(state, ref x, rep);
referror = ((rep.terminationtype<=0 | (double)(Math.Abs(x[0]-2))>(double)(0.001)) | (double)(Math.Abs(x[1]))>(double)(0.001)) | (double)(Math.Abs(x[2]-2))>(double)(0.001);
}
//
// nonconvex problems with complex surface: we start from point with very small
// gradient, but we need ever smaller gradient in the next step due to
// Wolfe conditions.
//
diffstep = 1.0E-6;
for(dkind=0; dkind<=1; dkind++)
{
x = new double[1];
n = 1;
m = 1;
v = -100;
while( (double)(v)<(double)(0.1) )
{
x[0] = v;
if( dkind==0 )
{
minlbfgs.minlbfgscreate(n, m, x, state);
}
if( dkind==1 )
{
minlbfgs.minlbfgscreatef(n, m, x, diffstep, state);
}
minlbfgs.minlbfgssetcond(state, 1.0E-9, 0, 0, 0);
while( minlbfgs.minlbfgsiteration(state) )
{
if( state.needf | state.needfg )
{
state.f = math.sqr(state.x[0])/(1+math.sqr(state.x[0]));
}
if( state.needfg )
{
state.g[0] = (2*state.x[0]*(1+math.sqr(state.x[0]))-math.sqr(state.x[0])*2*state.x[0])/math.sqr(1+math.sqr(state.x[0]));
}
}
minlbfgs.minlbfgsresults(state, ref x, rep);
nonconverror = (nonconverror | rep.terminationtype<=0) | (double)(Math.Abs(x[0]))>(double)(0.001);
v = v+0.1;
}
}
//
// F2 problem with restarts:
// * make several iterations and restart BEFORE termination
// * iterate and restart AFTER termination
//
// NOTE: step is bounded from above to avoid premature convergence
//
diffstep = 1.0E-6;
for(dkind=0; dkind<=1; dkind++)
{
x = new double[3];
n = 3;
m = 2;
x[0] = 10+10*math.randomreal();
x[1] = 10+10*math.randomreal();
x[2] = 10+10*math.randomreal();
if( dkind==0 )
{
minlbfgs.minlbfgscreate(n, m, x, state);
}
if( dkind==1 )
{
minlbfgs.minlbfgscreatef(n, m, x, diffstep, state);
}
minlbfgs.minlbfgssetstpmax(state, 0.1);
minlbfgs.minlbfgssetcond(state, 0.0000001, 0.0, 0.0, 0);
for(i=0; i<=10; i++)
{
if( !minlbfgs.minlbfgsiteration(state) )
{
break;
}
testfunc2(state);
}
x[0] = 10+10*math.randomreal();
x[1] = 10+10*math.randomreal();
x[2] = 10+10*math.randomreal();
minlbfgs.minlbfgsrestartfrom(state, x);
while( minlbfgs.minlbfgsiteration(state) )
{
testfunc2(state);
}
minlbfgs.minlbfgsresults(state, ref x, rep);
restartserror = (((restartserror | rep.terminationtype<=0) | (double)(Math.Abs(x[0]-Math.Log(2)))>(double)(0.01)) | (double)(Math.Abs(x[1]))>(double)(0.01)) | (double)(Math.Abs(x[2]-Math.Log(2)))>(double)(0.01);
x[0] = 10+10*math.randomreal();
x[1] = 10+10*math.randomreal();
x[2] = 10+10*math.randomreal();
minlbfgs.minlbfgsrestartfrom(state, x);
while( minlbfgs.minlbfgsiteration(state) )
{
testfunc2(state);
}
minlbfgs.minlbfgsresults(state, ref x, rep);
restartserror = (((restartserror | rep.terminationtype<=0) | (double)(Math.Abs(x[0]-Math.Log(2)))>(double)(0.01)) | (double)(Math.Abs(x[1]))>(double)(0.01)) | (double)(Math.Abs(x[2]-Math.Log(2)))>(double)(0.01);
}
//
// Linear equations
//
diffstep = 1.0E-6;
for(n=1; n<=10; n++)
{
//
// Prepare task
//
a = new double[n-1+1, n-1+1];
x = new double[n-1+1];
xe = new double[n-1+1];
b = new double[n-1+1];
for(i=0; i<=n-1; i++)
{
xe[i] = 2*math.randomreal()-1;
}
for(i=0; i<=n-1; i++)
{
for(j=0; j<=n-1; j++)
{
a[i,j] = 2*math.randomreal()-1;
}
a[i,i] = a[i,i]+3*Math.Sign(a[i,i]);
}
for(i=0; i<=n-1; i++)
{
v = 0.0;
for(i_=0; i_<=n-1;i_++)
{
v += a[i,i_]*xe[i_];
}
b[i] = v;
}
//
// Test different M/DKind
//
for(m=1; m<=n; m++)
{
for(dkind=0; dkind<=1; dkind++)
{
//
// Solve task
//
for(i=0; i<=n-1; i++)
{
x[i] = 2*math.randomreal()-1;
}
if( dkind==0 )
{
minlbfgs.minlbfgscreate(n, m, x, state);
}
if( dkind==1 )
{
minlbfgs.minlbfgscreatef(n, m, x, diffstep, state);
}
minlbfgs.minlbfgssetcond(state, 0, 0, 0, 0);
while( minlbfgs.minlbfgsiteration(state) )
{
if( state.needf | state.needfg )
{
state.f = 0;
}
if( state.needfg )
{
for(i=0; i<=n-1; i++)
{
state.g[i] = 0;
}
}
for(i=0; i<=n-1; i++)
{
v = 0.0;
for(i_=0; i_<=n-1;i_++)
{
v += a[i,i_]*state.x[i_];
}
if( state.needf | state.needfg )
{
state.f = state.f+math.sqr(v-b[i]);
}
if( state.needfg )
{
for(j=0; j<=n-1; j++)
{
state.g[j] = state.g[j]+2*(v-b[i])*a[i,j];
}
}
}
}
minlbfgs.minlbfgsresults(state, ref x, rep);
eqerror = eqerror | rep.terminationtype<=0;
for(i=0; i<=n-1; i++)
{
eqerror = eqerror | (double)(Math.Abs(x[i]-xe[i]))>(double)(0.001);
}
}
}
}
//
// Testing convergence properties
//
diffstep = 1.0E-6;
for(dkind=0; dkind<=1; dkind++)
{
x = new double[3];
n = 3;
m = 2;
for(i=0; i<=2; i++)
{
x[i] = 6*math.randomreal()-3;
}
if( dkind==0 )
{
minlbfgs.minlbfgscreate(n, m, x, state);
}
if( dkind==1 )
{
minlbfgs.minlbfgscreatef(n, m, x, diffstep, state);
}
minlbfgs.minlbfgssetcond(state, 0.001, 0, 0, 0);
while( minlbfgs.minlbfgsiteration(state) )
{
testfunc3(state);
}
minlbfgs.minlbfgsresults(state, ref x, rep);
converror = converror | rep.terminationtype!=4;
for(i=0; i<=2; i++)
{
x[i] = 6*math.randomreal()-3;
}
if( dkind==0 )
{
minlbfgs.minlbfgscreate(n, m, x, state);
}
if( dkind==1 )
{
minlbfgs.minlbfgscreatef(n, m, x, diffstep, state);
}
minlbfgs.minlbfgssetcond(state, 0, 0.001, 0, 0);
while( minlbfgs.minlbfgsiteration(state) )
{
testfunc3(state);
}
minlbfgs.minlbfgsresults(state, ref x, rep);
converror = converror | rep.terminationtype!=1;
for(i=0; i<=2; i++)
{
x[i] = 6*math.randomreal()-3;
}
if( dkind==0 )
{
minlbfgs.minlbfgscreate(n, m, x, state);
}
if( dkind==1 )
{
minlbfgs.minlbfgscreatef(n, m, x, diffstep, state);
}
minlbfgs.minlbfgssetcond(state, 0, 0, 0.001, 0);
while( minlbfgs.minlbfgsiteration(state) )
{
testfunc3(state);
}
minlbfgs.minlbfgsresults(state, ref x, rep);
converror = converror | rep.terminationtype!=2;
for(i=0; i<=2; i++)
{
x[i] = 2*math.randomreal()-1;
}
if( dkind==0 )
{
minlbfgs.minlbfgscreate(n, m, x, state);
}
if( dkind==1 )
{
minlbfgs.minlbfgscreatef(n, m, x, diffstep, state);
}
minlbfgs.minlbfgssetcond(state, 0, 0, 0, 10);
while( minlbfgs.minlbfgsiteration(state) )
{
testfunc3(state);
}
minlbfgs.minlbfgsresults(state, ref x, rep);
converror = (converror | rep.terminationtype!=5) | rep.iterationscount!=10;
}
//
// Crash test: too many iterations on a simple tasks
// May fail when encounter zero step, underflow or something like that
//
x = new double[2+1];
n = 3;
m = 2;
maxits = 10000;
for(i=0; i<=2; i++)
{
x[i] = 6*math.randomreal()-3;
}
minlbfgs.minlbfgscreate(n, m, x, state);
minlbfgs.minlbfgssetcond(state, 0, 0, 0, maxits);
while( minlbfgs.minlbfgsiteration(state) )
{
state.f = math.sqr(Math.Exp(state.x[0])-2)+math.sqr(state.x[1])+math.sqr(state.x[2]-state.x[0]);
state.g[0] = 2*(Math.Exp(state.x[0])-2)*Math.Exp(state.x[0])+2*(state.x[0]-state.x[2]);
state.g[1] = 2*state.x[1];
state.g[2] = 2*(state.x[2]-state.x[0]);
}
minlbfgs.minlbfgsresults(state, ref x, rep);
crashtest = crashtest | rep.terminationtype<=0;
//
// end
//
waserrors = ((((((referror | nonconverror) | eqerror) | converror) | crashtest) | othererrors) | restartserror) | precerror;
if( !silent )
{
System.Console.Write("TESTING L-BFGS OPTIMIZATION");
System.Console.WriteLine();
System.Console.Write("REFERENCE PROBLEM: ");
if( referror )
{
System.Console.Write("FAILED");
System.Console.WriteLine();
}
else
{
System.Console.Write("OK");
System.Console.WriteLine();
}
System.Console.Write("NON-CONVEX PROBLEM: ");
if( nonconverror )
{
System.Console.Write("FAILED");
System.Console.WriteLine();
}
else
{
System.Console.Write("OK");
System.Console.WriteLine();
}
System.Console.Write("LINEAR EQUATIONS: ");
if( eqerror )
{
System.Console.Write("FAILED");
System.Console.WriteLine();
}
else
{
System.Console.Write("OK");
System.Console.WriteLine();
}
System.Console.Write("RESTARTS: ");
if( restartserror )
{
System.Console.Write("FAILED");
System.Console.WriteLine();
}
else
{
System.Console.Write("OK");
System.Console.WriteLine();
}
System.Console.Write("PRECONDITIONER: ");
if( precerror )
{
System.Console.Write("FAILED");
System.Console.WriteLine();
}
else
{
System.Console.Write("OK");
System.Console.WriteLine();
}
System.Console.Write("CONVERGENCE PROPERTIES: ");
if( converror )
{
System.Console.Write("FAILED");
System.Console.WriteLine();
}
else
{
System.Console.Write("OK");
System.Console.WriteLine();
}
System.Console.Write("CRASH TEST: ");
if( crashtest )
{
System.Console.Write("FAILED");
System.Console.WriteLine();
}
else
{
System.Console.Write("OK");
System.Console.WriteLine();
}
System.Console.Write("OTHER PROPERTIES: ");
if( othererrors )
{
System.Console.Write("FAILED");
System.Console.WriteLine();
}
else
{
System.Console.Write("OK");
System.Console.WriteLine();
}
if( waserrors )
{
System.Console.Write("TEST FAILED");
System.Console.WriteLine();
}
else
{
System.Console.Write("TEST PASSED");
System.Console.WriteLine();
}
System.Console.WriteLine();
System.Console.WriteLine();
}
result = !waserrors;
return result;
}
public override void init()
{
bestparameters = new double[0];
network = new mlpbase.multilayerperceptron();
optimizer = new minlbfgs.minlbfgsstate();
optimizerrep = new minlbfgs.minlbfgsreport();
wbuf0 = new double[0];
wbuf1 = new double[0];
allminibatches = new int[0];
currentminibatch = new int[0];
rstate = new rcommstate();
generator = new hqrnd.hqrndstate();
}
public minlmstate()
{
x = new double[0];
fi = new double[0];
j = new double[0,0];
h = new double[0,0];
g = new double[0];
xbase = new double[0];
fibase = new double[0];
gbase = new double[0];
quadraticmodel = new double[0,0];
dampedmodel = new double[0,0];
xdir = new double[0];
deltax = new double[0];
deltaf = new double[0];
rstate = new rcommstate();
choleskybuf = new double[0];
fm2 = new double[0];
fm1 = new double[0];
fp2 = new double[0];
fp1 = new double[0];
internalstate = new minlbfgs.minlbfgsstate();
internalrep = new minlbfgs.minlbfgsreport();
}
