public densesolverreport(densesolver.densesolverreport obj)
{
_innerobj = obj;
}
/*************************************************************************
Neural network training using modified Levenberg-Marquardt with exact
Hessian calculation and regularization. Subroutine trains neural network
with restarts from random positions. Algorithm is well suited for small
and medium scale problems (hundreds of weights).
INPUT PARAMETERS:
Network - neural network with initialized geometry
XY - training set
NPoints - training set size
Decay - weight decay constant, >=0.001
Decay term 'Decay*||Weights||^2' is added to error
function.
If you don't know what Decay to choose, use 0.001.
Restarts - number of restarts from random position, >0.
If you don't know what Restarts to choose, use 2.
OUTPUT PARAMETERS:
Network - trained neural network.
Info - return code:
* -9, if internal matrix inverse subroutine failed
* -2, if there is a point with class number
outside of [0..NOut-1].
* -1, if wrong parameters specified
(NPoints<0, Restarts<1).
* 2, if task has been solved.
Rep - training report
-- ALGLIB --
Copyright 10.03.2009 by Bochkanov Sergey
*************************************************************************/
public static void mlptrainlm(mlpbase.multilayerperceptron network,
double[,] xy,
int npoints,
double decay,
int restarts,
ref int info,
mlpreport rep)
{
int nin = 0;
int nout = 0;
int wcount = 0;
double lmftol = 0;
double lmsteptol = 0;
int i = 0;
int k = 0;
double v = 0;
double e = 0;
double enew = 0;
double xnorm2 = 0;
double stepnorm = 0;
double[] g = new double[0];
double[] d = new double[0];
double[,] h = new double[0,0];
double[,] hmod = new double[0,0];
double[,] z = new double[0,0];
bool spd = new bool();
double nu = 0;
double lambdav = 0;
double lambdaup = 0;
double lambdadown = 0;
minlbfgs.minlbfgsreport internalrep = new minlbfgs.minlbfgsreport();
minlbfgs.minlbfgsstate state = new minlbfgs.minlbfgsstate();
double[] x = new double[0];
double[] y = new double[0];
double[] wbase = new double[0];
double[] wdir = new double[0];
double[] wt = new double[0];
double[] wx = new double[0];
int pass = 0;
double[] wbest = new double[0];
double ebest = 0;
int invinfo = 0;
matinv.matinvreport invrep = new matinv.matinvreport();
int solverinfo = 0;
densesolver.densesolverreport solverrep = new densesolver.densesolverreport();
int i_ = 0;
info = 0;
mlpbase.mlpproperties(network, ref nin, ref nout, ref wcount);
lambdaup = 10;
lambdadown = 0.3;
lmftol = 0.001;
lmsteptol = 0.001;
//
// Test for inputs
//
if( npoints<=0 || restarts<1 )
{
info = -1;
return;
}
if( mlpbase.mlpissoftmax(network) )
{
for(i=0; i<=npoints-1; i++)
{
if( (int)Math.Round(xy[i,nin])<0 || (int)Math.Round(xy[i,nin])>=nout )
{
info = -2;
return;
}
}
}
decay = Math.Max(decay, mindecay);
info = 2;
//
// Initialize data
//
rep.ngrad = 0;
rep.nhess = 0;
rep.ncholesky = 0;
//
// General case.
// Prepare task and network. Allocate space.
//
mlpbase.mlpinitpreprocessor(network, xy, npoints);
g = new double[wcount-1+1];
h = new double[wcount-1+1, wcount-1+1];
hmod = new double[wcount-1+1, wcount-1+1];
wbase = new double[wcount-1+1];
wdir = new double[wcount-1+1];
wbest = new double[wcount-1+1];
wt = new double[wcount-1+1];
wx = new double[wcount-1+1];
ebest = math.maxrealnumber;
//
// Multiple passes
//
for(pass=1; pass<=restarts; pass++)
{
//
// Initialize weights
//
mlpbase.mlprandomize(network);
//
// First stage of the hybrid algorithm: LBFGS
//
for(i_=0; i_<=wcount-1;i_++)
{
wbase[i_] = network.weights[i_];
}
minlbfgs.minlbfgscreate(wcount, Math.Min(wcount, 5), wbase, state);
minlbfgs.minlbfgssetcond(state, 0, 0, 0, Math.Max(25, wcount));
while( minlbfgs.minlbfgsiteration(state) )
{
//
// gradient
//
for(i_=0; i_<=wcount-1;i_++)
{
network.weights[i_] = state.x[i_];
}
mlpbase.mlpgradbatch(network, xy, npoints, ref state.f, ref state.g);
//
// weight decay
//
v = 0.0;
for(i_=0; i_<=wcount-1;i_++)
{
v += network.weights[i_]*network.weights[i_];
}
state.f = state.f+0.5*decay*v;
for(i_=0; i_<=wcount-1;i_++)
{
state.g[i_] = state.g[i_] + decay*network.weights[i_];
}
//
// next iteration
//
rep.ngrad = rep.ngrad+1;
}
minlbfgs.minlbfgsresults(state, ref wbase, internalrep);
for(i_=0; i_<=wcount-1;i_++)
{
network.weights[i_] = wbase[i_];
}
//
// Second stage of the hybrid algorithm: LM
//
// Initialize H with identity matrix,
// G with gradient,
// E with regularized error.
//
mlpbase.mlphessianbatch(network, xy, npoints, ref e, ref g, ref h);
v = 0.0;
for(i_=0; i_<=wcount-1;i_++)
{
v += network.weights[i_]*network.weights[i_];
}
e = e+0.5*decay*v;
for(i_=0; i_<=wcount-1;i_++)
{
g[i_] = g[i_] + decay*network.weights[i_];
}
for(k=0; k<=wcount-1; k++)
{
h[k,k] = h[k,k]+decay;
}
rep.nhess = rep.nhess+1;
lambdav = 0.001;
nu = 2;
while( true )
{
//
// 1. HMod = H+lambda*I
// 2. Try to solve (H+Lambda*I)*dx = -g.
// Increase lambda if left part is not positive definite.
//
for(i=0; i<=wcount-1; i++)
{
for(i_=0; i_<=wcount-1;i_++)
{
hmod[i,i_] = h[i,i_];
}
hmod[i,i] = hmod[i,i]+lambdav;
}
spd = trfac.spdmatrixcholesky(ref hmod, wcount, true);
rep.ncholesky = rep.ncholesky+1;
if( !spd )
{
lambdav = lambdav*lambdaup*nu;
nu = nu*2;
continue;
}
densesolver.spdmatrixcholeskysolve(hmod, wcount, true, g, ref solverinfo, solverrep, ref wdir);
if( solverinfo<0 )
{
lambdav = lambdav*lambdaup*nu;
nu = nu*2;
continue;
}
for(i_=0; i_<=wcount-1;i_++)
{
wdir[i_] = -1*wdir[i_];
}
//
// Lambda found.
// 1. Save old w in WBase
// 1. Test some stopping criterions
// 2. If error(w+wdir)>error(w), increase lambda
//
for(i_=0; i_<=wcount-1;i_++)
{
network.weights[i_] = network.weights[i_] + wdir[i_];
}
xnorm2 = 0.0;
for(i_=0; i_<=wcount-1;i_++)
{
xnorm2 += network.weights[i_]*network.weights[i_];
}
stepnorm = 0.0;
for(i_=0; i_<=wcount-1;i_++)
{
stepnorm += wdir[i_]*wdir[i_];
}
stepnorm = Math.Sqrt(stepnorm);
enew = mlpbase.mlperror(network, xy, npoints)+0.5*decay*xnorm2;
if( (double)(stepnorm)<(double)(lmsteptol*(1+Math.Sqrt(xnorm2))) )
{
break;
}
if( (double)(enew)>(double)(e) )
{
lambdav = lambdav*lambdaup*nu;
nu = nu*2;
continue;
}
//
// Optimize using inv(cholesky(H)) as preconditioner
//
matinv.rmatrixtrinverse(ref hmod, wcount, true, false, ref invinfo, invrep);
if( invinfo<=0 )
{
//
// if matrix can't be inverted then exit with errors
// TODO: make WCount steps in direction suggested by HMod
//
info = -9;
return;
}
for(i_=0; i_<=wcount-1;i_++)
{
wbase[i_] = network.weights[i_];
}
for(i=0; i<=wcount-1; i++)
{
wt[i] = 0;
}
minlbfgs.minlbfgscreatex(wcount, wcount, wt, 1, 0.0, state);
minlbfgs.minlbfgssetcond(state, 0, 0, 0, 5);
while( minlbfgs.minlbfgsiteration(state) )
{
//
// gradient
//
for(i=0; i<=wcount-1; i++)
{
v = 0.0;
for(i_=i; i_<=wcount-1;i_++)
{
v += state.x[i_]*hmod[i,i_];
}
network.weights[i] = wbase[i]+v;
}
mlpbase.mlpgradbatch(network, xy, npoints, ref state.f, ref g);
for(i=0; i<=wcount-1; i++)
{
state.g[i] = 0;
}
for(i=0; i<=wcount-1; i++)
{
v = g[i];
for(i_=i; i_<=wcount-1;i_++)
{
state.g[i_] = state.g[i_] + v*hmod[i,i_];
}
}
//
// weight decay
// grad(x'*x) = A'*(x0+A*t)
//
v = 0.0;
for(i_=0; i_<=wcount-1;i_++)
{
v += network.weights[i_]*network.weights[i_];
}
state.f = state.f+0.5*decay*v;
for(i=0; i<=wcount-1; i++)
{
v = decay*network.weights[i];
for(i_=i; i_<=wcount-1;i_++)
{
state.g[i_] = state.g[i_] + v*hmod[i,i_];
}
}
//
// next iteration
//
rep.ngrad = rep.ngrad+1;
}
minlbfgs.minlbfgsresults(state, ref wt, internalrep);
//
// Accept new position.
// Calculate Hessian
//
for(i=0; i<=wcount-1; i++)
{
v = 0.0;
for(i_=i; i_<=wcount-1;i_++)
{
v += wt[i_]*hmod[i,i_];
}
network.weights[i] = wbase[i]+v;
}
mlpbase.mlphessianbatch(network, xy, npoints, ref e, ref g, ref h);
v = 0.0;
for(i_=0; i_<=wcount-1;i_++)
{
v += network.weights[i_]*network.weights[i_];
}
e = e+0.5*decay*v;
for(i_=0; i_<=wcount-1;i_++)
{
g[i_] = g[i_] + decay*network.weights[i_];
}
for(k=0; k<=wcount-1; k++)
{
h[k,k] = h[k,k]+decay;
}
rep.nhess = rep.nhess+1;
//
// Update lambda
//
lambdav = lambdav*lambdadown;
nu = 2;
}
//
// update WBest
//
v = 0.0;
for(i_=0; i_<=wcount-1;i_++)
{
v += network.weights[i_]*network.weights[i_];
}
e = 0.5*decay*v+mlpbase.mlperror(network, xy, npoints);
if( (double)(e)<(double)(ebest) )
{
ebest = e;
for(i_=0; i_<=wcount-1;i_++)
{
wbest[i_] = network.weights[i_];
}
}
}
//
// copy WBest to output
//
for(i_=0; i_<=wcount-1;i_++)
{
network.weights[i_] = wbest[i_];
}
}
public densesolverreport()
{
_innerobj = new densesolver.densesolverreport();
}
public minlmstate()
{
x = new double[0];
fi = new double[0];
j = new double[0,0];
h = new double[0,0];
g = new double[0];
internalstate = new minlbfgs.minlbfgsstate();
internalrep = new minlbfgs.minlbfgsreport();
xprec = new double[0];
xbase = new double[0];
xdir = new double[0];
gbase = new double[0];
xprev = new double[0];
rawmodel = new double[0,0];
model = new double[0,0];
work = new double[0];
rstate = new rcommstate();
solverrep = new densesolver.densesolverreport();
invrep = new matinv.matinvreport();
}
/*************************************************************************
This subroutine trains logit model.
INPUT PARAMETERS:
XY - training set, array[0..NPoints-1,0..NVars]
First NVars columns store values of independent
variables, next column stores number of class (from 0
to NClasses-1) which dataset element belongs to. Fractional
values are rounded to nearest integer.
NPoints - training set size, NPoints>=1
NVars - number of independent variables, NVars>=1
NClasses - number of classes, NClasses>=2
OUTPUT PARAMETERS:
Info - return code:
* -2, if there is a point with class number
outside of [0..NClasses-1].
* -1, if incorrect parameters was passed
(NPoints<NVars+2, NVars<1, NClasses<2).
* 1, if task has been solved
LM - model built
Rep - training report
-- ALGLIB --
Copyright 10.09.2008 by Bochkanov Sergey
*************************************************************************/
public static void mnltrainh(double[,] xy,
int npoints,
int nvars,
int nclasses,
ref int info,
logitmodel lm,
mnlreport rep)
{
int i = 0;
int j = 0;
int k = 0;
int ssize = 0;
bool allsame = new bool();
int offs = 0;
double threshold = 0;
double wminstep = 0;
double decay = 0;
int wdim = 0;
int expoffs = 0;
double v = 0;
double s = 0;
mlpbase.multilayerperceptron network = new mlpbase.multilayerperceptron();
int nin = 0;
int nout = 0;
int wcount = 0;
double e = 0;
double[] g = new double[0];
double[,] h = new double[0,0];
bool spd = new bool();
double[] x = new double[0];
double[] y = new double[0];
double[] wbase = new double[0];
double wstep = 0;
double[] wdir = new double[0];
double[] work = new double[0];
int mcstage = 0;
logitmcstate mcstate = new logitmcstate();
int mcinfo = 0;
int mcnfev = 0;
int solverinfo = 0;
densesolver.densesolverreport solverrep = new densesolver.densesolverreport();
int i_ = 0;
int i1_ = 0;
info = 0;
threshold = 1000*math.machineepsilon;
wminstep = 0.001;
decay = 0.001;
//
// Test for inputs
//
if( (npoints<nvars+2 || nvars<1) || nclasses<2 )
{
info = -1;
return;
}
for(i=0; i<=npoints-1; i++)
{
if( (int)Math.Round(xy[i,nvars])<0 || (int)Math.Round(xy[i,nvars])>=nclasses )
{
info = -2;
return;
}
}
info = 1;
//
// Initialize data
//
rep.ngrad = 0;
rep.nhess = 0;
//
// Allocate array
//
wdim = (nvars+1)*(nclasses-1);
offs = 5;
expoffs = offs+wdim;
ssize = 5+(nvars+1)*(nclasses-1)+nclasses;
lm.w = new double[ssize-1+1];
lm.w[0] = ssize;
lm.w[1] = logitvnum;
lm.w[2] = nvars;
lm.w[3] = nclasses;
lm.w[4] = offs;
//
// Degenerate case: all outputs are equal
//
allsame = true;
for(i=1; i<=npoints-1; i++)
{
if( (int)Math.Round(xy[i,nvars])!=(int)Math.Round(xy[i-1,nvars]) )
{
allsame = false;
}
}
if( allsame )
{
for(i=0; i<=(nvars+1)*(nclasses-1)-1; i++)
{
lm.w[offs+i] = 0;
}
v = -(2*Math.Log(math.minrealnumber));
k = (int)Math.Round(xy[0,nvars]);
if( k==nclasses-1 )
{
for(i=0; i<=nclasses-2; i++)
{
lm.w[offs+i*(nvars+1)+nvars] = -v;
}
}
else
{
for(i=0; i<=nclasses-2; i++)
{
if( i==k )
{
lm.w[offs+i*(nvars+1)+nvars] = v;
}
else
{
lm.w[offs+i*(nvars+1)+nvars] = 0;
}
}
}
return;
}
//
// General case.
// Prepare task and network. Allocate space.
//
mlpbase.mlpcreatec0(nvars, nclasses, network);
mlpbase.mlpinitpreprocessor(network, xy, npoints);
mlpbase.mlpproperties(network, ref nin, ref nout, ref wcount);
for(i=0; i<=wcount-1; i++)
{
network.weights[i] = (2*math.randomreal()-1)/nvars;
}
g = new double[wcount-1+1];
h = new double[wcount-1+1, wcount-1+1];
wbase = new double[wcount-1+1];
wdir = new double[wcount-1+1];
work = new double[wcount-1+1];
//
// First stage: optimize in gradient direction.
//
for(k=0; k<=wcount/3+10; k++)
{
//
// Calculate gradient in starting point
//
mlpbase.mlpgradnbatch(network, xy, npoints, ref e, ref g);
v = 0.0;
for(i_=0; i_<=wcount-1;i_++)
{
v += network.weights[i_]*network.weights[i_];
}
e = e+0.5*decay*v;
for(i_=0; i_<=wcount-1;i_++)
{
g[i_] = g[i_] + decay*network.weights[i_];
}
rep.ngrad = rep.ngrad+1;
//
// Setup optimization scheme
//
for(i_=0; i_<=wcount-1;i_++)
{
wdir[i_] = -g[i_];
}
v = 0.0;
for(i_=0; i_<=wcount-1;i_++)
{
v += wdir[i_]*wdir[i_];
}
wstep = Math.Sqrt(v);
v = 1/Math.Sqrt(v);
for(i_=0; i_<=wcount-1;i_++)
{
wdir[i_] = v*wdir[i_];
}
mcstage = 0;
mnlmcsrch(wcount, ref network.weights, ref e, ref g, wdir, ref wstep, ref mcinfo, ref mcnfev, ref work, mcstate, ref mcstage);
while( mcstage!=0 )
{
mlpbase.mlpgradnbatch(network, xy, npoints, ref e, ref g);
v = 0.0;
for(i_=0; i_<=wcount-1;i_++)
{
v += network.weights[i_]*network.weights[i_];
}
e = e+0.5*decay*v;
for(i_=0; i_<=wcount-1;i_++)
{
g[i_] = g[i_] + decay*network.weights[i_];
}
rep.ngrad = rep.ngrad+1;
mnlmcsrch(wcount, ref network.weights, ref e, ref g, wdir, ref wstep, ref mcinfo, ref mcnfev, ref work, mcstate, ref mcstage);
}
}
//
// Second stage: use Hessian when we are close to the minimum
//
while( true )
{
//
// Calculate and update E/G/H
//
mlpbase.mlphessiannbatch(network, xy, npoints, ref e, ref g, ref h);
v = 0.0;
for(i_=0; i_<=wcount-1;i_++)
{
v += network.weights[i_]*network.weights[i_];
}
e = e+0.5*decay*v;
for(i_=0; i_<=wcount-1;i_++)
{
g[i_] = g[i_] + decay*network.weights[i_];
}
for(k=0; k<=wcount-1; k++)
{
h[k,k] = h[k,k]+decay;
}
rep.nhess = rep.nhess+1;
//
// Select step direction
// NOTE: it is important to use lower-triangle Cholesky
// factorization since it is much faster than higher-triangle version.
//
spd = trfac.spdmatrixcholesky(ref h, wcount, false);
densesolver.spdmatrixcholeskysolve(h, wcount, false, g, ref solverinfo, solverrep, ref wdir);
spd = solverinfo>0;
if( spd )
{
//
// H is positive definite.
// Step in Newton direction.
//
for(i_=0; i_<=wcount-1;i_++)
{
wdir[i_] = -1*wdir[i_];
}
spd = true;
}
else
{
//
// H is indefinite.
// Step in gradient direction.
//
for(i_=0; i_<=wcount-1;i_++)
{
wdir[i_] = -g[i_];
}
spd = false;
}
//
// Optimize in WDir direction
//
v = 0.0;
for(i_=0; i_<=wcount-1;i_++)
{
v += wdir[i_]*wdir[i_];
}
wstep = Math.Sqrt(v);
v = 1/Math.Sqrt(v);
for(i_=0; i_<=wcount-1;i_++)
{
wdir[i_] = v*wdir[i_];
}
mcstage = 0;
mnlmcsrch(wcount, ref network.weights, ref e, ref g, wdir, ref wstep, ref mcinfo, ref mcnfev, ref work, mcstate, ref mcstage);
while( mcstage!=0 )
{
mlpbase.mlpgradnbatch(network, xy, npoints, ref e, ref g);
v = 0.0;
for(i_=0; i_<=wcount-1;i_++)
{
v += network.weights[i_]*network.weights[i_];
}
e = e+0.5*decay*v;
for(i_=0; i_<=wcount-1;i_++)
{
g[i_] = g[i_] + decay*network.weights[i_];
}
rep.ngrad = rep.ngrad+1;
mnlmcsrch(wcount, ref network.weights, ref e, ref g, wdir, ref wstep, ref mcinfo, ref mcnfev, ref work, mcstate, ref mcstage);
}
if( spd && ((mcinfo==2 || mcinfo==4) || mcinfo==6) )
{
break;
}
}
//
// Convert from NN format to MNL format
//
i1_ = (0) - (offs);
for(i_=offs; i_<=offs+wcount-1;i_++)
{
lm.w[i_] = network.weights[i_+i1_];
}
for(k=0; k<=nvars-1; k++)
{
for(i=0; i<=nclasses-2; i++)
{
s = network.columnsigmas[k];
if( (double)(s)==(double)(0) )
{
s = 1;
}
j = offs+(nvars+1)*i;
v = lm.w[j+k];
lm.w[j+k] = v/s;
lm.w[j+nvars] = lm.w[j+nvars]+v*network.columnmeans[k]/s;
}
}
for(k=0; k<=nclasses-2; k++)
{
lm.w[offs+(nvars+1)*k+nvars] = -lm.w[offs+(nvars+1)*k+nvars];
}
}
/*************************************************************************
HPD test
*************************************************************************/
private static void testhpdsolver(int maxn,
int maxm,
int passcount,
double threshold,
ref bool hpderrors,
ref bool rfserrors)
{
complex[,] a = new complex[0,0];
complex[,] cha = new complex[0,0];
complex[,] atmp = new complex[0,0];
int[] p = new int[0];
complex[,] xe = new complex[0,0];
complex[,] b = new complex[0,0];
complex[] bv = new complex[0];
int i = 0;
int j = 0;
int k = 0;
int n = 0;
int m = 0;
int pass = 0;
int taskkind = 0;
complex v = 0;
bool isupper = new bool();
int info = 0;
densesolver.densesolverreport rep = new densesolver.densesolverreport();
densesolver.densesolverlsreport repls = new densesolver.densesolverlsreport();
complex[,] x = new complex[0,0];
complex[] xv = new complex[0];
complex[] y = new complex[0];
complex[] tx = new complex[0];
int i_ = 0;
//
// General square matrices:
// * test general solvers
// * test least squares solver
//
for(pass=1; pass<=passcount; pass++)
{
for(n=1; n<=maxn; n++)
{
for(m=1; m<=maxm; m++)
{
//
// ********************************************************
// WELL CONDITIONED TASKS
// ability to find correct solution is tested
// ********************************************************
//
// 1. generate random well conditioned matrix A.
// 2. generate random solution vector xe
// 3. generate right part b=A*xe
// 4. test different methods on original A
//
isupper = (double)(math.randomreal())>(double)(0.5);
matgen.hpdmatrixrndcond(n, 1000, ref a);
cmatrixmakeacopy(a, n, n, ref cha);
if( !trfac.hpdmatrixcholesky(ref cha, n, isupper) )
{
hpderrors = true;
return;
}
xe = new complex[n, m];
for(i=0; i<=n-1; i++)
{
for(j=0; j<=m-1; j++)
{
xe[i,j].x = 2*math.randomreal()-1;
xe[i,j].y = 2*math.randomreal()-1;
}
}
b = new complex[n, m];
for(i=0; i<=n-1; i++)
{
for(j=0; j<=m-1; j++)
{
v = 0.0;
for(i_=0; i_<=n-1;i_++)
{
v += a[i,i_]*xe[i_,j];
}
b[i,j] = v;
}
}
cmatrixdrophalf(ref a, n, isupper);
cmatrixdrophalf(ref cha, n, isupper);
//
// Test solvers
//
info = 0;
unsetrep(rep);
cunset2d(ref x);
densesolver.hpdmatrixsolvem(a, n, isupper, b, m, ref info, rep, ref x);
hpderrors = hpderrors | !cmatrixchecksolutionm(xe, n, m, threshold, info, rep, x);
info = 0;
unsetrep(rep);
cunset1d(ref xv);
bv = new complex[n];
for(i_=0; i_<=n-1;i_++)
{
bv[i_] = b[i_,0];
}
densesolver.hpdmatrixsolve(a, n, isupper, bv, ref info, rep, ref xv);
hpderrors = hpderrors | !cmatrixchecksolution(xe, n, threshold, info, rep, xv);
info = 0;
unsetrep(rep);
cunset2d(ref x);
densesolver.hpdmatrixcholeskysolvem(cha, n, isupper, b, m, ref info, rep, ref x);
hpderrors = hpderrors | !cmatrixchecksolutionm(xe, n, m, threshold, info, rep, x);
info = 0;
unsetrep(rep);
cunset1d(ref xv);
bv = new complex[n];
for(i_=0; i_<=n-1;i_++)
{
bv[i_] = b[i_,0];
}
densesolver.hpdmatrixcholeskysolve(cha, n, isupper, bv, ref info, rep, ref xv);
hpderrors = hpderrors | !cmatrixchecksolution(xe, n, threshold, info, rep, xv);
//
// ********************************************************
// EXACTLY SINGULAR MATRICES
// ability to detect singularity is tested
// ********************************************************
//
// 1. generate different types of singular matrices:
// * zero
// * with zero columns
// * with zero rows
// * with equal rows/columns
// 2. generate random solution vector xe
// 3. generate right part b=A*xe
// 4. test different methods
//
for(taskkind=0; taskkind<=3; taskkind++)
{
cunset2d(ref a);
if( taskkind==0 )
{
//
// all zeros
//
a = new complex[n, n];
for(i=0; i<=n-1; i++)
{
for(j=0; j<=n-1; j++)
{
a[i,j] = 0;
}
}
}
if( taskkind==1 )
{
//
// there is zero column
//
a = new complex[n, n];
for(i=0; i<=n-1; i++)
{
for(j=i; j<=n-1; j++)
{
a[i,j].x = 2*math.randomreal()-1;
a[i,j].y = 2*math.randomreal()-1;
if( i==j )
{
a[i,j].y = 0;
}
a[j,i] = a[i,j];
}
}
k = math.randominteger(n);
for(i_=0; i_<=n-1;i_++)
{
a[i_,k] = 0*a[i_,k];
}
for(i_=0; i_<=n-1;i_++)
{
a[k,i_] = 0*a[k,i_];
}
}
if( taskkind==2 )
{
//
// there is zero row
//
a = new complex[n, n];
for(i=0; i<=n-1; i++)
{
for(j=i; j<=n-1; j++)
{
a[i,j].x = 2*math.randomreal()-1;
a[i,j].y = 2*math.randomreal()-1;
if( i==j )
{
a[i,j].y = 0;
}
a[j,i] = a[i,j];
}
}
k = math.randominteger(n);
for(i_=0; i_<=n-1;i_++)
{
a[k,i_] = 0*a[k,i_];
}
for(i_=0; i_<=n-1;i_++)
{
a[i_,k] = 0*a[i_,k];
}
}
if( taskkind==3 )
{
//
// equal columns/rows
//
if( n<2 )
{
continue;
}
a = new complex[n, n];
for(i=0; i<=n-1; i++)
{
for(j=i; j<=n-1; j++)
{
a[i,j].x = 2*math.randomreal()-1;
a[i,j].y = 2*math.randomreal()-1;
if( i==j )
{
a[i,j].y = 0;
}
a[j,i] = a[i,j];
}
}
k = 1+math.randominteger(n-1);
for(i_=0; i_<=n-1;i_++)
{
a[i_,0] = a[i_,k];
}
for(i_=0; i_<=n-1;i_++)
{
a[0,i_] = a[k,i_];
}
}
xe = new complex[n, m];
for(i=0; i<=n-1; i++)
{
for(j=0; j<=m-1; j++)
{
xe[i,j] = 2*math.randomreal()-1;
}
}
b = new complex[n, m];
for(i=0; i<=n-1; i++)
{
for(j=0; j<=m-1; j++)
{
v = 0.0;
for(i_=0; i_<=n-1;i_++)
{
v += a[i,i_]*xe[i_,j];
}
b[i,j] = v;
}
}
cmatrixmakeacopy(a, n, n, ref cha);
cmatrixdrophalf(ref a, n, isupper);
cmatrixdrophalf(ref cha, n, isupper);
//
// Test SPDMatrixSolveM()
//
info = 0;
unsetrep(rep);
cunset2d(ref x);
densesolver.hpdmatrixsolvem(a, n, isupper, b, m, ref info, rep, ref x);
hpderrors = hpderrors | !cmatrixchecksingularm(n, m, info, rep, x);
//
// Test SPDMatrixSolve()
//
info = 0;
unsetrep(rep);
cunset2d(ref x);
bv = new complex[n];
for(i_=0; i_<=n-1;i_++)
{
bv[i_] = b[i_,0];
}
densesolver.hpdmatrixsolve(a, n, isupper, bv, ref info, rep, ref xv);
hpderrors = hpderrors | !cmatrixchecksingular(n, info, rep, xv);
//
// 'equal columns/rows' are degenerate, but
// Cholesky matrix with equal columns/rows IS NOT degenerate,
// so it is not used for testing purposes.
//
if( taskkind!=3 )
{
//
// Test SPDMatrixLUSolveM()
//
info = 0;
unsetrep(rep);
cunset2d(ref x);
densesolver.hpdmatrixcholeskysolvem(cha, n, isupper, b, m, ref info, rep, ref x);
hpderrors = hpderrors | !cmatrixchecksingularm(n, m, info, rep, x);
//
// Test SPDMatrixLUSolve()
//
info = 0;
unsetrep(rep);
cunset2d(ref x);
bv = new complex[n];
for(i_=0; i_<=n-1;i_++)
{
bv[i_] = b[i_,0];
}
densesolver.hpdmatrixcholeskysolve(cha, n, isupper, bv, ref info, rep, ref xv);
hpderrors = hpderrors | !cmatrixchecksingular(n, info, rep, xv);
}
}
}
}
}
}
/*************************************************************************
Real test
*************************************************************************/
private static void testcsolver(int maxn,
int maxm,
int passcount,
double threshold,
ref bool cerrors,
ref bool rfserrors)
{
complex[,] a = new complex[0,0];
complex[,] lua = new complex[0,0];
complex[,] atmp = new complex[0,0];
int[] p = new int[0];
complex[,] xe = new complex[0,0];
complex[,] b = new complex[0,0];
complex[] bv = new complex[0];
int i = 0;
int j = 0;
int k = 0;
int n = 0;
int m = 0;
int pass = 0;
int taskkind = 0;
double verr = 0;
complex v = 0;
int info = 0;
densesolver.densesolverreport rep = new densesolver.densesolverreport();
densesolver.densesolverlsreport repls = new densesolver.densesolverlsreport();
complex[,] x = new complex[0,0];
complex[] xv = new complex[0];
complex[] y = new complex[0];
double[] tx = new double[0];
int i_ = 0;
//
// General square matrices:
// * test general solvers
// * test least squares solver
//
for(pass=1; pass<=passcount; pass++)
{
for(n=1; n<=maxn; n++)
{
for(m=1; m<=maxm; m++)
{
//
// ********************************************************
// WELL CONDITIONED TASKS
// ability to find correct solution is tested
// ********************************************************
//
// 1. generate random well conditioned matrix A.
// 2. generate random solution vector xe
// 3. generate right part b=A*xe
// 4. test different methods on original A
//
matgen.cmatrixrndcond(n, 1000, ref a);
cmatrixmakeacopy(a, n, n, ref lua);
trfac.cmatrixlu(ref lua, n, n, ref p);
xe = new complex[n, m];
for(i=0; i<=n-1; i++)
{
for(j=0; j<=m-1; j++)
{
xe[i,j].x = 2*math.randomreal()-1;
xe[i,j].y = 2*math.randomreal()-1;
}
}
b = new complex[n, m];
for(i=0; i<=n-1; i++)
{
for(j=0; j<=m-1; j++)
{
v = 0.0;
for(i_=0; i_<=n-1;i_++)
{
v += a[i,i_]*xe[i_,j];
}
b[i,j] = v;
}
}
//
// Test solvers
//
info = 0;
unsetrep(rep);
cunset2d(ref x);
densesolver.cmatrixsolvem(a, n, b, m, (double)(math.randomreal())>(double)(0.5), ref info, rep, ref x);
cerrors = cerrors | !cmatrixchecksolutionm(xe, n, m, threshold, info, rep, x);
info = 0;
unsetrep(rep);
cunset1d(ref xv);
bv = new complex[n];
for(i_=0; i_<=n-1;i_++)
{
bv[i_] = b[i_,0];
}
densesolver.cmatrixsolve(a, n, bv, ref info, rep, ref xv);
cerrors = cerrors | !cmatrixchecksolution(xe, n, threshold, info, rep, xv);
info = 0;
unsetrep(rep);
cunset2d(ref x);
densesolver.cmatrixlusolvem(lua, p, n, b, m, ref info, rep, ref x);
cerrors = cerrors | !cmatrixchecksolutionm(xe, n, m, threshold, info, rep, x);
info = 0;
unsetrep(rep);
cunset1d(ref xv);
bv = new complex[n];
for(i_=0; i_<=n-1;i_++)
{
bv[i_] = b[i_,0];
}
densesolver.cmatrixlusolve(lua, p, n, bv, ref info, rep, ref xv);
cerrors = cerrors | !cmatrixchecksolution(xe, n, threshold, info, rep, xv);
info = 0;
unsetrep(rep);
cunset2d(ref x);
densesolver.cmatrixmixedsolvem(a, lua, p, n, b, m, ref info, rep, ref x);
cerrors = cerrors | !cmatrixchecksolutionm(xe, n, m, threshold, info, rep, x);
info = 0;
unsetrep(rep);
cunset1d(ref xv);
bv = new complex[n];
for(i_=0; i_<=n-1;i_++)
{
bv[i_] = b[i_,0];
}
densesolver.cmatrixmixedsolve(a, lua, p, n, bv, ref info, rep, ref xv);
cerrors = cerrors | !cmatrixchecksolution(xe, n, threshold, info, rep, xv);
//
// ********************************************************
// EXACTLY SINGULAR MATRICES
// ability to detect singularity is tested
// ********************************************************
//
// 1. generate different types of singular matrices:
// * zero
// * with zero columns
// * with zero rows
// * with equal rows/columns
// 2. generate random solution vector xe
// 3. generate right part b=A*xe
// 4. test different methods
//
for(taskkind=0; taskkind<=4; taskkind++)
{
cunset2d(ref a);
if( taskkind==0 )
{
//
// all zeros
//
a = new complex[n, n];
for(i=0; i<=n-1; i++)
{
for(j=0; j<=n-1; j++)
{
a[i,j] = 0;
}
}
}
if( taskkind==1 )
{
//
// there is zero column
//
a = new complex[n, n];
for(i=0; i<=n-1; i++)
{
for(j=0; j<=n-1; j++)
{
a[i,j].x = 2*math.randomreal()-1;
a[i,j].y = 2*math.randomreal()-1;
}
}
k = math.randominteger(n);
for(i_=0; i_<=n-1;i_++)
{
a[i_,k] = 0*a[i_,k];
}
}
if( taskkind==2 )
{
//
// there is zero row
//
a = new complex[n, n];
for(i=0; i<=n-1; i++)
{
for(j=0; j<=n-1; j++)
{
a[i,j].x = 2*math.randomreal()-1;
a[i,j].y = 2*math.randomreal()-1;
}
}
k = math.randominteger(n);
for(i_=0; i_<=n-1;i_++)
{
a[k,i_] = 0*a[k,i_];
}
}
if( taskkind==3 )
{
//
// equal columns
//
if( n<2 )
{
continue;
}
a = new complex[n, n];
for(i=0; i<=n-1; i++)
{
for(j=0; j<=n-1; j++)
{
a[i,j].x = 2*math.randomreal()-1;
a[i,j].y = 2*math.randomreal()-1;
}
}
k = 1+math.randominteger(n-1);
for(i_=0; i_<=n-1;i_++)
{
a[i_,0] = a[i_,k];
}
}
if( taskkind==4 )
{
//
// equal rows
//
if( n<2 )
{
continue;
}
a = new complex[n, n];
for(i=0; i<=n-1; i++)
{
for(j=0; j<=n-1; j++)
{
a[i,j].x = 2*math.randomreal()-1;
a[i,j].y = 2*math.randomreal()-1;
}
}
k = 1+math.randominteger(n-1);
for(i_=0; i_<=n-1;i_++)
{
a[0,i_] = a[k,i_];
}
}
xe = new complex[n, m];
for(i=0; i<=n-1; i++)
{
for(j=0; j<=m-1; j++)
{
xe[i,j] = 2*math.randomreal()-1;
}
}
b = new complex[n, m];
for(i=0; i<=n-1; i++)
{
for(j=0; j<=m-1; j++)
{
v = 0.0;
for(i_=0; i_<=n-1;i_++)
{
v += a[i,i_]*xe[i_,j];
}
b[i,j] = v;
}
}
cmatrixmakeacopy(a, n, n, ref lua);
trfac.cmatrixlu(ref lua, n, n, ref p);
//
// Test CMatrixSolveM()
//
info = 0;
unsetrep(rep);
cunset2d(ref x);
densesolver.cmatrixsolvem(a, n, b, m, (double)(math.randomreal())>(double)(0.5), ref info, rep, ref x);
cerrors = cerrors | !cmatrixchecksingularm(n, m, info, rep, x);
//
// Test CMatrixSolve()
//
info = 0;
unsetrep(rep);
cunset2d(ref x);
bv = new complex[n];
for(i_=0; i_<=n-1;i_++)
{
bv[i_] = b[i_,0];
}
densesolver.cmatrixsolve(a, n, bv, ref info, rep, ref xv);
cerrors = cerrors | !cmatrixchecksingular(n, info, rep, xv);
//
// Test CMatrixLUSolveM()
//
info = 0;
unsetrep(rep);
cunset2d(ref x);
densesolver.cmatrixlusolvem(lua, p, n, b, m, ref info, rep, ref x);
cerrors = cerrors | !cmatrixchecksingularm(n, m, info, rep, x);
//
// Test CMatrixLUSolve()
//
info = 0;
unsetrep(rep);
cunset2d(ref x);
bv = new complex[n];
for(i_=0; i_<=n-1;i_++)
{
bv[i_] = b[i_,0];
}
densesolver.cmatrixlusolve(lua, p, n, bv, ref info, rep, ref xv);
cerrors = cerrors | !cmatrixchecksingular(n, info, rep, xv);
//
// Test CMatrixMixedSolveM()
//
info = 0;
unsetrep(rep);
cunset2d(ref x);
densesolver.cmatrixmixedsolvem(a, lua, p, n, b, m, ref info, rep, ref x);
cerrors = cerrors | !cmatrixchecksingularm(n, m, info, rep, x);
//
// Test CMatrixMixedSolve()
//
info = 0;
unsetrep(rep);
cunset2d(ref x);
bv = new complex[n];
for(i_=0; i_<=n-1;i_++)
{
bv[i_] = b[i_,0];
}
densesolver.cmatrixmixedsolve(a, lua, p, n, bv, ref info, rep, ref xv);
cerrors = cerrors | !cmatrixchecksingular(n, info, rep, xv);
}
}
}
}
//
// test iterative improvement
//
for(pass=1; pass<=passcount; pass++)
{
//
// Test iterative improvement matrices
//
// A matrix/right part are constructed such that both matrix
// and solution components magnitudes are within (-1,+1).
// Such matrix/right part have nice properties - system can
// be solved using iterative improvement with |A*x-b| about
// several ulps of max(1,|b|).
//
n = 100;
a = new complex[n, n];
b = new complex[n, 1];
bv = new complex[n];
tx = new double[2*n];
xv = new complex[n];
y = new complex[n];
for(i=0; i<=n-1; i++)
{
xv[i].x = 2*math.randomreal()-1;
xv[i].y = 2*math.randomreal()-1;
}
for(i=0; i<=n-1; i++)
{
for(j=0; j<=n-1; j++)
{
a[i,j].x = 2*math.randomreal()-1;
a[i,j].y = 2*math.randomreal()-1;
}
for(i_=0; i_<=n-1;i_++)
{
y[i_] = a[i,i_];
}
xblas.xcdot(y, xv, n, ref tx, ref v, ref verr);
bv[i] = v;
}
for(i_=0; i_<=n-1;i_++)
{
b[i_,0] = bv[i_];
}
//
// Test CMatrixSolveM()
//
cunset2d(ref x);
densesolver.cmatrixsolvem(a, n, b, 1, true, ref info, rep, ref x);
if( info<=0 )
{
rfserrors = true;
}
else
{
xv = new complex[n];
for(i_=0; i_<=n-1;i_++)
{
xv[i_] = x[i_,0];
}
for(i=0; i<=n-1; i++)
{
for(i_=0; i_<=n-1;i_++)
{
y[i_] = a[i,i_];
}
xblas.xcdot(y, xv, n, ref tx, ref v, ref verr);
rfserrors = rfserrors | (double)(math.abscomplex(v-b[i,0]))>(double)(8*math.machineepsilon*Math.Max(1, math.abscomplex(b[i,0])));
}
}
//
// Test CMatrixSolve()
//
cunset1d(ref xv);
densesolver.cmatrixsolve(a, n, bv, ref info, rep, ref xv);
if( info<=0 )
{
rfserrors = true;
}
else
{
for(i=0; i<=n-1; i++)
{
for(i_=0; i_<=n-1;i_++)
{
y[i_] = a[i,i_];
}
xblas.xcdot(y, xv, n, ref tx, ref v, ref verr);
rfserrors = rfserrors | (double)(math.abscomplex(v-bv[i]))>(double)(8*math.machineepsilon*Math.Max(1, math.abscomplex(bv[i])));
}
}
//
// TODO: Test LS-solver on the same matrix
//
}
}
/*************************************************************************
Real test
*************************************************************************/
private static void testrsolver(int maxn,
int maxm,
int passcount,
double threshold,
ref bool rerrors,
ref bool rfserrors)
{
double[,] a = new double[0,0];
double[,] lua = new double[0,0];
double[,] atmp = new double[0,0];
int[] p = new int[0];
double[,] xe = new double[0,0];
double[,] b = new double[0,0];
double[] bv = new double[0];
int i = 0;
int j = 0;
int k = 0;
int n = 0;
int m = 0;
int pass = 0;
int taskkind = 0;
double v = 0;
double verr = 0;
int info = 0;
densesolver.densesolverreport rep = new densesolver.densesolverreport();
densesolver.densesolverlsreport repls = new densesolver.densesolverlsreport();
double[,] x = new double[0,0];
double[] xv = new double[0];
double[] y = new double[0];
double[] tx = new double[0];
int i_ = 0;
int i1_ = 0;
//
// General square matrices:
// * test general solvers
// * test least squares solver
//
for(pass=1; pass<=passcount; pass++)
{
for(n=1; n<=maxn; n++)
{
for(m=1; m<=maxm; m++)
{
//
// ********************************************************
// WELL CONDITIONED TASKS
// ability to find correct solution is tested
// ********************************************************
//
// 1. generate random well conditioned matrix A.
// 2. generate random solution vector xe
// 3. generate right part b=A*xe
// 4. test different methods on original A
//
matgen.rmatrixrndcond(n, 1000, ref a);
rmatrixmakeacopy(a, n, n, ref lua);
trfac.rmatrixlu(ref lua, n, n, ref p);
xe = new double[n, m];
for(i=0; i<=n-1; i++)
{
for(j=0; j<=m-1; j++)
{
xe[i,j] = 2*math.randomreal()-1;
}
}
b = new double[n, m];
for(i=0; i<=n-1; i++)
{
for(j=0; j<=m-1; j++)
{
v = 0.0;
for(i_=0; i_<=n-1;i_++)
{
v += a[i,i_]*xe[i_,j];
}
b[i,j] = v;
}
}
//
// Test solvers
//
info = 0;
unsetrep(rep);
unset2d(ref x);
densesolver.rmatrixsolvem(a, n, b, m, (double)(math.randomreal())>(double)(0.5), ref info, rep, ref x);
rerrors = rerrors | !rmatrixchecksolutionm(xe, n, m, threshold, info, rep, x);
info = 0;
unsetrep(rep);
unset1d(ref xv);
bv = new double[n];
for(i_=0; i_<=n-1;i_++)
{
bv[i_] = b[i_,0];
}
densesolver.rmatrixsolve(a, n, bv, ref info, rep, ref xv);
rerrors = rerrors | !rmatrixchecksolution(xe, n, threshold, info, rep, xv);
info = 0;
unsetrep(rep);
unset2d(ref x);
densesolver.rmatrixlusolvem(lua, p, n, b, m, ref info, rep, ref x);
rerrors = rerrors | !rmatrixchecksolutionm(xe, n, m, threshold, info, rep, x);
info = 0;
unsetrep(rep);
unset1d(ref xv);
bv = new double[n];
for(i_=0; i_<=n-1;i_++)
{
bv[i_] = b[i_,0];
}
densesolver.rmatrixlusolve(lua, p, n, bv, ref info, rep, ref xv);
rerrors = rerrors | !rmatrixchecksolution(xe, n, threshold, info, rep, xv);
info = 0;
unsetrep(rep);
unset2d(ref x);
densesolver.rmatrixmixedsolvem(a, lua, p, n, b, m, ref info, rep, ref x);
rerrors = rerrors | !rmatrixchecksolutionm(xe, n, m, threshold, info, rep, x);
info = 0;
unsetrep(rep);
unset1d(ref xv);
bv = new double[n];
for(i_=0; i_<=n-1;i_++)
{
bv[i_] = b[i_,0];
}
densesolver.rmatrixmixedsolve(a, lua, p, n, bv, ref info, rep, ref xv);
rerrors = rerrors | !rmatrixchecksolution(xe, n, threshold, info, rep, xv);
//
// Test DenseSolverRLS():
// * test on original system A*x = b
// * test on overdetermined system with the same solution: (A' A')'*x = (b' b')'
// * test on underdetermined system with the same solution: (A 0 0 0 ) * z = b
//
info = 0;
unsetlsrep(repls);
unset1d(ref xv);
bv = new double[n];
for(i_=0; i_<=n-1;i_++)
{
bv[i_] = b[i_,0];
}
densesolver.rmatrixsolvels(a, n, n, bv, 0.0, ref info, repls, ref xv);
if( info<=0 )
{
rerrors = true;
}
else
{
rerrors = (rerrors | (double)(repls.r2)<(double)(100*math.machineepsilon)) | (double)(repls.r2)>(double)(1+1000*math.machineepsilon);
rerrors = (rerrors | repls.n!=n) | repls.k!=0;
for(i=0; i<=n-1; i++)
{
rerrors = rerrors | (double)(Math.Abs(xe[i,0]-xv[i]))>(double)(threshold);
}
}
info = 0;
unsetlsrep(repls);
unset1d(ref xv);
bv = new double[2*n];
for(i_=0; i_<=n-1;i_++)
{
bv[i_] = b[i_,0];
}
i1_ = (0) - (n);
for(i_=n; i_<=2*n-1;i_++)
{
bv[i_] = b[i_+i1_,0];
}
atmp = new double[2*n, n];
blas.copymatrix(a, 0, n-1, 0, n-1, ref atmp, 0, n-1, 0, n-1);
blas.copymatrix(a, 0, n-1, 0, n-1, ref atmp, n, 2*n-1, 0, n-1);
densesolver.rmatrixsolvels(atmp, 2*n, n, bv, 0.0, ref info, repls, ref xv);
if( info<=0 )
{
rerrors = true;
}
else
{
rerrors = (rerrors | (double)(repls.r2)<(double)(100*math.machineepsilon)) | (double)(repls.r2)>(double)(1+1000*math.machineepsilon);
rerrors = (rerrors | repls.n!=n) | repls.k!=0;
for(i=0; i<=n-1; i++)
{
rerrors = rerrors | (double)(Math.Abs(xe[i,0]-xv[i]))>(double)(threshold);
}
}
info = 0;
unsetlsrep(repls);
unset1d(ref xv);
bv = new double[n];
for(i_=0; i_<=n-1;i_++)
{
bv[i_] = b[i_,0];
}
atmp = new double[n, 2*n];
blas.copymatrix(a, 0, n-1, 0, n-1, ref atmp, 0, n-1, 0, n-1);
for(i=0; i<=n-1; i++)
{
for(j=n; j<=2*n-1; j++)
{
atmp[i,j] = 0;
}
}
densesolver.rmatrixsolvels(atmp, n, 2*n, bv, 0.0, ref info, repls, ref xv);
if( info<=0 )
{
rerrors = true;
}
else
{
rerrors = rerrors | (double)(repls.r2)!=(double)(0);
rerrors = (rerrors | repls.n!=2*n) | repls.k!=n;
for(i=0; i<=n-1; i++)
{
rerrors = rerrors | (double)(Math.Abs(xe[i,0]-xv[i]))>(double)(threshold);
}
for(i=n; i<=2*n-1; i++)
{
rerrors = rerrors | (double)(Math.Abs(xv[i]))>(double)(threshold);
}
}
//
// ********************************************************
// EXACTLY SINGULAR MATRICES
// ability to detect singularity is tested
// ********************************************************
//
// 1. generate different types of singular matrices:
// * zero
// * with zero columns
// * with zero rows
// * with equal rows/columns
// 2. generate random solution vector xe
// 3. generate right part b=A*xe
// 4. test different methods
//
for(taskkind=0; taskkind<=4; taskkind++)
{
unset2d(ref a);
if( taskkind==0 )
{
//
// all zeros
//
a = new double[n, n];
for(i=0; i<=n-1; i++)
{
for(j=0; j<=n-1; j++)
{
a[i,j] = 0;
}
}
}
if( taskkind==1 )
{
//
// there is zero column
//
a = new double[n, n];
for(i=0; i<=n-1; i++)
{
for(j=0; j<=n-1; j++)
{
a[i,j] = 2*math.randomreal()-1;
}
}
k = math.randominteger(n);
for(i_=0; i_<=n-1;i_++)
{
a[i_,k] = 0*a[i_,k];
}
}
if( taskkind==2 )
{
//
// there is zero row
//
a = new double[n, n];
for(i=0; i<=n-1; i++)
{
for(j=0; j<=n-1; j++)
{
a[i,j] = 2*math.randomreal()-1;
}
}
k = math.randominteger(n);
for(i_=0; i_<=n-1;i_++)
{
a[k,i_] = 0*a[k,i_];
}
}
if( taskkind==3 )
{
//
// equal columns
//
if( n<2 )
{
continue;
}
a = new double[n, n];
for(i=0; i<=n-1; i++)
{
for(j=0; j<=n-1; j++)
{
a[i,j] = 2*math.randomreal()-1;
}
}
k = 1+math.randominteger(n-1);
for(i_=0; i_<=n-1;i_++)
{
a[i_,0] = a[i_,k];
}
}
if( taskkind==4 )
{
//
// equal rows
//
if( n<2 )
{
continue;
}
a = new double[n, n];
for(i=0; i<=n-1; i++)
{
for(j=0; j<=n-1; j++)
{
a[i,j] = 2*math.randomreal()-1;
}
}
k = 1+math.randominteger(n-1);
for(i_=0; i_<=n-1;i_++)
{
a[0,i_] = a[k,i_];
}
}
xe = new double[n, m];
for(i=0; i<=n-1; i++)
{
for(j=0; j<=m-1; j++)
{
xe[i,j] = 2*math.randomreal()-1;
}
}
b = new double[n, m];
for(i=0; i<=n-1; i++)
{
for(j=0; j<=m-1; j++)
{
v = 0.0;
for(i_=0; i_<=n-1;i_++)
{
v += a[i,i_]*xe[i_,j];
}
b[i,j] = v;
}
}
rmatrixmakeacopy(a, n, n, ref lua);
trfac.rmatrixlu(ref lua, n, n, ref p);
//
// Test RMatrixSolveM()
//
info = 0;
unsetrep(rep);
unset2d(ref x);
densesolver.rmatrixsolvem(a, n, b, m, (double)(math.randomreal())>(double)(0.5), ref info, rep, ref x);
rerrors = rerrors | !rmatrixchecksingularm(n, m, info, rep, x);
//
// Test RMatrixSolve()
//
info = 0;
unsetrep(rep);
unset2d(ref x);
bv = new double[n];
for(i_=0; i_<=n-1;i_++)
{
bv[i_] = b[i_,0];
}
densesolver.rmatrixsolve(a, n, bv, ref info, rep, ref xv);
rerrors = rerrors | !rmatrixchecksingular(n, info, rep, xv);
//
// Test RMatrixLUSolveM()
//
info = 0;
unsetrep(rep);
unset2d(ref x);
densesolver.rmatrixlusolvem(lua, p, n, b, m, ref info, rep, ref x);
rerrors = rerrors | !rmatrixchecksingularm(n, m, info, rep, x);
//
// Test RMatrixLUSolve()
//
info = 0;
unsetrep(rep);
unset2d(ref x);
bv = new double[n];
for(i_=0; i_<=n-1;i_++)
{
bv[i_] = b[i_,0];
}
densesolver.rmatrixlusolve(lua, p, n, bv, ref info, rep, ref xv);
rerrors = rerrors | !rmatrixchecksingular(n, info, rep, xv);
//
// Test RMatrixMixedSolveM()
//
info = 0;
unsetrep(rep);
unset2d(ref x);
densesolver.rmatrixmixedsolvem(a, lua, p, n, b, m, ref info, rep, ref x);
rerrors = rerrors | !rmatrixchecksingularm(n, m, info, rep, x);
//
// Test RMatrixMixedSolve()
//
info = 0;
unsetrep(rep);
unset2d(ref x);
bv = new double[n];
for(i_=0; i_<=n-1;i_++)
{
bv[i_] = b[i_,0];
}
densesolver.rmatrixmixedsolve(a, lua, p, n, bv, ref info, rep, ref xv);
rerrors = rerrors | !rmatrixchecksingular(n, info, rep, xv);
}
}
}
}
//
// test iterative improvement
//
for(pass=1; pass<=passcount; pass++)
{
//
// Test iterative improvement matrices
//
// A matrix/right part are constructed such that both matrix
// and solution components are within (-1,+1). Such matrix/right part
// have nice properties - system can be solved using iterative
// improvement with |A*x-b| about several ulps of max(1,|b|).
//
n = 100;
a = new double[n, n];
b = new double[n, 1];
bv = new double[n];
tx = new double[n];
xv = new double[n];
y = new double[n];
for(i=0; i<=n-1; i++)
{
xv[i] = 2*math.randomreal()-1;
}
for(i=0; i<=n-1; i++)
{
for(j=0; j<=n-1; j++)
{
a[i,j] = 2*math.randomreal()-1;
}
for(i_=0; i_<=n-1;i_++)
{
y[i_] = a[i,i_];
}
xblas.xdot(y, xv, n, ref tx, ref v, ref verr);
bv[i] = v;
}
for(i_=0; i_<=n-1;i_++)
{
b[i_,0] = bv[i_];
}
//
// Test RMatrixSolveM()
//
unset2d(ref x);
densesolver.rmatrixsolvem(a, n, b, 1, true, ref info, rep, ref x);
if( info<=0 )
{
rfserrors = true;
}
else
{
xv = new double[n];
for(i_=0; i_<=n-1;i_++)
{
xv[i_] = x[i_,0];
}
for(i=0; i<=n-1; i++)
{
for(i_=0; i_<=n-1;i_++)
{
y[i_] = a[i,i_];
}
xblas.xdot(y, xv, n, ref tx, ref v, ref verr);
rfserrors = rfserrors | (double)(Math.Abs(v-b[i,0]))>(double)(8*math.machineepsilon*Math.Max(1, Math.Abs(b[i,0])));
}
}
//
// Test RMatrixSolve()
//
unset1d(ref xv);
densesolver.rmatrixsolve(a, n, bv, ref info, rep, ref xv);
if( info<=0 )
{
rfserrors = true;
}
else
{
for(i=0; i<=n-1; i++)
{
for(i_=0; i_<=n-1;i_++)
{
y[i_] = a[i,i_];
}
xblas.xdot(y, xv, n, ref tx, ref v, ref verr);
rfserrors = rfserrors | (double)(Math.Abs(v-bv[i]))>(double)(8*math.machineepsilon*Math.Max(1, Math.Abs(bv[i])));
}
}
//
// Test LS-solver on the same matrix
//
densesolver.rmatrixsolvels(a, n, n, bv, 0.0, ref info, repls, ref xv);
if( info<=0 )
{
rfserrors = true;
}
else
{
for(i=0; i<=n-1; i++)
{
for(i_=0; i_<=n-1;i_++)
{
y[i_] = a[i,i_];
}
xblas.xdot(y, xv, n, ref tx, ref v, ref verr);
rfserrors = rfserrors | (double)(Math.Abs(v-bv[i]))>(double)(8*math.machineepsilon*Math.Max(1, Math.Abs(bv[i])));
}
}
}
}
public static bool testdensesolver(bool silent)
{
bool result = new bool();
double[,] a = new double[0,0];
double[,] lua = new double[0,0];
double[,] atmp = new double[0,0];
int[] p = new int[0];
double[,] xe = new double[0,0];
double[,] b = new double[0,0];
double[] bv = new double[0];
int i = 0;
int j = 0;
int k = 0;
int n = 0;
int m = 0;
int pass = 0;
int taskkind = 0;
double mx = 0;
double v = 0;
double verr = 0;
int info = 0;
densesolver.densesolverreport rep = new densesolver.densesolverreport();
densesolver.densesolverlsreport repls = new densesolver.densesolverlsreport();
double[,] x = new double[0,0];
double[] xv = new double[0];
double[] y = new double[0];
double[] tx = new double[0];
int maxn = 0;
int maxm = 0;
int passcount = 0;
double threshold = 0;
bool rerrors = new bool();
bool cerrors = new bool();
bool spderrors = new bool();
bool hpderrors = new bool();
bool rfserrors = new bool();
bool waserrors = new bool();
maxn = 10;
maxm = 5;
passcount = 5;
threshold = 10000*math.machineepsilon;
rfserrors = false;
rerrors = false;
cerrors = false;
spderrors = false;
hpderrors = false;
testrsolver(maxn, maxm, passcount, threshold, ref rerrors, ref rfserrors);
testspdsolver(maxn, maxm, passcount, threshold, ref spderrors, ref rfserrors);
testcsolver(maxn, maxm, passcount, threshold, ref cerrors, ref rfserrors);
testhpdsolver(maxn, maxm, passcount, threshold, ref hpderrors, ref rfserrors);
waserrors = (((rerrors | cerrors) | spderrors) | hpderrors) | rfserrors;
if( !silent )
{
System.Console.Write("TESTING DENSE SOLVER");
System.Console.WriteLine();
System.Console.Write("* REAL: ");
if( rerrors )
{
System.Console.Write("FAILED");
System.Console.WriteLine();
}
else
{
System.Console.Write("OK");
System.Console.WriteLine();
}
System.Console.Write("* COMPLEX: ");
if( cerrors )
{
System.Console.Write("FAILED");
System.Console.WriteLine();
}
else
{
System.Console.Write("OK");
System.Console.WriteLine();
}
System.Console.Write("* SPD: ");
if( spderrors )
{
System.Console.Write("FAILED");
System.Console.WriteLine();
}
else
{
System.Console.Write("OK");
System.Console.WriteLine();
}
System.Console.Write("* HPD: ");
if( hpderrors )
{
System.Console.Write("FAILED");
System.Console.WriteLine();
}
else
{
System.Console.Write("OK");
System.Console.WriteLine();
}
System.Console.Write("* ITERATIVE IMPROVEMENT: ");
if( rfserrors )
{
System.Console.Write("FAILED");
System.Console.WriteLine();
}
else
{
System.Console.Write("OK");
System.Console.WriteLine();
}
if( waserrors )
{
System.Console.Write("TEST FAILED");
System.Console.WriteLine();
}
else
{
System.Console.Write("TEST PASSED");
System.Console.WriteLine();
}
}
result = !waserrors;
return result;
}
/*************************************************************************
* Neural network training using modified Levenberg-Marquardt with exact
* Hessian calculation and regularization. Subroutine trains neural network
* with restarts from random positions. Algorithm is well suited for small
* and medium scale problems (hundreds of weights).
*
* INPUT PARAMETERS:
* Network - neural network with initialized geometry
* XY - training set
* NPoints - training set size
* Decay - weight decay constant, >=0.001
* Decay term 'Decay*||Weights||^2' is added to error
* function.
* If you don't know what Decay to choose, use 0.001.
* Restarts - number of restarts from random position, >0.
* If you don't know what Restarts to choose, use 2.
*
* OUTPUT PARAMETERS:
* Network - trained neural network.
* Info - return code:
* -9, if internal matrix inverse subroutine failed
* -2, if there is a point with class number
* outside of [0..NOut-1].
* -1, if wrong parameters specified
* (NPoints<0, Restarts<1).
* 2, if task has been solved.
* Rep - training report
*
* -- ALGLIB --
* Copyright 10.03.2009 by Bochkanov Sergey
*************************************************************************/
public static void mlptrainlm(ref mlpbase.multilayerperceptron network,
ref double[,] xy,
int npoints,
double decay,
int restarts,
ref int info,
ref mlpreport rep)
{
int nin = 0;
int nout = 0;
int wcount = 0;
double lmftol = 0;
double lmsteptol = 0;
int i = 0;
int k = 0;
double v = 0;
double e = 0;
double enew = 0;
double xnorm2 = 0;
double stepnorm = 0;
double[] g = new double[0];
double[] d = new double[0];
double[,] h = new double[0, 0];
double[,] hmod = new double[0, 0];
double[,] z = new double[0, 0];
bool spd = new bool();
double nu = 0;
double lambda = 0;
double lambdaup = 0;
double lambdadown = 0;
minlbfgs.minlbfgsreport internalrep = new minlbfgs.minlbfgsreport();
minlbfgs.minlbfgsstate state = new minlbfgs.minlbfgsstate();
double[] x = new double[0];
double[] y = new double[0];
double[] wbase = new double[0];
double[] wdir = new double[0];
double[] wt = new double[0];
double[] wx = new double[0];
int pass = 0;
double[] wbest = new double[0];
double ebest = 0;
int invinfo = 0;
matinv.matinvreport invrep = new matinv.matinvreport();
int solverinfo = 0;
densesolver.densesolverreport solverrep = new densesolver.densesolverreport();
int i_ = 0;
mlpbase.mlpproperties(ref network, ref nin, ref nout, ref wcount);
lambdaup = 10;
lambdadown = 0.3;
lmftol = 0.001;
lmsteptol = 0.001;
//
// Test for inputs
//
if (npoints <= 0 | restarts < 1)
{
info = -1;
return;
}
if (mlpbase.mlpissoftmax(ref network))
{
for (i = 0; i <= npoints - 1; i++)
{
if ((int)Math.Round(xy[i, nin]) < 0 | (int)Math.Round(xy[i, nin]) >= nout)
{
info = -2;
return;
}
}
}
decay = Math.Max(decay, mindecay);
info = 2;
//
// Initialize data
//
rep.ngrad = 0;
rep.nhess = 0;
rep.ncholesky = 0;
//
// General case.
// Prepare task and network. Allocate space.
//
mlpbase.mlpinitpreprocessor(ref network, ref xy, npoints);
g = new double[wcount - 1 + 1];
h = new double[wcount - 1 + 1, wcount - 1 + 1];
hmod = new double[wcount - 1 + 1, wcount - 1 + 1];
wbase = new double[wcount - 1 + 1];
wdir = new double[wcount - 1 + 1];
wbest = new double[wcount - 1 + 1];
wt = new double[wcount - 1 + 1];
wx = new double[wcount - 1 + 1];
ebest = AP.Math.MaxRealNumber;
//
// Multiple passes
//
for (pass = 1; pass <= restarts; pass++)
{
//
// Initialize weights
//
mlpbase.mlprandomize(ref network);
//
// First stage of the hybrid algorithm: LBFGS
//
for (i_ = 0; i_ <= wcount - 1; i_++)
{
wbase[i_] = network.weights[i_];
}
minlbfgs.minlbfgscreate(wcount, Math.Min(wcount, 5), ref wbase, ref state);
minlbfgs.minlbfgssetcond(ref state, 0, 0, 0, Math.Max(25, wcount));
while (minlbfgs.minlbfgsiteration(ref state))
{
//
// gradient
//
for (i_ = 0; i_ <= wcount - 1; i_++)
{
network.weights[i_] = state.x[i_];
}
mlpbase.mlpgradbatch(ref network, ref xy, npoints, ref state.f, ref state.g);
//
// weight decay
//
v = 0.0;
for (i_ = 0; i_ <= wcount - 1; i_++)
{
v += network.weights[i_] * network.weights[i_];
}
state.f = state.f + 0.5 * decay * v;
for (i_ = 0; i_ <= wcount - 1; i_++)
{
state.g[i_] = state.g[i_] + decay * network.weights[i_];
}
//
// next iteration
//
rep.ngrad = rep.ngrad + 1;
}
minlbfgs.minlbfgsresults(ref state, ref wbase, ref internalrep);
for (i_ = 0; i_ <= wcount - 1; i_++)
{
network.weights[i_] = wbase[i_];
}
//
// Second stage of the hybrid algorithm: LM
//
// Initialize H with identity matrix,
// G with gradient,
// E with regularized error.
//
mlpbase.mlphessianbatch(ref network, ref xy, npoints, ref e, ref g, ref h);
v = 0.0;
for (i_ = 0; i_ <= wcount - 1; i_++)
{
v += network.weights[i_] * network.weights[i_];
}
e = e + 0.5 * decay * v;
for (i_ = 0; i_ <= wcount - 1; i_++)
{
g[i_] = g[i_] + decay * network.weights[i_];
}
for (k = 0; k <= wcount - 1; k++)
{
h[k, k] = h[k, k] + decay;
}
rep.nhess = rep.nhess + 1;
lambda = 0.001;
nu = 2;
while (true)
{
//
// 1. HMod = H+lambda*I
// 2. Try to solve (H+Lambda*I)*dx = -g.
// Increase lambda if left part is not positive definite.
//
for (i = 0; i <= wcount - 1; i++)
{
for (i_ = 0; i_ <= wcount - 1; i_++)
{
hmod[i, i_] = h[i, i_];
}
hmod[i, i] = hmod[i, i] + lambda;
}
spd = trfac.spdmatrixcholesky(ref hmod, wcount, true);
rep.ncholesky = rep.ncholesky + 1;
if (!spd)
{
lambda = lambda * lambdaup * nu;
nu = nu * 2;
continue;
}
densesolver.spdmatrixcholeskysolve(ref hmod, wcount, true, ref g, ref solverinfo, ref solverrep, ref wdir);
if (solverinfo < 0)
{
lambda = lambda * lambdaup * nu;
nu = nu * 2;
continue;
}
for (i_ = 0; i_ <= wcount - 1; i_++)
{
wdir[i_] = -1 * wdir[i_];
}
//
// Lambda found.
// 1. Save old w in WBase
// 1. Test some stopping criterions
// 2. If error(w+wdir)>error(w), increase lambda
//
for (i_ = 0; i_ <= wcount - 1; i_++)
{
network.weights[i_] = network.weights[i_] + wdir[i_];
}
xnorm2 = 0.0;
for (i_ = 0; i_ <= wcount - 1; i_++)
{
xnorm2 += network.weights[i_] * network.weights[i_];
}
stepnorm = 0.0;
for (i_ = 0; i_ <= wcount - 1; i_++)
{
stepnorm += wdir[i_] * wdir[i_];
}
stepnorm = Math.Sqrt(stepnorm);
enew = mlpbase.mlperror(ref network, ref xy, npoints) + 0.5 * decay * xnorm2;
if ((double)(stepnorm) < (double)(lmsteptol * (1 + Math.Sqrt(xnorm2))))
{
break;
}
if ((double)(enew) > (double)(e))
{
lambda = lambda * lambdaup * nu;
nu = nu * 2;
continue;
}
//
// Optimize using inv(cholesky(H)) as preconditioner
//
matinv.rmatrixtrinverse(ref hmod, wcount, true, false, ref invinfo, ref invrep);
if (invinfo <= 0)
{
//
// if matrix can't be inverted then exit with errors
// TODO: make WCount steps in direction suggested by HMod
//
info = -9;
return;
}
for (i_ = 0; i_ <= wcount - 1; i_++)
{
wbase[i_] = network.weights[i_];
}
for (i = 0; i <= wcount - 1; i++)
{
wt[i] = 0;
}
minlbfgs.minlbfgscreatex(wcount, wcount, ref wt, 1, ref state);
minlbfgs.minlbfgssetcond(ref state, 0, 0, 0, 5);
while (minlbfgs.minlbfgsiteration(ref state))
{
//
// gradient
//
for (i = 0; i <= wcount - 1; i++)
{
v = 0.0;
for (i_ = i; i_ <= wcount - 1; i_++)
{
v += state.x[i_] * hmod[i, i_];
}
network.weights[i] = wbase[i] + v;
}
mlpbase.mlpgradbatch(ref network, ref xy, npoints, ref state.f, ref g);
for (i = 0; i <= wcount - 1; i++)
{
state.g[i] = 0;
}
for (i = 0; i <= wcount - 1; i++)
{
v = g[i];
for (i_ = i; i_ <= wcount - 1; i_++)
{
state.g[i_] = state.g[i_] + v * hmod[i, i_];
}
}
//
// weight decay
// grad(x'*x) = A'*(x0+A*t)
//
v = 0.0;
for (i_ = 0; i_ <= wcount - 1; i_++)
{
v += network.weights[i_] * network.weights[i_];
}
state.f = state.f + 0.5 * decay * v;
for (i = 0; i <= wcount - 1; i++)
{
v = decay * network.weights[i];
for (i_ = i; i_ <= wcount - 1; i_++)
{
state.g[i_] = state.g[i_] + v * hmod[i, i_];
}
}
//
// next iteration
//
rep.ngrad = rep.ngrad + 1;
}
minlbfgs.minlbfgsresults(ref state, ref wt, ref internalrep);
//
// Accept new position.
// Calculate Hessian
//
for (i = 0; i <= wcount - 1; i++)
{
v = 0.0;
for (i_ = i; i_ <= wcount - 1; i_++)
{
v += wt[i_] * hmod[i, i_];
}
network.weights[i] = wbase[i] + v;
}
mlpbase.mlphessianbatch(ref network, ref xy, npoints, ref e, ref g, ref h);
v = 0.0;
for (i_ = 0; i_ <= wcount - 1; i_++)
{
v += network.weights[i_] * network.weights[i_];
}
e = e + 0.5 * decay * v;
for (i_ = 0; i_ <= wcount - 1; i_++)
{
g[i_] = g[i_] + decay * network.weights[i_];
}
for (k = 0; k <= wcount - 1; k++)
{
h[k, k] = h[k, k] + decay;
}
rep.nhess = rep.nhess + 1;
//
// Update lambda
//
lambda = lambda * lambdadown;
nu = 2;
}
//
// update WBest
//
v = 0.0;
for (i_ = 0; i_ <= wcount - 1; i_++)
{
v += network.weights[i_] * network.weights[i_];
}
e = 0.5 * decay * v + mlpbase.mlperror(ref network, ref xy, npoints);
if ((double)(e) < (double)(ebest))
{
ebest = e;
for (i_ = 0; i_ <= wcount - 1; i_++)
{
wbest[i_] = network.weights[i_];
}
}
}
//
// copy WBest to output
//
for (i_ = 0; i_ <= wcount - 1; i_++)
{
network.weights[i_] = wbest[i_];
}
}
