/*************************************************************************
Real test
*************************************************************************/
private static void testrsolver(int maxn,
int maxm,
int passcount,
double threshold,
ref bool rerrors,
ref bool rfserrors)
{
double[,] a = new double[0,0];
double[,] lua = new double[0,0];
double[,] atmp = new double[0,0];
int[] p = new int[0];
double[,] xe = new double[0,0];
double[,] b = new double[0,0];
double[] bv = new double[0];
int i = 0;
int j = 0;
int k = 0;
int n = 0;
int m = 0;
int pass = 0;
int taskkind = 0;
double mx = 0;
double v = 0;
double verr = 0;
int info = 0;
densesolver.densesolverreport rep = new densesolver.densesolverreport();
densesolver.densesolverlsreport repls = new densesolver.densesolverlsreport();
double[,] x = new double[0,0];
double[] xv = new double[0];
double[] y = new double[0];
double[] tx = new double[0];
int i_ = 0;
int i1_ = 0;
//
// General square matrices:
// * test general solvers
// * test least squares solver
//
for(pass=1; pass<=passcount; pass++)
{
for(n=1; n<=maxn; n++)
{
for(m=1; m<=maxm; m++)
{
//
// ********************************************************
// WELL CONDITIONED TASKS
// ability to find correct solution is tested
// ********************************************************
//
// 1. generate random well conditioned matrix A.
// 2. generate random solution vector xe
// 3. generate right part b=A*xe
// 4. test different methods on original A
//
matgen.rmatrixrndcond(n, 1000, ref a);
rmatrixmakeacopy(ref a, n, n, ref lua);
trfac.rmatrixlu(ref lua, n, n, ref p);
xe = new double[n, m];
for(i=0; i<=n-1; i++)
{
for(j=0; j<=m-1; j++)
{
xe[i,j] = 2*AP.Math.RandomReal()-1;
}
}
b = new double[n, m];
for(i=0; i<=n-1; i++)
{
for(j=0; j<=m-1; j++)
{
v = 0.0;
for(i_=0; i_<=n-1;i_++)
{
v += a[i,i_]*xe[i_,j];
}
b[i,j] = v;
}
}
//
// Test solvers
//
info = 0;
unsetrep(ref rep);
unset2d(ref x);
densesolver.rmatrixsolvem(ref a, n, ref b, m, (double)(AP.Math.RandomReal())>(double)(0.5), ref info, ref rep, ref x);
rerrors = rerrors | !rmatrixchecksolutionm(ref xe, n, m, threshold, info, ref rep, ref x);
info = 0;
unsetrep(ref rep);
unset1d(ref xv);
bv = new double[n];
for(i_=0; i_<=n-1;i_++)
{
bv[i_] = b[i_,0];
}
densesolver.rmatrixsolve(ref a, n, ref bv, ref info, ref rep, ref xv);
rerrors = rerrors | !rmatrixchecksolution(ref xe, n, threshold, info, ref rep, ref xv);
info = 0;
unsetrep(ref rep);
unset2d(ref x);
densesolver.rmatrixlusolvem(ref lua, ref p, n, ref b, m, ref info, ref rep, ref x);
rerrors = rerrors | !rmatrixchecksolutionm(ref xe, n, m, threshold, info, ref rep, ref x);
info = 0;
unsetrep(ref rep);
unset1d(ref xv);
bv = new double[n];
for(i_=0; i_<=n-1;i_++)
{
bv[i_] = b[i_,0];
}
densesolver.rmatrixlusolve(ref lua, ref p, n, ref bv, ref info, ref rep, ref xv);
rerrors = rerrors | !rmatrixchecksolution(ref xe, n, threshold, info, ref rep, ref xv);
info = 0;
unsetrep(ref rep);
unset2d(ref x);
densesolver.rmatrixmixedsolvem(ref a, ref lua, ref p, n, ref b, m, ref info, ref rep, ref x);
rerrors = rerrors | !rmatrixchecksolutionm(ref xe, n, m, threshold, info, ref rep, ref x);
info = 0;
unsetrep(ref rep);
unset1d(ref xv);
bv = new double[n];
for(i_=0; i_<=n-1;i_++)
{
bv[i_] = b[i_,0];
}
densesolver.rmatrixmixedsolve(ref a, ref lua, ref p, n, ref bv, ref info, ref rep, ref xv);
rerrors = rerrors | !rmatrixchecksolution(ref xe, n, threshold, info, ref rep, ref xv);
//
// Test DenseSolverRLS():
// * test on original system A*x = b
// * test on overdetermined system with the same solution: (A' A')'*x = (b' b')'
// * test on underdetermined system with the same solution: (A 0 0 0 ) * z = b
//
info = 0;
unsetlsrep(ref repls);
unset1d(ref xv);
bv = new double[n];
for(i_=0; i_<=n-1;i_++)
{
bv[i_] = b[i_,0];
}
densesolver.rmatrixsolvels(ref a, n, n, ref bv, 0.0, ref info, ref repls, ref xv);
if( info<=0 )
{
rerrors = true;
}
else
{
rerrors = rerrors | (double)(repls.r2)<(double)(100*AP.Math.MachineEpsilon) | (double)(repls.r2)>(double)(1+1000*AP.Math.MachineEpsilon);
rerrors = rerrors | repls.n!=n | repls.k!=0;
for(i=0; i<=n-1; i++)
{
rerrors = rerrors | (double)(Math.Abs(xe[i,0]-xv[i]))>(double)(threshold);
}
}
info = 0;
unsetlsrep(ref repls);
unset1d(ref xv);
bv = new double[2*n];
for(i_=0; i_<=n-1;i_++)
{
bv[i_] = b[i_,0];
}
i1_ = (0) - (n);
for(i_=n; i_<=2*n-1;i_++)
{
bv[i_] = b[i_+i1_,0];
}
atmp = new double[2*n, n];
blas.copymatrix(ref a, 0, n-1, 0, n-1, ref atmp, 0, n-1, 0, n-1);
blas.copymatrix(ref a, 0, n-1, 0, n-1, ref atmp, n, 2*n-1, 0, n-1);
densesolver.rmatrixsolvels(ref atmp, 2*n, n, ref bv, 0.0, ref info, ref repls, ref xv);
if( info<=0 )
{
rerrors = true;
}
else
{
rerrors = rerrors | (double)(repls.r2)<(double)(100*AP.Math.MachineEpsilon) | (double)(repls.r2)>(double)(1+1000*AP.Math.MachineEpsilon);
rerrors = rerrors | repls.n!=n | repls.k!=0;
for(i=0; i<=n-1; i++)
{
rerrors = rerrors | (double)(Math.Abs(xe[i,0]-xv[i]))>(double)(threshold);
}
}
info = 0;
unsetlsrep(ref repls);
unset1d(ref xv);
bv = new double[n];
for(i_=0; i_<=n-1;i_++)
{
bv[i_] = b[i_,0];
}
atmp = new double[n, 2*n];
blas.copymatrix(ref a, 0, n-1, 0, n-1, ref atmp, 0, n-1, 0, n-1);
for(i=0; i<=n-1; i++)
{
for(j=n; j<=2*n-1; j++)
{
atmp[i,j] = 0;
}
}
densesolver.rmatrixsolvels(ref atmp, n, 2*n, ref bv, 0.0, ref info, ref repls, ref xv);
if( info<=0 )
{
rerrors = true;
}
else
{
rerrors = rerrors | (double)(repls.r2)!=(double)(0);
rerrors = rerrors | repls.n!=2*n | repls.k!=n;
for(i=0; i<=n-1; i++)
{
rerrors = rerrors | (double)(Math.Abs(xe[i,0]-xv[i]))>(double)(threshold);
}
for(i=n; i<=2*n-1; i++)
{
rerrors = rerrors | (double)(Math.Abs(xv[i]))>(double)(threshold);
}
}
//
// ********************************************************
// EXACTLY SINGULAR MATRICES
// ability to detect singularity is tested
// ********************************************************
//
// 1. generate different types of singular matrices:
// * zero
// * with zero columns
// * with zero rows
// * with equal rows/columns
// 2. generate random solution vector xe
// 3. generate right part b=A*xe
// 4. test different methods
//
for(taskkind=0; taskkind<=4; taskkind++)
{
unset2d(ref a);
if( taskkind==0 )
{
//
// all zeros
//
a = new double[n, n];
for(i=0; i<=n-1; i++)
{
for(j=0; j<=n-1; j++)
{
a[i,j] = 0;
}
}
}
if( taskkind==1 )
{
//
// there is zero column
//
a = new double[n, n];
for(i=0; i<=n-1; i++)
{
for(j=0; j<=n-1; j++)
{
a[i,j] = 2*AP.Math.RandomReal()-1;
}
}
k = AP.Math.RandomInteger(n);
for(i_=0; i_<=n-1;i_++)
{
a[i_,k] = 0*a[i_,k];
}
}
if( taskkind==2 )
{
//
// there is zero row
//
a = new double[n, n];
for(i=0; i<=n-1; i++)
{
for(j=0; j<=n-1; j++)
{
a[i,j] = 2*AP.Math.RandomReal()-1;
}
}
k = AP.Math.RandomInteger(n);
for(i_=0; i_<=n-1;i_++)
{
a[k,i_] = 0*a[k,i_];
}
}
if( taskkind==3 )
{
//
// equal columns
//
if( n<2 )
{
continue;
}
a = new double[n, n];
for(i=0; i<=n-1; i++)
{
for(j=0; j<=n-1; j++)
{
a[i,j] = 2*AP.Math.RandomReal()-1;
}
}
k = 1+AP.Math.RandomInteger(n-1);
for(i_=0; i_<=n-1;i_++)
{
a[i_,0] = a[i_,k];
}
}
if( taskkind==4 )
{
//
// equal rows
//
if( n<2 )
{
continue;
}
a = new double[n, n];
for(i=0; i<=n-1; i++)
{
for(j=0; j<=n-1; j++)
{
a[i,j] = 2*AP.Math.RandomReal()-1;
}
}
k = 1+AP.Math.RandomInteger(n-1);
for(i_=0; i_<=n-1;i_++)
{
a[0,i_] = a[k,i_];
}
}
xe = new double[n, m];
for(i=0; i<=n-1; i++)
{
for(j=0; j<=m-1; j++)
{
xe[i,j] = 2*AP.Math.RandomReal()-1;
}
}
b = new double[n, m];
for(i=0; i<=n-1; i++)
{
for(j=0; j<=m-1; j++)
{
v = 0.0;
for(i_=0; i_<=n-1;i_++)
{
v += a[i,i_]*xe[i_,j];
}
b[i,j] = v;
}
}
rmatrixmakeacopy(ref a, n, n, ref lua);
trfac.rmatrixlu(ref lua, n, n, ref p);
//
// Test RMatrixSolveM()
//
info = 0;
unsetrep(ref rep);
unset2d(ref x);
densesolver.rmatrixsolvem(ref a, n, ref b, m, (double)(AP.Math.RandomReal())>(double)(0.5), ref info, ref rep, ref x);
rerrors = rerrors | !rmatrixchecksingularm(n, m, info, ref rep, ref x);
//
// Test RMatrixSolve()
//
info = 0;
unsetrep(ref rep);
unset2d(ref x);
bv = new double[n];
for(i_=0; i_<=n-1;i_++)
{
bv[i_] = b[i_,0];
}
densesolver.rmatrixsolve(ref a, n, ref bv, ref info, ref rep, ref xv);
rerrors = rerrors | !rmatrixchecksingular(n, info, ref rep, ref xv);
//
// Test RMatrixLUSolveM()
//
info = 0;
unsetrep(ref rep);
unset2d(ref x);
densesolver.rmatrixlusolvem(ref lua, ref p, n, ref b, m, ref info, ref rep, ref x);
rerrors = rerrors | !rmatrixchecksingularm(n, m, info, ref rep, ref x);
//
// Test RMatrixLUSolve()
//
info = 0;
unsetrep(ref rep);
unset2d(ref x);
bv = new double[n];
for(i_=0; i_<=n-1;i_++)
{
bv[i_] = b[i_,0];
}
densesolver.rmatrixlusolve(ref lua, ref p, n, ref bv, ref info, ref rep, ref xv);
rerrors = rerrors | !rmatrixchecksingular(n, info, ref rep, ref xv);
//
// Test RMatrixMixedSolveM()
//
info = 0;
unsetrep(ref rep);
unset2d(ref x);
densesolver.rmatrixmixedsolvem(ref a, ref lua, ref p, n, ref b, m, ref info, ref rep, ref x);
rerrors = rerrors | !rmatrixchecksingularm(n, m, info, ref rep, ref x);
//
// Test RMatrixMixedSolve()
//
info = 0;
unsetrep(ref rep);
unset2d(ref x);
bv = new double[n];
for(i_=0; i_<=n-1;i_++)
{
bv[i_] = b[i_,0];
}
densesolver.rmatrixmixedsolve(ref a, ref lua, ref p, n, ref bv, ref info, ref rep, ref xv);
rerrors = rerrors | !rmatrixchecksingular(n, info, ref rep, ref xv);
}
}
}
}
//
// test iterative improvement
//
for(pass=1; pass<=passcount; pass++)
{
//
// Test iterative improvement matrices
//
// A matrix/right part are constructed such that both matrix
// and solution components are within (-1,+1). Such matrix/right part
// have nice properties - system can be solved using iterative
// improvement with |A*x-b| about several ulps of max(1,|b|).
//
n = 100;
a = new double[n, n];
b = new double[n, 1];
bv = new double[n];
tx = new double[n];
xv = new double[n];
y = new double[n];
for(i=0; i<=n-1; i++)
{
xv[i] = 2*AP.Math.RandomReal()-1;
}
for(i=0; i<=n-1; i++)
{
for(j=0; j<=n-1; j++)
{
a[i,j] = 2*AP.Math.RandomReal()-1;
}
for(i_=0; i_<=n-1;i_++)
{
y[i_] = a[i,i_];
}
xblas.xdot(ref y, ref xv, n, ref tx, ref v, ref verr);
bv[i] = v;
}
for(i_=0; i_<=n-1;i_++)
{
b[i_,0] = bv[i_];
}
//
// Test RMatrixSolveM()
//
unset2d(ref x);
densesolver.rmatrixsolvem(ref a, n, ref b, 1, true, ref info, ref rep, ref x);
if( info<=0 )
{
rfserrors = true;
}
else
{
xv = new double[n];
for(i_=0; i_<=n-1;i_++)
{
xv[i_] = x[i_,0];
}
for(i=0; i<=n-1; i++)
{
for(i_=0; i_<=n-1;i_++)
{
y[i_] = a[i,i_];
}
xblas.xdot(ref y, ref xv, n, ref tx, ref v, ref verr);
rfserrors = rfserrors | (double)(Math.Abs(v-b[i,0]))>(double)(8*AP.Math.MachineEpsilon*Math.Max(1, Math.Abs(b[i,0])));
}
}
//
// Test RMatrixSolve()
//
unset1d(ref xv);
densesolver.rmatrixsolve(ref a, n, ref bv, ref info, ref rep, ref xv);
if( info<=0 )
{
rfserrors = true;
}
else
{
for(i=0; i<=n-1; i++)
{
for(i_=0; i_<=n-1;i_++)
{
y[i_] = a[i,i_];
}
xblas.xdot(ref y, ref xv, n, ref tx, ref v, ref verr);
rfserrors = rfserrors | (double)(Math.Abs(v-bv[i]))>(double)(8*AP.Math.MachineEpsilon*Math.Max(1, Math.Abs(bv[i])));
}
}
//
// Test LS-solver on the same matrix
//
densesolver.rmatrixsolvels(ref a, n, n, ref bv, 0.0, ref info, ref repls, ref xv);
if( info<=0 )
{
rfserrors = true;
}
else
{
for(i=0; i<=n-1; i++)
{
for(i_=0; i_<=n-1;i_++)
{
y[i_] = a[i,i_];
}
xblas.xdot(ref y, ref xv, n, ref tx, ref v, ref verr);
rfserrors = rfserrors | (double)(Math.Abs(v-bv[i]))>(double)(8*AP.Math.MachineEpsilon*Math.Max(1, Math.Abs(bv[i])));
}
}
}
}
/*************************************************************************
HPD test
*************************************************************************/
private static void testhpdsolver(int maxn,
int maxm,
int passcount,
double threshold,
ref bool hpderrors,
ref bool rfserrors)
{
AP.Complex[,] a = new AP.Complex[0,0];
AP.Complex[,] cha = new AP.Complex[0,0];
AP.Complex[,] atmp = new AP.Complex[0,0];
int[] p = new int[0];
AP.Complex[,] xe = new AP.Complex[0,0];
AP.Complex[,] b = new AP.Complex[0,0];
AP.Complex[] bv = new AP.Complex[0];
int i = 0;
int j = 0;
int k = 0;
int n = 0;
int m = 0;
int pass = 0;
int taskkind = 0;
double mx = 0;
AP.Complex v = 0;
bool isupper = new bool();
int info = 0;
densesolver.densesolverreport rep = new densesolver.densesolverreport();
densesolver.densesolverlsreport repls = new densesolver.densesolverlsreport();
AP.Complex[,] x = new AP.Complex[0,0];
AP.Complex[] xv = new AP.Complex[0];
AP.Complex[] y = new AP.Complex[0];
AP.Complex[] tx = new AP.Complex[0];
int i_ = 0;
//
// General square matrices:
// * test general solvers
// * test least squares solver
//
for(pass=1; pass<=passcount; pass++)
{
for(n=1; n<=maxn; n++)
{
for(m=1; m<=maxm; m++)
{
//
// ********************************************************
// WELL CONDITIONED TASKS
// ability to find correct solution is tested
// ********************************************************
//
// 1. generate random well conditioned matrix A.
// 2. generate random solution vector xe
// 3. generate right part b=A*xe
// 4. test different methods on original A
//
isupper = (double)(AP.Math.RandomReal())>(double)(0.5);
matgen.hpdmatrixrndcond(n, 1000, ref a);
cmatrixmakeacopy(ref a, n, n, ref cha);
if( !trfac.hpdmatrixcholesky(ref cha, n, isupper) )
{
hpderrors = true;
return;
}
xe = new AP.Complex[n, m];
for(i=0; i<=n-1; i++)
{
for(j=0; j<=m-1; j++)
{
xe[i,j].x = 2*AP.Math.RandomReal()-1;
xe[i,j].y = 2*AP.Math.RandomReal()-1;
}
}
b = new AP.Complex[n, m];
for(i=0; i<=n-1; i++)
{
for(j=0; j<=m-1; j++)
{
v = 0.0;
for(i_=0; i_<=n-1;i_++)
{
v += a[i,i_]*xe[i_,j];
}
b[i,j] = v;
}
}
cmatrixdrophalf(ref a, n, isupper);
cmatrixdrophalf(ref cha, n, isupper);
//
// Test solvers
//
info = 0;
unsetrep(ref rep);
cunset2d(ref x);
densesolver.hpdmatrixsolvem(ref a, n, isupper, ref b, m, ref info, ref rep, ref x);
hpderrors = hpderrors | !cmatrixchecksolutionm(ref xe, n, m, threshold, info, ref rep, ref x);
info = 0;
unsetrep(ref rep);
cunset1d(ref xv);
bv = new AP.Complex[n];
for(i_=0; i_<=n-1;i_++)
{
bv[i_] = b[i_,0];
}
densesolver.hpdmatrixsolve(ref a, n, isupper, ref bv, ref info, ref rep, ref xv);
hpderrors = hpderrors | !cmatrixchecksolution(ref xe, n, threshold, info, ref rep, ref xv);
info = 0;
unsetrep(ref rep);
cunset2d(ref x);
densesolver.hpdmatrixcholeskysolvem(ref cha, n, isupper, ref b, m, ref info, ref rep, ref x);
hpderrors = hpderrors | !cmatrixchecksolutionm(ref xe, n, m, threshold, info, ref rep, ref x);
info = 0;
unsetrep(ref rep);
cunset1d(ref xv);
bv = new AP.Complex[n];
for(i_=0; i_<=n-1;i_++)
{
bv[i_] = b[i_,0];
}
densesolver.hpdmatrixcholeskysolve(ref cha, n, isupper, ref bv, ref info, ref rep, ref xv);
hpderrors = hpderrors | !cmatrixchecksolution(ref xe, n, threshold, info, ref rep, ref xv);
//
// ********************************************************
// EXACTLY SINGULAR MATRICES
// ability to detect singularity is tested
// ********************************************************
//
// 1. generate different types of singular matrices:
// * zero
// * with zero columns
// * with zero rows
// * with equal rows/columns
// 2. generate random solution vector xe
// 3. generate right part b=A*xe
// 4. test different methods
//
for(taskkind=0; taskkind<=3; taskkind++)
{
cunset2d(ref a);
if( taskkind==0 )
{
//
// all zeros
//
a = new AP.Complex[n, n];
for(i=0; i<=n-1; i++)
{
for(j=0; j<=n-1; j++)
{
a[i,j] = 0;
}
}
}
if( taskkind==1 )
{
//
// there is zero column
//
a = new AP.Complex[n, n];
for(i=0; i<=n-1; i++)
{
for(j=i; j<=n-1; j++)
{
a[i,j].x = 2*AP.Math.RandomReal()-1;
a[i,j].y = 2*AP.Math.RandomReal()-1;
if( i==j )
{
a[i,j].y = 0;
}
a[j,i] = a[i,j];
}
}
k = AP.Math.RandomInteger(n);
for(i_=0; i_<=n-1;i_++)
{
a[i_,k] = 0*a[i_,k];
}
for(i_=0; i_<=n-1;i_++)
{
a[k,i_] = 0*a[k,i_];
}
}
if( taskkind==2 )
{
//
// there is zero row
//
a = new AP.Complex[n, n];
for(i=0; i<=n-1; i++)
{
for(j=i; j<=n-1; j++)
{
a[i,j].x = 2*AP.Math.RandomReal()-1;
a[i,j].y = 2*AP.Math.RandomReal()-1;
if( i==j )
{
a[i,j].y = 0;
}
a[j,i] = a[i,j];
}
}
k = AP.Math.RandomInteger(n);
for(i_=0; i_<=n-1;i_++)
{
a[k,i_] = 0*a[k,i_];
}
for(i_=0; i_<=n-1;i_++)
{
a[i_,k] = 0*a[i_,k];
}
}
if( taskkind==3 )
{
//
// equal columns/rows
//
if( n<2 )
{
continue;
}
a = new AP.Complex[n, n];
for(i=0; i<=n-1; i++)
{
for(j=i; j<=n-1; j++)
{
a[i,j].x = 2*AP.Math.RandomReal()-1;
a[i,j].y = 2*AP.Math.RandomReal()-1;
if( i==j )
{
a[i,j].y = 0;
}
a[j,i] = a[i,j];
}
}
k = 1+AP.Math.RandomInteger(n-1);
for(i_=0; i_<=n-1;i_++)
{
a[i_,0] = a[i_,k];
}
for(i_=0; i_<=n-1;i_++)
{
a[0,i_] = a[k,i_];
}
}
xe = new AP.Complex[n, m];
for(i=0; i<=n-1; i++)
{
for(j=0; j<=m-1; j++)
{
xe[i,j] = 2*AP.Math.RandomReal()-1;
}
}
b = new AP.Complex[n, m];
for(i=0; i<=n-1; i++)
{
for(j=0; j<=m-1; j++)
{
v = 0.0;
for(i_=0; i_<=n-1;i_++)
{
v += a[i,i_]*xe[i_,j];
}
b[i,j] = v;
}
}
cmatrixmakeacopy(ref a, n, n, ref cha);
cmatrixdrophalf(ref a, n, isupper);
cmatrixdrophalf(ref cha, n, isupper);
//
// Test SPDMatrixSolveM()
//
info = 0;
unsetrep(ref rep);
cunset2d(ref x);
densesolver.hpdmatrixsolvem(ref a, n, isupper, ref b, m, ref info, ref rep, ref x);
hpderrors = hpderrors | !cmatrixchecksingularm(n, m, info, ref rep, ref x);
//
// Test SPDMatrixSolve()
//
info = 0;
unsetrep(ref rep);
cunset2d(ref x);
bv = new AP.Complex[n];
for(i_=0; i_<=n-1;i_++)
{
bv[i_] = b[i_,0];
}
densesolver.hpdmatrixsolve(ref a, n, isupper, ref bv, ref info, ref rep, ref xv);
hpderrors = hpderrors | !cmatrixchecksingular(n, info, ref rep, ref xv);
//
// 'equal columns/rows' are degenerate, but
// Cholesky matrix with equal columns/rows IS NOT degenerate,
// so it is not used for testing purposes.
//
if( taskkind!=3 )
{
//
// Test SPDMatrixLUSolveM()
//
info = 0;
unsetrep(ref rep);
cunset2d(ref x);
densesolver.hpdmatrixcholeskysolvem(ref cha, n, isupper, ref b, m, ref info, ref rep, ref x);
hpderrors = hpderrors | !cmatrixchecksingularm(n, m, info, ref rep, ref x);
//
// Test SPDMatrixLUSolve()
//
info = 0;
unsetrep(ref rep);
cunset2d(ref x);
bv = new AP.Complex[n];
for(i_=0; i_<=n-1;i_++)
{
bv[i_] = b[i_,0];
}
densesolver.hpdmatrixcholeskysolve(ref cha, n, isupper, ref bv, ref info, ref rep, ref xv);
hpderrors = hpderrors | !cmatrixchecksingular(n, info, ref rep, ref xv);
}
}
}
}
}
}
/*************************************************************************
Real test
*************************************************************************/
private static void testcsolver(int maxn,
int maxm,
int passcount,
double threshold,
ref bool cerrors,
ref bool rfserrors)
{
AP.Complex[,] a = new AP.Complex[0,0];
AP.Complex[,] lua = new AP.Complex[0,0];
AP.Complex[,] atmp = new AP.Complex[0,0];
int[] p = new int[0];
AP.Complex[,] xe = new AP.Complex[0,0];
AP.Complex[,] b = new AP.Complex[0,0];
AP.Complex[] bv = new AP.Complex[0];
int i = 0;
int j = 0;
int k = 0;
int n = 0;
int m = 0;
int pass = 0;
int taskkind = 0;
double mx = 0;
double verr = 0;
AP.Complex v = 0;
int info = 0;
densesolver.densesolverreport rep = new densesolver.densesolverreport();
densesolver.densesolverlsreport repls = new densesolver.densesolverlsreport();
AP.Complex[,] x = new AP.Complex[0,0];
AP.Complex[] xv = new AP.Complex[0];
AP.Complex[] y = new AP.Complex[0];
double[] tx = new double[0];
int i_ = 0;
//
// General square matrices:
// * test general solvers
// * test least squares solver
//
for(pass=1; pass<=passcount; pass++)
{
for(n=1; n<=maxn; n++)
{
for(m=1; m<=maxm; m++)
{
//
// ********************************************************
// WELL CONDITIONED TASKS
// ability to find correct solution is tested
// ********************************************************
//
// 1. generate random well conditioned matrix A.
// 2. generate random solution vector xe
// 3. generate right part b=A*xe
// 4. test different methods on original A
//
matgen.cmatrixrndcond(n, 1000, ref a);
cmatrixmakeacopy(ref a, n, n, ref lua);
trfac.cmatrixlu(ref lua, n, n, ref p);
xe = new AP.Complex[n, m];
for(i=0; i<=n-1; i++)
{
for(j=0; j<=m-1; j++)
{
xe[i,j].x = 2*AP.Math.RandomReal()-1;
xe[i,j].y = 2*AP.Math.RandomReal()-1;
}
}
b = new AP.Complex[n, m];
for(i=0; i<=n-1; i++)
{
for(j=0; j<=m-1; j++)
{
v = 0.0;
for(i_=0; i_<=n-1;i_++)
{
v += a[i,i_]*xe[i_,j];
}
b[i,j] = v;
}
}
//
// Test solvers
//
info = 0;
unsetrep(ref rep);
cunset2d(ref x);
densesolver.cmatrixsolvem(ref a, n, ref b, m, (double)(AP.Math.RandomReal())>(double)(0.5), ref info, ref rep, ref x);
cerrors = cerrors | !cmatrixchecksolutionm(ref xe, n, m, threshold, info, ref rep, ref x);
info = 0;
unsetrep(ref rep);
cunset1d(ref xv);
bv = new AP.Complex[n];
for(i_=0; i_<=n-1;i_++)
{
bv[i_] = b[i_,0];
}
densesolver.cmatrixsolve(ref a, n, ref bv, ref info, ref rep, ref xv);
cerrors = cerrors | !cmatrixchecksolution(ref xe, n, threshold, info, ref rep, ref xv);
info = 0;
unsetrep(ref rep);
cunset2d(ref x);
densesolver.cmatrixlusolvem(ref lua, ref p, n, ref b, m, ref info, ref rep, ref x);
cerrors = cerrors | !cmatrixchecksolutionm(ref xe, n, m, threshold, info, ref rep, ref x);
info = 0;
unsetrep(ref rep);
cunset1d(ref xv);
bv = new AP.Complex[n];
for(i_=0; i_<=n-1;i_++)
{
bv[i_] = b[i_,0];
}
densesolver.cmatrixlusolve(ref lua, ref p, n, ref bv, ref info, ref rep, ref xv);
cerrors = cerrors | !cmatrixchecksolution(ref xe, n, threshold, info, ref rep, ref xv);
info = 0;
unsetrep(ref rep);
cunset2d(ref x);
densesolver.cmatrixmixedsolvem(ref a, ref lua, ref p, n, ref b, m, ref info, ref rep, ref x);
cerrors = cerrors | !cmatrixchecksolutionm(ref xe, n, m, threshold, info, ref rep, ref x);
info = 0;
unsetrep(ref rep);
cunset1d(ref xv);
bv = new AP.Complex[n];
for(i_=0; i_<=n-1;i_++)
{
bv[i_] = b[i_,0];
}
densesolver.cmatrixmixedsolve(ref a, ref lua, ref p, n, ref bv, ref info, ref rep, ref xv);
cerrors = cerrors | !cmatrixchecksolution(ref xe, n, threshold, info, ref rep, ref xv);
//
// ********************************************************
// EXACTLY SINGULAR MATRICES
// ability to detect singularity is tested
// ********************************************************
//
// 1. generate different types of singular matrices:
// * zero
// * with zero columns
// * with zero rows
// * with equal rows/columns
// 2. generate random solution vector xe
// 3. generate right part b=A*xe
// 4. test different methods
//
for(taskkind=0; taskkind<=4; taskkind++)
{
cunset2d(ref a);
if( taskkind==0 )
{
//
// all zeros
//
a = new AP.Complex[n, n];
for(i=0; i<=n-1; i++)
{
for(j=0; j<=n-1; j++)
{
a[i,j] = 0;
}
}
}
if( taskkind==1 )
{
//
// there is zero column
//
a = new AP.Complex[n, n];
for(i=0; i<=n-1; i++)
{
for(j=0; j<=n-1; j++)
{
a[i,j].x = 2*AP.Math.RandomReal()-1;
a[i,j].y = 2*AP.Math.RandomReal()-1;
}
}
k = AP.Math.RandomInteger(n);
for(i_=0; i_<=n-1;i_++)
{
a[i_,k] = 0*a[i_,k];
}
}
if( taskkind==2 )
{
//
// there is zero row
//
a = new AP.Complex[n, n];
for(i=0; i<=n-1; i++)
{
for(j=0; j<=n-1; j++)
{
a[i,j].x = 2*AP.Math.RandomReal()-1;
a[i,j].y = 2*AP.Math.RandomReal()-1;
}
}
k = AP.Math.RandomInteger(n);
for(i_=0; i_<=n-1;i_++)
{
a[k,i_] = 0*a[k,i_];
}
}
if( taskkind==3 )
{
//
// equal columns
//
if( n<2 )
{
continue;
}
a = new AP.Complex[n, n];
for(i=0; i<=n-1; i++)
{
for(j=0; j<=n-1; j++)
{
a[i,j].x = 2*AP.Math.RandomReal()-1;
a[i,j].y = 2*AP.Math.RandomReal()-1;
}
}
k = 1+AP.Math.RandomInteger(n-1);
for(i_=0; i_<=n-1;i_++)
{
a[i_,0] = a[i_,k];
}
}
if( taskkind==4 )
{
//
// equal rows
//
if( n<2 )
{
continue;
}
a = new AP.Complex[n, n];
for(i=0; i<=n-1; i++)
{
for(j=0; j<=n-1; j++)
{
a[i,j].x = 2*AP.Math.RandomReal()-1;
a[i,j].y = 2*AP.Math.RandomReal()-1;
}
}
k = 1+AP.Math.RandomInteger(n-1);
for(i_=0; i_<=n-1;i_++)
{
a[0,i_] = a[k,i_];
}
}
xe = new AP.Complex[n, m];
for(i=0; i<=n-1; i++)
{
for(j=0; j<=m-1; j++)
{
xe[i,j] = 2*AP.Math.RandomReal()-1;
}
}
b = new AP.Complex[n, m];
for(i=0; i<=n-1; i++)
{
for(j=0; j<=m-1; j++)
{
v = 0.0;
for(i_=0; i_<=n-1;i_++)
{
v += a[i,i_]*xe[i_,j];
}
b[i,j] = v;
}
}
cmatrixmakeacopy(ref a, n, n, ref lua);
trfac.cmatrixlu(ref lua, n, n, ref p);
//
// Test CMatrixSolveM()
//
info = 0;
unsetrep(ref rep);
cunset2d(ref x);
densesolver.cmatrixsolvem(ref a, n, ref b, m, (double)(AP.Math.RandomReal())>(double)(0.5), ref info, ref rep, ref x);
cerrors = cerrors | !cmatrixchecksingularm(n, m, info, ref rep, ref x);
//
// Test CMatrixSolve()
//
info = 0;
unsetrep(ref rep);
cunset2d(ref x);
bv = new AP.Complex[n];
for(i_=0; i_<=n-1;i_++)
{
bv[i_] = b[i_,0];
}
densesolver.cmatrixsolve(ref a, n, ref bv, ref info, ref rep, ref xv);
cerrors = cerrors | !cmatrixchecksingular(n, info, ref rep, ref xv);
//
// Test CMatrixLUSolveM()
//
info = 0;
unsetrep(ref rep);
cunset2d(ref x);
densesolver.cmatrixlusolvem(ref lua, ref p, n, ref b, m, ref info, ref rep, ref x);
cerrors = cerrors | !cmatrixchecksingularm(n, m, info, ref rep, ref x);
//
// Test CMatrixLUSolve()
//
info = 0;
unsetrep(ref rep);
cunset2d(ref x);
bv = new AP.Complex[n];
for(i_=0; i_<=n-1;i_++)
{
bv[i_] = b[i_,0];
}
densesolver.cmatrixlusolve(ref lua, ref p, n, ref bv, ref info, ref rep, ref xv);
cerrors = cerrors | !cmatrixchecksingular(n, info, ref rep, ref xv);
//
// Test CMatrixMixedSolveM()
//
info = 0;
unsetrep(ref rep);
cunset2d(ref x);
densesolver.cmatrixmixedsolvem(ref a, ref lua, ref p, n, ref b, m, ref info, ref rep, ref x);
cerrors = cerrors | !cmatrixchecksingularm(n, m, info, ref rep, ref x);
//
// Test CMatrixMixedSolve()
//
info = 0;
unsetrep(ref rep);
cunset2d(ref x);
bv = new AP.Complex[n];
for(i_=0; i_<=n-1;i_++)
{
bv[i_] = b[i_,0];
}
densesolver.cmatrixmixedsolve(ref a, ref lua, ref p, n, ref bv, ref info, ref rep, ref xv);
cerrors = cerrors | !cmatrixchecksingular(n, info, ref rep, ref xv);
}
}
}
}
//
// test iterative improvement
//
for(pass=1; pass<=passcount; pass++)
{
//
// Test iterative improvement matrices
//
// A matrix/right part are constructed such that both matrix
// and solution components magnitudes are within (-1,+1).
// Such matrix/right part have nice properties - system can
// be solved using iterative improvement with |A*x-b| about
// several ulps of max(1,|b|).
//
n = 100;
a = new AP.Complex[n, n];
b = new AP.Complex[n, 1];
bv = new AP.Complex[n];
tx = new double[2*n];
xv = new AP.Complex[n];
y = new AP.Complex[n];
for(i=0; i<=n-1; i++)
{
xv[i].x = 2*AP.Math.RandomReal()-1;
xv[i].y = 2*AP.Math.RandomReal()-1;
}
for(i=0; i<=n-1; i++)
{
for(j=0; j<=n-1; j++)
{
a[i,j].x = 2*AP.Math.RandomReal()-1;
a[i,j].y = 2*AP.Math.RandomReal()-1;
}
for(i_=0; i_<=n-1;i_++)
{
y[i_] = a[i,i_];
}
xblas.xcdot(ref y, ref xv, n, ref tx, ref v, ref verr);
bv[i] = v;
}
for(i_=0; i_<=n-1;i_++)
{
b[i_,0] = bv[i_];
}
//
// Test CMatrixSolveM()
//
cunset2d(ref x);
densesolver.cmatrixsolvem(ref a, n, ref b, 1, true, ref info, ref rep, ref x);
if( info<=0 )
{
rfserrors = true;
}
else
{
xv = new AP.Complex[n];
for(i_=0; i_<=n-1;i_++)
{
xv[i_] = x[i_,0];
}
for(i=0; i<=n-1; i++)
{
for(i_=0; i_<=n-1;i_++)
{
y[i_] = a[i,i_];
}
xblas.xcdot(ref y, ref xv, n, ref tx, ref v, ref verr);
rfserrors = rfserrors | (double)(AP.Math.AbsComplex(v-b[i,0]))>(double)(8*AP.Math.MachineEpsilon*Math.Max(1, AP.Math.AbsComplex(b[i,0])));
}
}
//
// Test CMatrixSolve()
//
cunset1d(ref xv);
densesolver.cmatrixsolve(ref a, n, ref bv, ref info, ref rep, ref xv);
if( info<=0 )
{
rfserrors = true;
}
else
{
for(i=0; i<=n-1; i++)
{
for(i_=0; i_<=n-1;i_++)
{
y[i_] = a[i,i_];
}
xblas.xcdot(ref y, ref xv, n, ref tx, ref v, ref verr);
rfserrors = rfserrors | (double)(AP.Math.AbsComplex(v-bv[i]))>(double)(8*AP.Math.MachineEpsilon*Math.Max(1, AP.Math.AbsComplex(bv[i])));
}
}
//
// TODO: Test LS-solver on the same matrix
//
}
}
/*************************************************************************
Test
*************************************************************************/
public static bool testdensesolver(bool silent)
{
bool result = new bool();
double[,] a = new double[0,0];
double[,] lua = new double[0,0];
double[,] atmp = new double[0,0];
int[] p = new int[0];
double[,] xe = new double[0,0];
double[,] b = new double[0,0];
double[] bv = new double[0];
int i = 0;
int j = 0;
int k = 0;
int n = 0;
int m = 0;
int pass = 0;
int taskkind = 0;
double mx = 0;
double v = 0;
double verr = 0;
int info = 0;
densesolver.densesolverreport rep = new densesolver.densesolverreport();
densesolver.densesolverlsreport repls = new densesolver.densesolverlsreport();
double[,] x = new double[0,0];
double[] xv = new double[0];
double[] y = new double[0];
double[] tx = new double[0];
int maxn = 0;
int maxm = 0;
int passcount = 0;
double threshold = 0;
bool rerrors = new bool();
bool cerrors = new bool();
bool spderrors = new bool();
bool hpderrors = new bool();
bool rfserrors = new bool();
bool waserrors = new bool();
maxn = 10;
maxm = 5;
passcount = 5;
threshold = 10000*AP.Math.MachineEpsilon;
rfserrors = false;
rerrors = false;
cerrors = false;
spderrors = false;
hpderrors = false;
testrsolver(maxn, maxm, passcount, threshold, ref rerrors, ref rfserrors);
testspdsolver(maxn, maxm, passcount, threshold, ref spderrors, ref rfserrors);
testcsolver(maxn, maxm, passcount, threshold, ref cerrors, ref rfserrors);
testhpdsolver(maxn, maxm, passcount, threshold, ref hpderrors, ref rfserrors);
waserrors = rerrors | cerrors | spderrors | hpderrors | rfserrors;
if( !silent )
{
System.Console.Write("TESTING DENSE SOLVER");
System.Console.WriteLine();
System.Console.Write("* REAL: ");
if( rerrors )
{
System.Console.Write("FAILED");
System.Console.WriteLine();
}
else
{
System.Console.Write("OK");
System.Console.WriteLine();
}
System.Console.Write("* COMPLEX: ");
if( cerrors )
{
System.Console.Write("FAILED");
System.Console.WriteLine();
}
else
{
System.Console.Write("OK");
System.Console.WriteLine();
}
System.Console.Write("* SPD: ");
if( spderrors )
{
System.Console.Write("FAILED");
System.Console.WriteLine();
}
else
{
System.Console.Write("OK");
System.Console.WriteLine();
}
System.Console.Write("* HPD: ");
if( hpderrors )
{
System.Console.Write("FAILED");
System.Console.WriteLine();
}
else
{
System.Console.Write("OK");
System.Console.WriteLine();
}
System.Console.Write("* ITERATIVE IMPROVEMENT: ");
if( rfserrors )
{
System.Console.Write("FAILED");
System.Console.WriteLine();
}
else
{
System.Console.Write("OK");
System.Console.WriteLine();
}
if( waserrors )
{
System.Console.Write("TEST FAILED");
System.Console.WriteLine();
}
else
{
System.Console.Write("TEST PASSED");
System.Console.WriteLine();
}
}
result = !waserrors;
return result;
}
/*************************************************************************
This subroutine trains logit model.
INPUT PARAMETERS:
XY - training set, array[0..NPoints-1,0..NVars]
First NVars columns store values of independent
variables, next column stores number of class (from 0
to NClasses-1) which dataset element belongs to. Fractional
values are rounded to nearest integer.
NPoints - training set size, NPoints>=1
NVars - number of independent variables, NVars>=1
NClasses - number of classes, NClasses>=2
OUTPUT PARAMETERS:
Info - return code:
* -2, if there is a point with class number
outside of [0..NClasses-1].
* -1, if incorrect parameters was passed
(NPoints<NVars+2, NVars<1, NClasses<2).
* 1, if task has been solved
LM - model built
Rep - training report
-- ALGLIB --
Copyright 10.09.2008 by Bochkanov Sergey
*************************************************************************/
public static void mnltrainh(ref double[,] xy,
int npoints,
int nvars,
int nclasses,
ref int info,
ref logitmodel lm,
ref mnlreport rep)
{
int i = 0;
int j = 0;
int k = 0;
int ssize = 0;
bool allsame = new bool();
int offs = 0;
double threshold = 0;
double wminstep = 0;
double decay = 0;
int wdim = 0;
int expoffs = 0;
double v = 0;
double s = 0;
mlpbase.multilayerperceptron network = new mlpbase.multilayerperceptron();
int nin = 0;
int nout = 0;
int wcount = 0;
double e = 0;
double[] g = new double[0];
double[,] h = new double[0,0];
bool spd = new bool();
double[] x = new double[0];
double[] y = new double[0];
double[] wbase = new double[0];
double wstep = 0;
double[] wdir = new double[0];
double[] work = new double[0];
int mcstage = 0;
logitmcstate mcstate = new logitmcstate();
int mcinfo = 0;
int mcnfev = 0;
int solverinfo = 0;
densesolver.densesolverreport solverrep = new densesolver.densesolverreport();
int i_ = 0;
int i1_ = 0;
threshold = 1000*AP.Math.MachineEpsilon;
wminstep = 0.001;
decay = 0.001;
//
// Test for inputs
//
if( npoints<nvars+2 | nvars<1 | nclasses<2 )
{
info = -1;
return;
}
for(i=0; i<=npoints-1; i++)
{
if( (int)Math.Round(xy[i,nvars])<0 | (int)Math.Round(xy[i,nvars])>=nclasses )
{
info = -2;
return;
}
}
info = 1;
//
// Initialize data
//
rep.ngrad = 0;
rep.nhess = 0;
//
// Allocate array
//
wdim = (nvars+1)*(nclasses-1);
offs = 5;
expoffs = offs+wdim;
ssize = 5+(nvars+1)*(nclasses-1)+nclasses;
lm.w = new double[ssize-1+1];
lm.w[0] = ssize;
lm.w[1] = logitvnum;
lm.w[2] = nvars;
lm.w[3] = nclasses;
lm.w[4] = offs;
//
// Degenerate case: all outputs are equal
//
allsame = true;
for(i=1; i<=npoints-1; i++)
{
if( (int)Math.Round(xy[i,nvars])!=(int)Math.Round(xy[i-1,nvars]) )
{
allsame = false;
}
}
if( allsame )
{
for(i=0; i<=(nvars+1)*(nclasses-1)-1; i++)
{
lm.w[offs+i] = 0;
}
v = -(2*Math.Log(AP.Math.MinRealNumber));
k = (int)Math.Round(xy[0,nvars]);
if( k==nclasses-1 )
{
for(i=0; i<=nclasses-2; i++)
{
lm.w[offs+i*(nvars+1)+nvars] = -v;
}
}
else
{
for(i=0; i<=nclasses-2; i++)
{
if( i==k )
{
lm.w[offs+i*(nvars+1)+nvars] = +v;
}
else
{
lm.w[offs+i*(nvars+1)+nvars] = 0;
}
}
}
return;
}
//
// General case.
// Prepare task and network. Allocate space.
//
mlpbase.mlpcreatec0(nvars, nclasses, ref network);
mlpbase.mlpinitpreprocessor(ref network, ref xy, npoints);
mlpbase.mlpproperties(ref network, ref nin, ref nout, ref wcount);
for(i=0; i<=wcount-1; i++)
{
network.weights[i] = (2*AP.Math.RandomReal()-1)/nvars;
}
g = new double[wcount-1+1];
h = new double[wcount-1+1, wcount-1+1];
wbase = new double[wcount-1+1];
wdir = new double[wcount-1+1];
work = new double[wcount-1+1];
//
// First stage: optimize in gradient direction.
//
for(k=0; k<=wcount/3+10; k++)
{
//
// Calculate gradient in starting point
//
mlpbase.mlpgradnbatch(ref network, ref xy, npoints, ref e, ref g);
v = 0.0;
for(i_=0; i_<=wcount-1;i_++)
{
v += network.weights[i_]*network.weights[i_];
}
e = e+0.5*decay*v;
for(i_=0; i_<=wcount-1;i_++)
{
g[i_] = g[i_] + decay*network.weights[i_];
}
rep.ngrad = rep.ngrad+1;
//
// Setup optimization scheme
//
for(i_=0; i_<=wcount-1;i_++)
{
wdir[i_] = -g[i_];
}
v = 0.0;
for(i_=0; i_<=wcount-1;i_++)
{
v += wdir[i_]*wdir[i_];
}
wstep = Math.Sqrt(v);
v = 1/Math.Sqrt(v);
for(i_=0; i_<=wcount-1;i_++)
{
wdir[i_] = v*wdir[i_];
}
mcstage = 0;
mnlmcsrch(wcount, ref network.weights, ref e, ref g, ref wdir, ref wstep, ref mcinfo, ref mcnfev, ref work, ref mcstate, ref mcstage);
while( mcstage!=0 )
{
mlpbase.mlpgradnbatch(ref network, ref xy, npoints, ref e, ref g);
v = 0.0;
for(i_=0; i_<=wcount-1;i_++)
{
v += network.weights[i_]*network.weights[i_];
}
e = e+0.5*decay*v;
for(i_=0; i_<=wcount-1;i_++)
{
g[i_] = g[i_] + decay*network.weights[i_];
}
rep.ngrad = rep.ngrad+1;
mnlmcsrch(wcount, ref network.weights, ref e, ref g, ref wdir, ref wstep, ref mcinfo, ref mcnfev, ref work, ref mcstate, ref mcstage);
}
}
//
// Second stage: use Hessian when we are close to the minimum
//
while( true )
{
//
// Calculate and update E/G/H
//
mlpbase.mlphessiannbatch(ref network, ref xy, npoints, ref e, ref g, ref h);
v = 0.0;
for(i_=0; i_<=wcount-1;i_++)
{
v += network.weights[i_]*network.weights[i_];
}
e = e+0.5*decay*v;
for(i_=0; i_<=wcount-1;i_++)
{
g[i_] = g[i_] + decay*network.weights[i_];
}
for(k=0; k<=wcount-1; k++)
{
h[k,k] = h[k,k]+decay;
}
rep.nhess = rep.nhess+1;
//
// Select step direction
// NOTE: it is important to use lower-triangle Cholesky
// factorization since it is much faster than higher-triangle version.
//
spd = trfac.spdmatrixcholesky(ref h, wcount, false);
densesolver.spdmatrixcholeskysolve(ref h, wcount, false, ref g, ref solverinfo, ref solverrep, ref wdir);
spd = solverinfo>0;
if( spd )
{
//
// H is positive definite.
// Step in Newton direction.
//
for(i_=0; i_<=wcount-1;i_++)
{
wdir[i_] = -1*wdir[i_];
}
spd = true;
}
else
{
//
// H is indefinite.
// Step in gradient direction.
//
for(i_=0; i_<=wcount-1;i_++)
{
wdir[i_] = -g[i_];
}
spd = false;
}
//
// Optimize in WDir direction
//
v = 0.0;
for(i_=0; i_<=wcount-1;i_++)
{
v += wdir[i_]*wdir[i_];
}
wstep = Math.Sqrt(v);
v = 1/Math.Sqrt(v);
for(i_=0; i_<=wcount-1;i_++)
{
wdir[i_] = v*wdir[i_];
}
mcstage = 0;
mnlmcsrch(wcount, ref network.weights, ref e, ref g, ref wdir, ref wstep, ref mcinfo, ref mcnfev, ref work, ref mcstate, ref mcstage);
while( mcstage!=0 )
{
mlpbase.mlpgradnbatch(ref network, ref xy, npoints, ref e, ref g);
v = 0.0;
for(i_=0; i_<=wcount-1;i_++)
{
v += network.weights[i_]*network.weights[i_];
}
e = e+0.5*decay*v;
for(i_=0; i_<=wcount-1;i_++)
{
g[i_] = g[i_] + decay*network.weights[i_];
}
rep.ngrad = rep.ngrad+1;
mnlmcsrch(wcount, ref network.weights, ref e, ref g, ref wdir, ref wstep, ref mcinfo, ref mcnfev, ref work, ref mcstate, ref mcstage);
}
if( spd & (mcinfo==2 | mcinfo==4 | mcinfo==6) )
{
break;
}
}
//
// Convert from NN format to MNL format
//
i1_ = (0) - (offs);
for(i_=offs; i_<=offs+wcount-1;i_++)
{
lm.w[i_] = network.weights[i_+i1_];
}
for(k=0; k<=nvars-1; k++)
{
for(i=0; i<=nclasses-2; i++)
{
s = network.columnsigmas[k];
if( (double)(s)==(double)(0) )
{
s = 1;
}
j = offs+(nvars+1)*i;
v = lm.w[j+k];
lm.w[j+k] = v/s;
lm.w[j+nvars] = lm.w[j+nvars]+v*network.columnmeans[k]/s;
}
}
for(k=0; k<=nclasses-2; k++)
{
lm.w[offs+(nvars+1)*k+nvars] = -lm.w[offs+(nvars+1)*k+nvars];
}
}
/*************************************************************************
Neural network training using modified Levenberg-Marquardt with exact
Hessian calculation and regularization. Subroutine trains neural network
with restarts from random positions. Algorithm is well suited for small
and medium scale problems (hundreds of weights).
INPUT PARAMETERS:
Network - neural network with initialized geometry
XY - training set
NPoints - training set size
Decay - weight decay constant, >=0.001
Decay term 'Decay*||Weights||^2' is added to error
function.
If you don't know what Decay to choose, use 0.001.
Restarts - number of restarts from random position, >0.
If you don't know what Restarts to choose, use 2.
OUTPUT PARAMETERS:
Network - trained neural network.
Info - return code:
* -9, if internal matrix inverse subroutine failed
* -2, if there is a point with class number
outside of [0..NOut-1].
* -1, if wrong parameters specified
(NPoints<0, Restarts<1).
* 2, if task has been solved.
Rep - training report
-- ALGLIB --
Copyright 10.03.2009 by Bochkanov Sergey
*************************************************************************/
public static void mlptrainlm(ref mlpbase.multilayerperceptron network,
ref double[,] xy,
int npoints,
double decay,
int restarts,
ref int info,
ref mlpreport rep)
{
int nin = 0;
int nout = 0;
int wcount = 0;
double lmftol = 0;
double lmsteptol = 0;
int i = 0;
int k = 0;
double v = 0;
double e = 0;
double enew = 0;
double xnorm2 = 0;
double stepnorm = 0;
double[] g = new double[0];
double[] d = new double[0];
double[,] h = new double[0,0];
double[,] hmod = new double[0,0];
double[,] z = new double[0,0];
bool spd = new bool();
double nu = 0;
double lambda = 0;
double lambdaup = 0;
double lambdadown = 0;
minlbfgs.minlbfgsreport internalrep = new minlbfgs.minlbfgsreport();
minlbfgs.minlbfgsstate state = new minlbfgs.minlbfgsstate();
double[] x = new double[0];
double[] y = new double[0];
double[] wbase = new double[0];
double[] wdir = new double[0];
double[] wt = new double[0];
double[] wx = new double[0];
int pass = 0;
double[] wbest = new double[0];
double ebest = 0;
int invinfo = 0;
matinv.matinvreport invrep = new matinv.matinvreport();
int solverinfo = 0;
densesolver.densesolverreport solverrep = new densesolver.densesolverreport();
int i_ = 0;
mlpbase.mlpproperties(ref network, ref nin, ref nout, ref wcount);
lambdaup = 10;
lambdadown = 0.3;
lmftol = 0.001;
lmsteptol = 0.001;
//
// Test for inputs
//
if( npoints<=0 | restarts<1 )
{
info = -1;
return;
}
if( mlpbase.mlpissoftmax(ref network) )
{
for(i=0; i<=npoints-1; i++)
{
if( (int)Math.Round(xy[i,nin])<0 | (int)Math.Round(xy[i,nin])>=nout )
{
info = -2;
return;
}
}
}
decay = Math.Max(decay, mindecay);
info = 2;
//
// Initialize data
//
rep.ngrad = 0;
rep.nhess = 0;
rep.ncholesky = 0;
//
// General case.
// Prepare task and network. Allocate space.
//
mlpbase.mlpinitpreprocessor(ref network, ref xy, npoints);
g = new double[wcount-1+1];
h = new double[wcount-1+1, wcount-1+1];
hmod = new double[wcount-1+1, wcount-1+1];
wbase = new double[wcount-1+1];
wdir = new double[wcount-1+1];
wbest = new double[wcount-1+1];
wt = new double[wcount-1+1];
wx = new double[wcount-1+1];
ebest = AP.Math.MaxRealNumber;
//
// Multiple passes
//
for(pass=1; pass<=restarts; pass++)
{
//
// Initialize weights
//
mlpbase.mlprandomize(ref network);
//
// First stage of the hybrid algorithm: LBFGS
//
for(i_=0; i_<=wcount-1;i_++)
{
wbase[i_] = network.weights[i_];
}
minlbfgs.minlbfgscreate(wcount, Math.Min(wcount, 5), ref wbase, ref state);
minlbfgs.minlbfgssetcond(ref state, 0, 0, 0, Math.Max(25, wcount));
while( minlbfgs.minlbfgsiteration(ref state) )
{
//
// gradient
//
for(i_=0; i_<=wcount-1;i_++)
{
network.weights[i_] = state.x[i_];
}
mlpbase.mlpgradbatch(ref network, ref xy, npoints, ref state.f, ref state.g);
//
// weight decay
//
v = 0.0;
for(i_=0; i_<=wcount-1;i_++)
{
v += network.weights[i_]*network.weights[i_];
}
state.f = state.f+0.5*decay*v;
for(i_=0; i_<=wcount-1;i_++)
{
state.g[i_] = state.g[i_] + decay*network.weights[i_];
}
//
// next iteration
//
rep.ngrad = rep.ngrad+1;
}
minlbfgs.minlbfgsresults(ref state, ref wbase, ref internalrep);
for(i_=0; i_<=wcount-1;i_++)
{
network.weights[i_] = wbase[i_];
}
//
// Second stage of the hybrid algorithm: LM
//
// Initialize H with identity matrix,
// G with gradient,
// E with regularized error.
//
mlpbase.mlphessianbatch(ref network, ref xy, npoints, ref e, ref g, ref h);
v = 0.0;
for(i_=0; i_<=wcount-1;i_++)
{
v += network.weights[i_]*network.weights[i_];
}
e = e+0.5*decay*v;
for(i_=0; i_<=wcount-1;i_++)
{
g[i_] = g[i_] + decay*network.weights[i_];
}
for(k=0; k<=wcount-1; k++)
{
h[k,k] = h[k,k]+decay;
}
rep.nhess = rep.nhess+1;
lambda = 0.001;
nu = 2;
while( true )
{
//
// 1. HMod = H+lambda*I
// 2. Try to solve (H+Lambda*I)*dx = -g.
// Increase lambda if left part is not positive definite.
//
for(i=0; i<=wcount-1; i++)
{
for(i_=0; i_<=wcount-1;i_++)
{
hmod[i,i_] = h[i,i_];
}
hmod[i,i] = hmod[i,i]+lambda;
}
spd = trfac.spdmatrixcholesky(ref hmod, wcount, true);
rep.ncholesky = rep.ncholesky+1;
if( !spd )
{
lambda = lambda*lambdaup*nu;
nu = nu*2;
continue;
}
densesolver.spdmatrixcholeskysolve(ref hmod, wcount, true, ref g, ref solverinfo, ref solverrep, ref wdir);
if( solverinfo<0 )
{
lambda = lambda*lambdaup*nu;
nu = nu*2;
continue;
}
for(i_=0; i_<=wcount-1;i_++)
{
wdir[i_] = -1*wdir[i_];
}
//
// Lambda found.
// 1. Save old w in WBase
// 1. Test some stopping criterions
// 2. If error(w+wdir)>error(w), increase lambda
//
for(i_=0; i_<=wcount-1;i_++)
{
network.weights[i_] = network.weights[i_] + wdir[i_];
}
xnorm2 = 0.0;
for(i_=0; i_<=wcount-1;i_++)
{
xnorm2 += network.weights[i_]*network.weights[i_];
}
stepnorm = 0.0;
for(i_=0; i_<=wcount-1;i_++)
{
stepnorm += wdir[i_]*wdir[i_];
}
stepnorm = Math.Sqrt(stepnorm);
enew = mlpbase.mlperror(ref network, ref xy, npoints)+0.5*decay*xnorm2;
if( (double)(stepnorm)<(double)(lmsteptol*(1+Math.Sqrt(xnorm2))) )
{
break;
}
if( (double)(enew)>(double)(e) )
{
lambda = lambda*lambdaup*nu;
nu = nu*2;
continue;
}
//
// Optimize using inv(cholesky(H)) as preconditioner
//
matinv.rmatrixtrinverse(ref hmod, wcount, true, false, ref invinfo, ref invrep);
if( invinfo<=0 )
{
//
// if matrix can't be inverted then exit with errors
// TODO: make WCount steps in direction suggested by HMod
//
info = -9;
return;
}
for(i_=0; i_<=wcount-1;i_++)
{
wbase[i_] = network.weights[i_];
}
for(i=0; i<=wcount-1; i++)
{
wt[i] = 0;
}
minlbfgs.minlbfgscreatex(wcount, wcount, ref wt, 1, ref state);
minlbfgs.minlbfgssetcond(ref state, 0, 0, 0, 5);
while( minlbfgs.minlbfgsiteration(ref state) )
{
//
// gradient
//
for(i=0; i<=wcount-1; i++)
{
v = 0.0;
for(i_=i; i_<=wcount-1;i_++)
{
v += state.x[i_]*hmod[i,i_];
}
network.weights[i] = wbase[i]+v;
}
mlpbase.mlpgradbatch(ref network, ref xy, npoints, ref state.f, ref g);
for(i=0; i<=wcount-1; i++)
{
state.g[i] = 0;
}
for(i=0; i<=wcount-1; i++)
{
v = g[i];
for(i_=i; i_<=wcount-1;i_++)
{
state.g[i_] = state.g[i_] + v*hmod[i,i_];
}
}
//
// weight decay
// grad(x'*x) = A'*(x0+A*t)
//
v = 0.0;
for(i_=0; i_<=wcount-1;i_++)
{
v += network.weights[i_]*network.weights[i_];
}
state.f = state.f+0.5*decay*v;
for(i=0; i<=wcount-1; i++)
{
v = decay*network.weights[i];
for(i_=i; i_<=wcount-1;i_++)
{
state.g[i_] = state.g[i_] + v*hmod[i,i_];
}
}
//
// next iteration
//
rep.ngrad = rep.ngrad+1;
}
minlbfgs.minlbfgsresults(ref state, ref wt, ref internalrep);
//
// Accept new position.
// Calculate Hessian
//
for(i=0; i<=wcount-1; i++)
{
v = 0.0;
for(i_=i; i_<=wcount-1;i_++)
{
v += wt[i_]*hmod[i,i_];
}
network.weights[i] = wbase[i]+v;
}
mlpbase.mlphessianbatch(ref network, ref xy, npoints, ref e, ref g, ref h);
v = 0.0;
for(i_=0; i_<=wcount-1;i_++)
{
v += network.weights[i_]*network.weights[i_];
}
e = e+0.5*decay*v;
for(i_=0; i_<=wcount-1;i_++)
{
g[i_] = g[i_] + decay*network.weights[i_];
}
for(k=0; k<=wcount-1; k++)
{
h[k,k] = h[k,k]+decay;
}
rep.nhess = rep.nhess+1;
//
// Update lambda
//
lambda = lambda*lambdadown;
nu = 2;
}
//
// update WBest
//
v = 0.0;
for(i_=0; i_<=wcount-1;i_++)
{
v += network.weights[i_]*network.weights[i_];
}
e = 0.5*decay*v+mlpbase.mlperror(ref network, ref xy, npoints);
if( (double)(e)<(double)(ebest) )
{
ebest = e;
for(i_=0; i_<=wcount-1;i_++)
{
wbest[i_] = network.weights[i_];
}
}
}
//
// copy WBest to output
//
for(i_=0; i_<=wcount-1;i_++)
{
network.weights[i_] = wbest[i_];
}
}
