/*************************************************************************
* Cross-validation estimate of generalization error.
*
* Base algorithm - Levenberg-Marquardt.
*
* INPUT PARAMETERS:
* Network - neural network with initialized geometry. Network is
* not changed during cross-validation - it is used only
* as a representative of its architecture.
* XY - training set.
* SSize - training set size
* Decay - weight decay, same as in MLPTrainLBFGS
* Restarts - number of restarts, >0.
* restarts are counted for each partition separately, so
* total number of restarts will be Restarts*FoldsCount.
* FoldsCount - number of folds in k-fold cross-validation,
* 2<=FoldsCount<=SSize.
* recommended value: 10.
*
* OUTPUT PARAMETERS:
* Info - return code, same as in MLPTrainLBFGS
* Rep - report, same as in MLPTrainLM/MLPTrainLBFGS
* CVRep - generalization error estimates
*
* -- ALGLIB --
* Copyright 09.12.2007 by Bochkanov Sergey
*************************************************************************/
public static void mlpkfoldcvlm(ref mlpbase.multilayerperceptron network,
ref double[,] xy,
int npoints,
double decay,
int restarts,
int foldscount,
ref int info,
ref mlpreport rep,
ref mlpcvreport cvrep)
{
mlpkfoldcvgeneral(ref network, ref xy, npoints, decay, restarts, foldscount, true, 0.0, 0, ref info, ref rep, ref cvrep);
}
/*************************************************************************
* Cross-validation estimate of generalization error.
*
* Base algorithm - L-BFGS.
*
* INPUT PARAMETERS:
* Network - neural network with initialized geometry. Network is
* not changed during cross-validation - it is used only
* as a representative of its architecture.
* XY - training set.
* SSize - training set size
* Decay - weight decay, same as in MLPTrainLBFGS
* Restarts - number of restarts, >0.
* restarts are counted for each partition separately, so
* total number of restarts will be Restarts*FoldsCount.
* WStep - stopping criterion, same as in MLPTrainLBFGS
* MaxIts - stopping criterion, same as in MLPTrainLBFGS
* FoldsCount - number of folds in k-fold cross-validation,
* 2<=FoldsCount<=SSize.
* recommended value: 10.
*
* OUTPUT PARAMETERS:
* Info - return code, same as in MLPTrainLBFGS
* Rep - report, same as in MLPTrainLM/MLPTrainLBFGS
* CVRep - generalization error estimates
*
* -- ALGLIB --
* Copyright 09.12.2007 by Bochkanov Sergey
*************************************************************************/
public static void mlpkfoldcvlbfgs(ref mlpbase.multilayerperceptron network,
ref double[,] xy,
int npoints,
double decay,
int restarts,
double wstep,
int maxits,
int foldscount,
ref int info,
ref mlpreport rep,
ref mlpcvreport cvrep)
{
mlpkfoldcvgeneral(ref network, ref xy, npoints, decay, restarts, foldscount, false, wstep, maxits, ref info, ref rep, ref cvrep);
}
/*************************************************************************
This function trains neural network passed to this function, using current
dataset (one which was passed to MLPSetDataset() or MLPSetSparseDataset())
and current training settings. Training from NRestarts random starting
positions is performed, best network is chosen.
Training is performed using current training algorithm.
INPUT PARAMETERS:
S - trainer object
Network - neural network. It must have same number of inputs and
output/classes as was specified during creation of the
trainer object.
NRestarts - number of restarts, >=0:
* NRestarts>0 means that specified number of random
restarts are performed, best network is chosen after
training
* NRestarts=0 means that current state of the network
is used for training.
OUTPUT PARAMETERS:
Network - trained network
NOTE: when no dataset was specified with MLPSetDataset/SetSparseDataset(),
network is filled by zero values. Same behavior for functions
MLPStartTraining and MLPContinueTraining.
NOTE: this method uses sum-of-squares error function for training.
-- ALGLIB --
Copyright 23.07.2012 by Bochkanov Sergey
*************************************************************************/
public static void mlptrainnetwork(mlptrainer s,
mlpbase.multilayerperceptron network,
int nrestarts,
mlpreport rep)
{
int nin = 0;
int nout = 0;
int wcount = 0;
int ntype = 0;
int ttype = 0;
alglib.ap.assert(s.npoints>=0, "MLPTrainNetwork: parameter S is not initialized or is spoiled(S.NPoints<0)");
if( !mlpbase.mlpissoftmax(network) )
{
ntype = 0;
}
else
{
ntype = 1;
}
if( s.rcpar )
{
ttype = 0;
}
else
{
ttype = 1;
}
alglib.ap.assert(ntype==ttype, "MLPTrainNetwork: type of input network is not similar to network type in trainer object");
mlpbase.mlpproperties(network, ref nin, ref nout, ref wcount);
alglib.ap.assert(s.nin==nin, "MLPTrainNetwork: number of inputs in trainer is not equal to number of inputs in network");
alglib.ap.assert(s.nout==nout, "MLPTrainNetwork: number of outputs in trainer is not equal to number of outputs in network");
alglib.ap.assert(nrestarts>=0, "MLPTrainNetwork: NRestarts<0.");
apserv.rvectorsetlengthatleast(ref s.wbest, wcount);
apserv.rvectorsetlengthatleast(ref s.wfinal, wcount);
//
// Create LBFGS optimizer
//
minlbfgs.minlbfgscreate(wcount, Math.Min(wcount, s.lbfgsfactor), network.weights, s.tstate);
minlbfgs.minlbfgssetcond(s.tstate, 0.0, 0.0, s.wstep, s.maxits);
minlbfgs.minlbfgssetxrep(s.tstate, true);
//
// Create duplicate of the network
//
mlpbase.mlpcopy(network, s.tnetwork);
//
// Train
//
mlptrainnetworkx(s, network, s.tnetwork, s.tstate, nrestarts, s.subset, -1, s.subset, 0, s.wbest, s.wfinal, rep);
}
/*************************************************************************
Cross-validation estimate of generalization error.
Base algorithm - Levenberg-Marquardt.
INPUT PARAMETERS:
Network - neural network with initialized geometry. Network is
not changed during cross-validation - it is used only
as a representative of its architecture.
XY - training set.
SSize - training set size
Decay - weight decay, same as in MLPTrainLBFGS
Restarts - number of restarts, >0.
restarts are counted for each partition separately, so
total number of restarts will be Restarts*FoldsCount.
FoldsCount - number of folds in k-fold cross-validation,
2<=FoldsCount<=SSize.
recommended value: 10.
OUTPUT PARAMETERS:
Info - return code, same as in MLPTrainLBFGS
Rep - report, same as in MLPTrainLM/MLPTrainLBFGS
CVRep - generalization error estimates
-- ALGLIB --
Copyright 09.12.2007 by Bochkanov Sergey
*************************************************************************/
public static void mlpkfoldcvlm(mlpbase.multilayerperceptron network,
double[,] xy,
int npoints,
double decay,
int restarts,
int foldscount,
ref int info,
mlpreport rep,
mlpcvreport cvrep)
{
info = 0;
mlpkfoldcvgeneral(network, xy, npoints, decay, restarts, foldscount, true, 0.0, 0, ref info, rep, cvrep);
}
/*************************************************************************
Neural network training using early stopping (base algorithm - L-BFGS with
regularization).
INPUT PARAMETERS:
Network - neural network with initialized geometry
TrnXY - training set
TrnSize - training set size, TrnSize>0
ValXY - validation set
ValSize - validation set size, ValSize>0
Decay - weight decay constant, >=0.001
Decay term 'Decay*||Weights||^2' is added to error
function.
If you don't know what Decay to choose, use 0.001.
Restarts - number of restarts, either:
* strictly positive number - algorithm make specified
number of restarts from random position.
* -1, in which case algorithm makes exactly one run
from the initial state of the network (no randomization).
If you don't know what Restarts to choose, choose one
one the following:
* -1 (deterministic start)
* +1 (one random restart)
* +5 (moderate amount of random restarts)
OUTPUT PARAMETERS:
Network - trained neural network.
Info - return code:
* -2, if there is a point with class number
outside of [0..NOut-1].
* -1, if wrong parameters specified
(NPoints<0, Restarts<1, ...).
* 2, task has been solved, stopping criterion met -
sufficiently small step size. Not expected (we
use EARLY stopping) but possible and not an
error.
* 6, task has been solved, stopping criterion met -
increasing of validation set error.
Rep - training report
NOTE:
Algorithm stops if validation set error increases for a long enough or
step size is small enought (there are task where validation set may
decrease for eternity). In any case solution returned corresponds to the
minimum of validation set error.
-- ALGLIB --
Copyright 10.03.2009 by Bochkanov Sergey
*************************************************************************/
public static void mlptraines(mlpbase.multilayerperceptron network,
double[,] trnxy,
int trnsize,
double[,] valxy,
int valsize,
double decay,
int restarts,
ref int info,
mlpreport rep)
{
int i = 0;
int pass = 0;
int nin = 0;
int nout = 0;
int wcount = 0;
double[] w = new double[0];
double[] wbest = new double[0];
double e = 0;
double v = 0;
double ebest = 0;
double[] wfinal = new double[0];
double efinal = 0;
int itcnt = 0;
int itbest = 0;
minlbfgs.minlbfgsreport internalrep = new minlbfgs.minlbfgsreport();
minlbfgs.minlbfgsstate state = new minlbfgs.minlbfgsstate();
double wstep = 0;
bool needrandomization = new bool();
int i_ = 0;
info = 0;
wstep = 0.001;
//
// Test inputs, parse flags, read network geometry
//
if( ((trnsize<=0 || valsize<=0) || (restarts<1 && restarts!=-1)) || (double)(decay)<(double)(0) )
{
info = -1;
return;
}
if( restarts==-1 )
{
needrandomization = false;
restarts = 1;
}
else
{
needrandomization = true;
}
mlpbase.mlpproperties(network, ref nin, ref nout, ref wcount);
if( mlpbase.mlpissoftmax(network) )
{
for(i=0; i<=trnsize-1; i++)
{
if( (int)Math.Round(trnxy[i,nin])<0 || (int)Math.Round(trnxy[i,nin])>=nout )
{
info = -2;
return;
}
}
for(i=0; i<=valsize-1; i++)
{
if( (int)Math.Round(valxy[i,nin])<0 || (int)Math.Round(valxy[i,nin])>=nout )
{
info = -2;
return;
}
}
}
info = 2;
//
// Prepare
//
mlpbase.mlpinitpreprocessor(network, trnxy, trnsize);
w = new double[wcount-1+1];
wbest = new double[wcount-1+1];
wfinal = new double[wcount-1+1];
efinal = math.maxrealnumber;
for(i=0; i<=wcount-1; i++)
{
wfinal[i] = 0;
}
//
// Multiple starts
//
rep.ncholesky = 0;
rep.nhess = 0;
rep.ngrad = 0;
for(pass=1; pass<=restarts; pass++)
{
//
// Process
//
if( needrandomization )
{
mlpbase.mlprandomize(network);
}
ebest = mlpbase.mlperror(network, valxy, valsize);
for(i_=0; i_<=wcount-1;i_++)
{
wbest[i_] = network.weights[i_];
}
itbest = 0;
itcnt = 0;
for(i_=0; i_<=wcount-1;i_++)
{
w[i_] = network.weights[i_];
}
minlbfgs.minlbfgscreate(wcount, Math.Min(wcount, 10), w, state);
minlbfgs.minlbfgssetcond(state, 0.0, 0.0, wstep, 0);
minlbfgs.minlbfgssetxrep(state, true);
while( minlbfgs.minlbfgsiteration(state) )
{
//
// Calculate gradient
//
if( state.needfg )
{
for(i_=0; i_<=wcount-1;i_++)
{
network.weights[i_] = state.x[i_];
}
mlpbase.mlpgradnbatch(network, trnxy, trnsize, ref state.f, ref state.g);
v = 0.0;
for(i_=0; i_<=wcount-1;i_++)
{
v += network.weights[i_]*network.weights[i_];
}
state.f = state.f+0.5*decay*v;
for(i_=0; i_<=wcount-1;i_++)
{
state.g[i_] = state.g[i_] + decay*network.weights[i_];
}
rep.ngrad = rep.ngrad+1;
}
//
// Validation set
//
if( state.xupdated )
{
for(i_=0; i_<=wcount-1;i_++)
{
network.weights[i_] = state.x[i_];
}
e = mlpbase.mlperror(network, valxy, valsize);
if( (double)(e)<(double)(ebest) )
{
ebest = e;
for(i_=0; i_<=wcount-1;i_++)
{
wbest[i_] = network.weights[i_];
}
itbest = itcnt;
}
if( itcnt>30 && (double)(itcnt)>(double)(1.5*itbest) )
{
info = 6;
break;
}
itcnt = itcnt+1;
}
}
minlbfgs.minlbfgsresults(state, ref w, internalrep);
//
// Compare with final answer
//
if( (double)(ebest)<(double)(efinal) )
{
for(i_=0; i_<=wcount-1;i_++)
{
wfinal[i_] = wbest[i_];
}
efinal = ebest;
}
}
//
// The best network
//
for(i_=0; i_<=wcount-1;i_++)
{
network.weights[i_] = wfinal[i_];
}
}
/*************************************************************************
Neural network training using modified Levenberg-Marquardt with exact
Hessian calculation and regularization. Subroutine trains neural network
with restarts from random positions. Algorithm is well suited for small
and medium scale problems (hundreds of weights).
INPUT PARAMETERS:
Network - neural network with initialized geometry
XY - training set
NPoints - training set size
Decay - weight decay constant, >=0.001
Decay term 'Decay*||Weights||^2' is added to error
function.
If you don't know what Decay to choose, use 0.001.
Restarts - number of restarts from random position, >0.
If you don't know what Restarts to choose, use 2.
OUTPUT PARAMETERS:
Network - trained neural network.
Info - return code:
* -9, if internal matrix inverse subroutine failed
* -2, if there is a point with class number
outside of [0..NOut-1].
* -1, if wrong parameters specified
(NPoints<0, Restarts<1).
* 2, if task has been solved.
Rep - training report
-- ALGLIB --
Copyright 10.03.2009 by Bochkanov Sergey
*************************************************************************/
public static void mlptrainlm(mlpbase.multilayerperceptron network,
double[,] xy,
int npoints,
double decay,
int restarts,
ref int info,
mlpreport rep)
{
int nin = 0;
int nout = 0;
int wcount = 0;
double lmftol = 0;
double lmsteptol = 0;
int i = 0;
int k = 0;
double v = 0;
double e = 0;
double enew = 0;
double xnorm2 = 0;
double stepnorm = 0;
double[] g = new double[0];
double[] d = new double[0];
double[,] h = new double[0,0];
double[,] hmod = new double[0,0];
double[,] z = new double[0,0];
bool spd = new bool();
double nu = 0;
double lambdav = 0;
double lambdaup = 0;
double lambdadown = 0;
minlbfgs.minlbfgsreport internalrep = new minlbfgs.minlbfgsreport();
minlbfgs.minlbfgsstate state = new minlbfgs.minlbfgsstate();
double[] x = new double[0];
double[] y = new double[0];
double[] wbase = new double[0];
double[] wdir = new double[0];
double[] wt = new double[0];
double[] wx = new double[0];
int pass = 0;
double[] wbest = new double[0];
double ebest = 0;
int invinfo = 0;
matinv.matinvreport invrep = new matinv.matinvreport();
int solverinfo = 0;
densesolver.densesolverreport solverrep = new densesolver.densesolverreport();
int i_ = 0;
info = 0;
mlpbase.mlpproperties(network, ref nin, ref nout, ref wcount);
lambdaup = 10;
lambdadown = 0.3;
lmftol = 0.001;
lmsteptol = 0.001;
//
// Test for inputs
//
if( npoints<=0 || restarts<1 )
{
info = -1;
return;
}
if( mlpbase.mlpissoftmax(network) )
{
for(i=0; i<=npoints-1; i++)
{
if( (int)Math.Round(xy[i,nin])<0 || (int)Math.Round(xy[i,nin])>=nout )
{
info = -2;
return;
}
}
}
decay = Math.Max(decay, mindecay);
info = 2;
//
// Initialize data
//
rep.ngrad = 0;
rep.nhess = 0;
rep.ncholesky = 0;
//
// General case.
// Prepare task and network. Allocate space.
//
mlpbase.mlpinitpreprocessor(network, xy, npoints);
g = new double[wcount-1+1];
h = new double[wcount-1+1, wcount-1+1];
hmod = new double[wcount-1+1, wcount-1+1];
wbase = new double[wcount-1+1];
wdir = new double[wcount-1+1];
wbest = new double[wcount-1+1];
wt = new double[wcount-1+1];
wx = new double[wcount-1+1];
ebest = math.maxrealnumber;
//
// Multiple passes
//
for(pass=1; pass<=restarts; pass++)
{
//
// Initialize weights
//
mlpbase.mlprandomize(network);
//
// First stage of the hybrid algorithm: LBFGS
//
for(i_=0; i_<=wcount-1;i_++)
{
wbase[i_] = network.weights[i_];
}
minlbfgs.minlbfgscreate(wcount, Math.Min(wcount, 5), wbase, state);
minlbfgs.minlbfgssetcond(state, 0, 0, 0, Math.Max(25, wcount));
while( minlbfgs.minlbfgsiteration(state) )
{
//
// gradient
//
for(i_=0; i_<=wcount-1;i_++)
{
network.weights[i_] = state.x[i_];
}
mlpbase.mlpgradbatch(network, xy, npoints, ref state.f, ref state.g);
//
// weight decay
//
v = 0.0;
for(i_=0; i_<=wcount-1;i_++)
{
v += network.weights[i_]*network.weights[i_];
}
state.f = state.f+0.5*decay*v;
for(i_=0; i_<=wcount-1;i_++)
{
state.g[i_] = state.g[i_] + decay*network.weights[i_];
}
//
// next iteration
//
rep.ngrad = rep.ngrad+1;
}
minlbfgs.minlbfgsresults(state, ref wbase, internalrep);
for(i_=0; i_<=wcount-1;i_++)
{
network.weights[i_] = wbase[i_];
}
//
// Second stage of the hybrid algorithm: LM
//
// Initialize H with identity matrix,
// G with gradient,
// E with regularized error.
//
mlpbase.mlphessianbatch(network, xy, npoints, ref e, ref g, ref h);
v = 0.0;
for(i_=0; i_<=wcount-1;i_++)
{
v += network.weights[i_]*network.weights[i_];
}
e = e+0.5*decay*v;
for(i_=0; i_<=wcount-1;i_++)
{
g[i_] = g[i_] + decay*network.weights[i_];
}
for(k=0; k<=wcount-1; k++)
{
h[k,k] = h[k,k]+decay;
}
rep.nhess = rep.nhess+1;
lambdav = 0.001;
nu = 2;
while( true )
{
//
// 1. HMod = H+lambda*I
// 2. Try to solve (H+Lambda*I)*dx = -g.
// Increase lambda if left part is not positive definite.
//
for(i=0; i<=wcount-1; i++)
{
for(i_=0; i_<=wcount-1;i_++)
{
hmod[i,i_] = h[i,i_];
}
hmod[i,i] = hmod[i,i]+lambdav;
}
spd = trfac.spdmatrixcholesky(ref hmod, wcount, true);
rep.ncholesky = rep.ncholesky+1;
if( !spd )
{
lambdav = lambdav*lambdaup*nu;
nu = nu*2;
continue;
}
densesolver.spdmatrixcholeskysolve(hmod, wcount, true, g, ref solverinfo, solverrep, ref wdir);
if( solverinfo<0 )
{
lambdav = lambdav*lambdaup*nu;
nu = nu*2;
continue;
}
for(i_=0; i_<=wcount-1;i_++)
{
wdir[i_] = -1*wdir[i_];
}
//
// Lambda found.
// 1. Save old w in WBase
// 1. Test some stopping criterions
// 2. If error(w+wdir)>error(w), increase lambda
//
for(i_=0; i_<=wcount-1;i_++)
{
network.weights[i_] = network.weights[i_] + wdir[i_];
}
xnorm2 = 0.0;
for(i_=0; i_<=wcount-1;i_++)
{
xnorm2 += network.weights[i_]*network.weights[i_];
}
stepnorm = 0.0;
for(i_=0; i_<=wcount-1;i_++)
{
stepnorm += wdir[i_]*wdir[i_];
}
stepnorm = Math.Sqrt(stepnorm);
enew = mlpbase.mlperror(network, xy, npoints)+0.5*decay*xnorm2;
if( (double)(stepnorm)<(double)(lmsteptol*(1+Math.Sqrt(xnorm2))) )
{
break;
}
if( (double)(enew)>(double)(e) )
{
lambdav = lambdav*lambdaup*nu;
nu = nu*2;
continue;
}
//
// Optimize using inv(cholesky(H)) as preconditioner
//
matinv.rmatrixtrinverse(ref hmod, wcount, true, false, ref invinfo, invrep);
if( invinfo<=0 )
{
//
// if matrix can't be inverted then exit with errors
// TODO: make WCount steps in direction suggested by HMod
//
info = -9;
return;
}
for(i_=0; i_<=wcount-1;i_++)
{
wbase[i_] = network.weights[i_];
}
for(i=0; i<=wcount-1; i++)
{
wt[i] = 0;
}
minlbfgs.minlbfgscreatex(wcount, wcount, wt, 1, 0.0, state);
minlbfgs.minlbfgssetcond(state, 0, 0, 0, 5);
while( minlbfgs.minlbfgsiteration(state) )
{
//
// gradient
//
for(i=0; i<=wcount-1; i++)
{
v = 0.0;
for(i_=i; i_<=wcount-1;i_++)
{
v += state.x[i_]*hmod[i,i_];
}
network.weights[i] = wbase[i]+v;
}
mlpbase.mlpgradbatch(network, xy, npoints, ref state.f, ref g);
for(i=0; i<=wcount-1; i++)
{
state.g[i] = 0;
}
for(i=0; i<=wcount-1; i++)
{
v = g[i];
for(i_=i; i_<=wcount-1;i_++)
{
state.g[i_] = state.g[i_] + v*hmod[i,i_];
}
}
//
// weight decay
// grad(x'*x) = A'*(x0+A*t)
//
v = 0.0;
for(i_=0; i_<=wcount-1;i_++)
{
v += network.weights[i_]*network.weights[i_];
}
state.f = state.f+0.5*decay*v;
for(i=0; i<=wcount-1; i++)
{
v = decay*network.weights[i];
for(i_=i; i_<=wcount-1;i_++)
{
state.g[i_] = state.g[i_] + v*hmod[i,i_];
}
}
//
// next iteration
//
rep.ngrad = rep.ngrad+1;
}
minlbfgs.minlbfgsresults(state, ref wt, internalrep);
//
// Accept new position.
// Calculate Hessian
//
for(i=0; i<=wcount-1; i++)
{
v = 0.0;
for(i_=i; i_<=wcount-1;i_++)
{
v += wt[i_]*hmod[i,i_];
}
network.weights[i] = wbase[i]+v;
}
mlpbase.mlphessianbatch(network, xy, npoints, ref e, ref g, ref h);
v = 0.0;
for(i_=0; i_<=wcount-1;i_++)
{
v += network.weights[i_]*network.weights[i_];
}
e = e+0.5*decay*v;
for(i_=0; i_<=wcount-1;i_++)
{
g[i_] = g[i_] + decay*network.weights[i_];
}
for(k=0; k<=wcount-1; k++)
{
h[k,k] = h[k,k]+decay;
}
rep.nhess = rep.nhess+1;
//
// Update lambda
//
lambdav = lambdav*lambdadown;
nu = 2;
}
//
// update WBest
//
v = 0.0;
for(i_=0; i_<=wcount-1;i_++)
{
v += network.weights[i_]*network.weights[i_];
}
e = 0.5*decay*v+mlpbase.mlperror(network, xy, npoints);
if( (double)(e)<(double)(ebest) )
{
ebest = e;
for(i_=0; i_<=wcount-1;i_++)
{
wbest[i_] = network.weights[i_];
}
}
}
//
// copy WBest to output
//
for(i_=0; i_<=wcount-1;i_++)
{
network.weights[i_] = wbest[i_];
}
}
/*************************************************************************
This function trains neural network passed to this function, using current
dataset (one which was passed to MLPSetDataset() or MLPSetSparseDataset())
and current training settings. Training from NRestarts random starting
positions is performed, best network is chosen.
This function is inteded to be used internally. It may be used in several
settings:
* training with ValSubsetSize=0, corresponds to "normal" training with
termination criteria based on S.MaxIts (steps count) and S.WStep (step
size). Training sample is given by TrnSubset/TrnSubsetSize.
* training with ValSubsetSize>0, corresponds to early stopping training
with additional MaxIts/WStep stopping criteria. Training sample is given
by TrnSubset/TrnSubsetSize, validation sample is given by ValSubset/
ValSubsetSize.
-- ALGLIB --
Copyright 13.08.2012 by Bochkanov Sergey
*************************************************************************/
private static void mlptrainnetworkx(mlptrainer s,
int nrestarts,
int algokind,
int[] trnsubset,
int trnsubsetsize,
int[] valsubset,
int valsubsetsize,
mlpbase.multilayerperceptron network,
mlpreport rep,
bool isrootcall,
alglib.smp.shared_pool sessions)
{
mlpbase.modelerrors modrep = new mlpbase.modelerrors();
double eval = 0;
double ebest = 0;
int ngradbatch = 0;
int nin = 0;
int nout = 0;
int wcount = 0;
int pcount = 0;
int itbest = 0;
int itcnt = 0;
int ntype = 0;
int ttype = 0;
bool rndstart = new bool();
int i = 0;
int nr0 = 0;
int nr1 = 0;
mlpreport rep0 = new mlpreport();
mlpreport rep1 = new mlpreport();
bool randomizenetwork = new bool();
double bestrmserror = 0;
smlptrnsession psession = null;
int i_ = 0;
mlpbase.mlpproperties(network, ref nin, ref nout, ref wcount);
//
// Process root call
//
if( isrootcall )
{
//
// Check correctness of parameters
//
alglib.ap.assert(algokind==0 || algokind==-1, "MLPTrainNetworkX: unexpected AlgoKind");
alglib.ap.assert(s.npoints>=0, "MLPTrainNetworkX: internal error - parameter S is not initialized or is spoiled(S.NPoints<0)");
if( s.rcpar )
{
ttype = 0;
}
else
{
ttype = 1;
}
if( !mlpbase.mlpissoftmax(network) )
{
ntype = 0;
}
else
{
ntype = 1;
}
alglib.ap.assert(ntype==ttype, "MLPTrainNetworkX: internal error - type of the training network is not similar to network type in trainer object");
alglib.ap.assert(s.nin==nin, "MLPTrainNetworkX: internal error - number of inputs in trainer is not equal to number of inputs in the training network.");
alglib.ap.assert(s.nout==nout, "MLPTrainNetworkX: internal error - number of outputs in trainer is not equal to number of outputs in the training network.");
alglib.ap.assert(nrestarts>=0, "MLPTrainNetworkX: internal error - NRestarts<0.");
alglib.ap.assert(alglib.ap.len(trnsubset)>=trnsubsetsize, "MLPTrainNetworkX: internal error - parameter TrnSubsetSize more than input subset size(Length(TrnSubset)<TrnSubsetSize)");
for(i=0; i<=trnsubsetsize-1; i++)
{
alglib.ap.assert(trnsubset[i]>=0 && trnsubset[i]<=s.npoints-1, "MLPTrainNetworkX: internal error - parameter TrnSubset contains incorrect index(TrnSubset[I]<0 or TrnSubset[I]>S.NPoints-1)");
}
alglib.ap.assert(alglib.ap.len(valsubset)>=valsubsetsize, "MLPTrainNetworkX: internal error - parameter ValSubsetSize more than input subset size(Length(ValSubset)<ValSubsetSize)");
for(i=0; i<=valsubsetsize-1; i++)
{
alglib.ap.assert(valsubset[i]>=0 && valsubset[i]<=s.npoints-1, "MLPTrainNetworkX: internal error - parameter ValSubset contains incorrect index(ValSubset[I]<0 or ValSubset[I]>S.NPoints-1)");
}
//
// Train
//
randomizenetwork = nrestarts>0;
initmlptrnsessions(network, randomizenetwork, s, sessions);
mlptrainnetworkx(s, nrestarts, algokind, trnsubset, trnsubsetsize, valsubset, valsubsetsize, network, rep, false, sessions);
//
// Choose best network
//
bestrmserror = math.maxrealnumber;
alglib.smp.ae_shared_pool_first_recycled(sessions, ref psession);
while( psession!=null )
{
if( (double)(psession.bestrmserror)<(double)(bestrmserror) )
{
mlpbase.mlpimporttunableparameters(network, psession.bestparameters);
bestrmserror = psession.bestrmserror;
}
alglib.smp.ae_shared_pool_next_recycled(sessions, ref psession);
}
//
// Calculate errors
//
if( s.datatype==0 )
{
mlpbase.mlpallerrorssubset(network, s.densexy, s.npoints, trnsubset, trnsubsetsize, modrep);
}
if( s.datatype==1 )
{
mlpbase.mlpallerrorssparsesubset(network, s.sparsexy, s.npoints, trnsubset, trnsubsetsize, modrep);
}
rep.relclserror = modrep.relclserror;
rep.avgce = modrep.avgce;
rep.rmserror = modrep.rmserror;
rep.avgerror = modrep.avgerror;
rep.avgrelerror = modrep.avgrelerror;
//
// Done
//
return;
}
//
// Split problem, if we have more than 1 restart
//
if( nrestarts>=2 )
{
//
// Divide problem with NRestarts into two: NR0 and NR1.
//
nr0 = nrestarts/2;
nr1 = nrestarts-nr0;
mlptrainnetworkx(s, nr0, algokind, trnsubset, trnsubsetsize, valsubset, valsubsetsize, network, rep0, false, sessions);
mlptrainnetworkx(s, nr1, algokind, trnsubset, trnsubsetsize, valsubset, valsubsetsize, network, rep1, false, sessions);
//
// Aggregate results
//
rep.ngrad = rep0.ngrad+rep1.ngrad;
rep.nhess = rep0.nhess+rep1.nhess;
rep.ncholesky = rep0.ncholesky+rep1.ncholesky;
//
// Done :)
//
return;
}
//
// Execution with NRestarts=1 or NRestarts=0:
// * NRestarts=1 means that network is restarted from random position
// * NRestarts=0 means that network is not randomized
//
alglib.ap.assert(nrestarts==0 || nrestarts==1, "MLPTrainNetworkX: internal error");
rep.ngrad = 0;
rep.nhess = 0;
rep.ncholesky = 0;
alglib.smp.ae_shared_pool_retrieve(sessions, ref psession);
if( ((s.datatype==0 || s.datatype==1) && s.npoints>0) && trnsubsetsize!=0 )
{
//
// Train network using combination of early stopping and step-size
// and step-count based criteria. Network state with best value of
// validation set error is stored in WBuf0. When validation set is
// zero, most recent state of network is stored.
//
rndstart = nrestarts!=0;
ngradbatch = 0;
eval = 0;
ebest = 0;
itbest = 0;
itcnt = 0;
mlpstarttrainingx(s, rndstart, algokind, trnsubset, trnsubsetsize, psession);
if( s.datatype==0 )
{
ebest = mlpbase.mlperrorsubset(psession.network, s.densexy, s.npoints, valsubset, valsubsetsize);
}
if( s.datatype==1 )
{
ebest = mlpbase.mlperrorsparsesubset(psession.network, s.sparsexy, s.npoints, valsubset, valsubsetsize);
}
for(i_=0; i_<=wcount-1;i_++)
{
psession.wbuf0[i_] = psession.network.weights[i_];
}
while( mlpcontinuetrainingx(s, trnsubset, trnsubsetsize, ref ngradbatch, psession) )
{
if( s.datatype==0 )
{
eval = mlpbase.mlperrorsubset(psession.network, s.densexy, s.npoints, valsubset, valsubsetsize);
}
if( s.datatype==1 )
{
eval = mlpbase.mlperrorsparsesubset(psession.network, s.sparsexy, s.npoints, valsubset, valsubsetsize);
}
if( (double)(eval)<=(double)(ebest) || valsubsetsize==0 )
{
for(i_=0; i_<=wcount-1;i_++)
{
psession.wbuf0[i_] = psession.network.weights[i_];
}
ebest = eval;
itbest = itcnt;
}
if( itcnt>30 && (double)(itcnt)>(double)(1.5*itbest) )
{
break;
}
itcnt = itcnt+1;
}
for(i_=0; i_<=wcount-1;i_++)
{
psession.network.weights[i_] = psession.wbuf0[i_];
}
rep.ngrad = ngradbatch;
}
else
{
for(i=0; i<=wcount-1; i++)
{
psession.network.weights[i] = 0;
}
}
//
// Evaluate network performance and update PSession.BestParameters/BestRMSError
// (if needed).
//
if( s.datatype==0 )
{
mlpbase.mlpallerrorssubset(psession.network, s.densexy, s.npoints, trnsubset, trnsubsetsize, modrep);
}
if( s.datatype==1 )
{
mlpbase.mlpallerrorssparsesubset(psession.network, s.sparsexy, s.npoints, trnsubset, trnsubsetsize, modrep);
}
if( (double)(modrep.rmserror)<(double)(psession.bestrmserror) )
{
mlpbase.mlpexporttunableparameters(psession.network, ref psession.bestparameters, ref pcount);
psession.bestrmserror = modrep.rmserror;
}
//
// Move session back to pool
//
alglib.smp.ae_shared_pool_recycle(sessions, ref psession);
}
/*************************************************************************
This function trains neural network ensemble passed to this function using
current dataset and early stopping training algorithm. Each early stopping
round performs NRestarts random restarts (thus, EnsembleSize*NRestarts
training rounds is performed in total).
FOR USERS OF COMMERCIAL EDITION:
! Commercial version of ALGLIB includes two important improvements of
! this function:
! * multicore support (C++ and C# computational cores)
! * SSE support (C++ computational core)
!
! Second improvement gives constant speedup (2-3X). First improvement
! gives close-to-linear speedup on multicore systems. Following
! operations can be executed in parallel:
! * EnsembleSize training sessions performed for each of ensemble
! members (always parallelized)
! * NRestarts training sessions performed within each of training
! sessions (if NRestarts>1)
! * gradient calculation over large dataset (if dataset is large enough)
!
! In order to use multicore features you have to:
! * use commercial version of ALGLIB
! * call this function with "smp_" prefix, which indicates that
! multicore code will be used (for multicore support)
!
! In order to use SSE features you have to:
! * use commercial version of ALGLIB on Intel processors
! * use C++ computational core
!
! This note is given for users of commercial edition; if you use GPL
! edition, you still will be able to call smp-version of this function,
! but all computations will be done serially.
!
! We recommend you to carefully read ALGLIB Reference Manual, section
! called 'SMP support', before using parallel version of this function.
INPUT PARAMETERS:
S - trainer object;
Ensemble - neural network ensemble. It must have same number of
inputs and outputs/classes as was specified during
creation of the trainer object.
NRestarts - number of restarts, >=0:
* NRestarts>0 means that specified number of random
restarts are performed during each ES round;
* NRestarts=0 is silently replaced by 1.
OUTPUT PARAMETERS:
Ensemble - trained ensemble;
Rep - it contains all type of errors.
NOTE: this training method uses BOTH early stopping and weight decay! So,
you should select weight decay before starting training just as you
select it before training "conventional" networks.
NOTE: when no dataset was specified with MLPSetDataset/SetSparseDataset(),
or single-point dataset was passed, ensemble is filled by zero
values.
NOTE: this method uses sum-of-squares error function for training.
-- ALGLIB --
Copyright 22.08.2012 by Bochkanov Sergey
*************************************************************************/
public static void mlptrainensemblees(mlptrainer s,
mlpe.mlpensemble ensemble,
int nrestarts,
mlpreport rep)
{
int nin = 0;
int nout = 0;
int ntype = 0;
int ttype = 0;
alglib.smp.shared_pool esessions = new alglib.smp.shared_pool();
apserv.sinteger sgrad = new apserv.sinteger();
mlpbase.modelerrors tmprep = new mlpbase.modelerrors();
alglib.ap.assert(s.npoints>=0, "MLPTrainEnsembleES: parameter S is not initialized or is spoiled(S.NPoints<0)");
if( !mlpe.mlpeissoftmax(ensemble) )
{
ntype = 0;
}
else
{
ntype = 1;
}
if( s.rcpar )
{
ttype = 0;
}
else
{
ttype = 1;
}
alglib.ap.assert(ntype==ttype, "MLPTrainEnsembleES: internal error - type of input network is not similar to network type in trainer object");
nin = mlpbase.mlpgetinputscount(ensemble.network);
alglib.ap.assert(s.nin==nin, "MLPTrainEnsembleES: number of inputs in trainer is not equal to number of inputs in ensemble network");
nout = mlpbase.mlpgetoutputscount(ensemble.network);
alglib.ap.assert(s.nout==nout, "MLPTrainEnsembleES: number of outputs in trainer is not equal to number of outputs in ensemble network");
alglib.ap.assert(nrestarts>=0, "MLPTrainEnsembleES: NRestarts<0.");
//
// Initialize parameter Rep
//
rep.relclserror = 0;
rep.avgce = 0;
rep.rmserror = 0;
rep.avgerror = 0;
rep.avgrelerror = 0;
rep.ngrad = 0;
rep.nhess = 0;
rep.ncholesky = 0;
//
// Allocate
//
apserv.ivectorsetlengthatleast(ref s.subset, s.npoints);
apserv.ivectorsetlengthatleast(ref s.valsubset, s.npoints);
//
// Start training
//
// NOTE: ESessions is not initialized because MLPTrainEnsembleX
// needs uninitialized pool.
//
sgrad.val = 0;
mlptrainensemblex(s, ensemble, 0, ensemble.ensemblesize, nrestarts, 0, sgrad, true, esessions);
rep.ngrad = sgrad.val;
//
// Calculate errors.
//
if( s.datatype==0 )
{
mlpe.mlpeallerrorsx(ensemble, s.densexy, s.sparsexy, s.npoints, 0, ensemble.network.dummyidx, 0, s.npoints, 0, ensemble.network.buf, tmprep);
}
if( s.datatype==1 )
{
mlpe.mlpeallerrorsx(ensemble, s.densexy, s.sparsexy, s.npoints, 1, ensemble.network.dummyidx, 0, s.npoints, 0, ensemble.network.buf, tmprep);
}
rep.relclserror = tmprep.relclserror;
rep.avgce = tmprep.avgce;
rep.rmserror = tmprep.rmserror;
rep.avgerror = tmprep.avgerror;
rep.avgrelerror = tmprep.avgrelerror;
}
/*************************************************************************
Training neural networks ensemble using bootstrap aggregating (bagging).
L-BFGS algorithm is used as base training method.
INPUT PARAMETERS:
Ensemble - model with initialized geometry
XY - training set
NPoints - training set size
Decay - weight decay coefficient, >=0.001
Restarts - restarts, >0.
WStep - stopping criterion, same as in MLPTrainLBFGS
MaxIts - stopping criterion, same as in MLPTrainLBFGS
OUTPUT PARAMETERS:
Ensemble - trained model
Info - return code:
* -8, if both WStep=0 and MaxIts=0
* -2, if there is a point with class number
outside of [0..NClasses-1].
* -1, if incorrect parameters was passed
(NPoints<0, Restarts<1).
* 2, if task has been solved.
Rep - training report.
OOBErrors - out-of-bag generalization error estimate
-- ALGLIB --
Copyright 17.02.2009 by Bochkanov Sergey
*************************************************************************/
public static void mlpebagginglbfgs(mlpe.mlpensemble ensemble,
double[,] xy,
int npoints,
double decay,
int restarts,
double wstep,
int maxits,
ref int info,
mlpreport rep,
mlpcvreport ooberrors)
{
info = 0;
mlpebagginginternal(ensemble, xy, npoints, decay, restarts, wstep, maxits, false, ref info, rep, ooberrors);
}
/*************************************************************************
* Neural network training using early stopping (base algorithm - L-BFGS with
* regularization).
*
* INPUT PARAMETERS:
* Network - neural network with initialized geometry
* TrnXY - training set
* TrnSize - training set size
* ValXY - validation set
* ValSize - validation set size
* Decay - weight decay constant, >=0.001
* Decay term 'Decay*||Weights||^2' is added to error
* function.
* If you don't know what Decay to choose, use 0.001.
* Restarts - number of restarts from random position, >0.
* If you don't know what Restarts to choose, use 2.
*
* OUTPUT PARAMETERS:
* Network - trained neural network.
* Info - return code:
* -2, if there is a point with class number
* outside of [0..NOut-1].
* -1, if wrong parameters specified
* (NPoints<0, Restarts<1, ...).
* 2, task has been solved, stopping criterion met -
* sufficiently small step size. Not expected (we
* use EARLY stopping) but possible and not an
* error.
* 6, task has been solved, stopping criterion met -
* increasing of validation set error.
* Rep - training report
*
* NOTE:
*
* Algorithm stops if validation set error increases for a long enough or
* step size is small enought (there are task where validation set may
* decrease for eternity). In any case solution returned corresponds to the
* minimum of validation set error.
*
* -- ALGLIB --
* Copyright 10.03.2009 by Bochkanov Sergey
*************************************************************************/
public static void mlptraines(ref mlpbase.multilayerperceptron network,
ref double[,] trnxy,
int trnsize,
ref double[,] valxy,
int valsize,
double decay,
int restarts,
ref int info,
ref mlpreport rep)
{
int i = 0;
int pass = 0;
int nin = 0;
int nout = 0;
int wcount = 0;
double[] w = new double[0];
double[] wbest = new double[0];
double e = 0;
double v = 0;
double ebest = 0;
double[] wfinal = new double[0];
double efinal = 0;
int itbest = 0;
minlbfgs.minlbfgsreport internalrep = new minlbfgs.minlbfgsreport();
minlbfgs.minlbfgsstate state = new minlbfgs.minlbfgsstate();
double wstep = 0;
int i_ = 0;
wstep = 0.001;
//
// Test inputs, parse flags, read network geometry
//
if (trnsize <= 0 | valsize <= 0 | restarts < 1 | (double)(decay) < (double)(0))
{
info = -1;
return;
}
mlpbase.mlpproperties(ref network, ref nin, ref nout, ref wcount);
if (mlpbase.mlpissoftmax(ref network))
{
for (i = 0; i <= trnsize - 1; i++)
{
if ((int)Math.Round(trnxy[i, nin]) < 0 | (int)Math.Round(trnxy[i, nin]) >= nout)
{
info = -2;
return;
}
}
for (i = 0; i <= valsize - 1; i++)
{
if ((int)Math.Round(valxy[i, nin]) < 0 | (int)Math.Round(valxy[i, nin]) >= nout)
{
info = -2;
return;
}
}
}
info = 2;
//
// Prepare
//
mlpbase.mlpinitpreprocessor(ref network, ref trnxy, trnsize);
w = new double[wcount - 1 + 1];
wbest = new double[wcount - 1 + 1];
wfinal = new double[wcount - 1 + 1];
efinal = AP.Math.MaxRealNumber;
for (i = 0; i <= wcount - 1; i++)
{
wfinal[i] = 0;
}
//
// Multiple starts
//
rep.ncholesky = 0;
rep.nhess = 0;
rep.ngrad = 0;
for (pass = 1; pass <= restarts; pass++)
{
//
// Process
//
mlpbase.mlprandomize(ref network);
ebest = mlpbase.mlperror(ref network, ref valxy, valsize);
for (i_ = 0; i_ <= wcount - 1; i_++)
{
wbest[i_] = network.weights[i_];
}
itbest = 0;
for (i_ = 0; i_ <= wcount - 1; i_++)
{
w[i_] = network.weights[i_];
}
minlbfgs.minlbfgscreate(wcount, Math.Min(wcount, 10), ref w, ref state);
minlbfgs.minlbfgssetcond(ref state, 0.0, 0.0, wstep, 0);
minlbfgs.minlbfgssetxrep(ref state, true);
while (minlbfgs.minlbfgsiteration(ref state))
{
//
// Calculate gradient
//
for (i_ = 0; i_ <= wcount - 1; i_++)
{
network.weights[i_] = state.x[i_];
}
mlpbase.mlpgradnbatch(ref network, ref trnxy, trnsize, ref state.f, ref state.g);
v = 0.0;
for (i_ = 0; i_ <= wcount - 1; i_++)
{
v += network.weights[i_] * network.weights[i_];
}
state.f = state.f + 0.5 * decay * v;
for (i_ = 0; i_ <= wcount - 1; i_++)
{
state.g[i_] = state.g[i_] + decay * network.weights[i_];
}
rep.ngrad = rep.ngrad + 1;
//
// Validation set
//
if (state.xupdated)
{
for (i_ = 0; i_ <= wcount - 1; i_++)
{
network.weights[i_] = w[i_];
}
e = mlpbase.mlperror(ref network, ref valxy, valsize);
if ((double)(e) < (double)(ebest))
{
ebest = e;
for (i_ = 0; i_ <= wcount - 1; i_++)
{
wbest[i_] = network.weights[i_];
}
itbest = internalrep.iterationscount;
}
if (internalrep.iterationscount > 30 & (double)(internalrep.iterationscount) > (double)(1.5 * itbest))
{
info = 6;
break;
}
}
}
minlbfgs.minlbfgsresults(ref state, ref w, ref internalrep);
//
// Compare with final answer
//
if ((double)(ebest) < (double)(efinal))
{
for (i_ = 0; i_ <= wcount - 1; i_++)
{
wfinal[i_] = wbest[i_];
}
efinal = ebest;
}
}
//
// The best network
//
for (i_ = 0; i_ <= wcount - 1; i_++)
{
network.weights[i_] = wfinal[i_];
}
}
/*************************************************************************
* Neural network training using modified Levenberg-Marquardt with exact
* Hessian calculation and regularization. Subroutine trains neural network
* with restarts from random positions. Algorithm is well suited for small
* and medium scale problems (hundreds of weights).
*
* INPUT PARAMETERS:
* Network - neural network with initialized geometry
* XY - training set
* NPoints - training set size
* Decay - weight decay constant, >=0.001
* Decay term 'Decay*||Weights||^2' is added to error
* function.
* If you don't know what Decay to choose, use 0.001.
* Restarts - number of restarts from random position, >0.
* If you don't know what Restarts to choose, use 2.
*
* OUTPUT PARAMETERS:
* Network - trained neural network.
* Info - return code:
* -9, if internal matrix inverse subroutine failed
* -2, if there is a point with class number
* outside of [0..NOut-1].
* -1, if wrong parameters specified
* (NPoints<0, Restarts<1).
* 2, if task has been solved.
* Rep - training report
*
* -- ALGLIB --
* Copyright 10.03.2009 by Bochkanov Sergey
*************************************************************************/
public static void mlptrainlm(ref mlpbase.multilayerperceptron network,
ref double[,] xy,
int npoints,
double decay,
int restarts,
ref int info,
ref mlpreport rep)
{
int nin = 0;
int nout = 0;
int wcount = 0;
double lmftol = 0;
double lmsteptol = 0;
int i = 0;
int k = 0;
double v = 0;
double e = 0;
double enew = 0;
double xnorm2 = 0;
double stepnorm = 0;
double[] g = new double[0];
double[] d = new double[0];
double[,] h = new double[0, 0];
double[,] hmod = new double[0, 0];
double[,] z = new double[0, 0];
bool spd = new bool();
double nu = 0;
double lambda = 0;
double lambdaup = 0;
double lambdadown = 0;
minlbfgs.minlbfgsreport internalrep = new minlbfgs.minlbfgsreport();
minlbfgs.minlbfgsstate state = new minlbfgs.minlbfgsstate();
double[] x = new double[0];
double[] y = new double[0];
double[] wbase = new double[0];
double[] wdir = new double[0];
double[] wt = new double[0];
double[] wx = new double[0];
int pass = 0;
double[] wbest = new double[0];
double ebest = 0;
int invinfo = 0;
matinv.matinvreport invrep = new matinv.matinvreport();
int solverinfo = 0;
densesolver.densesolverreport solverrep = new densesolver.densesolverreport();
int i_ = 0;
mlpbase.mlpproperties(ref network, ref nin, ref nout, ref wcount);
lambdaup = 10;
lambdadown = 0.3;
lmftol = 0.001;
lmsteptol = 0.001;
//
// Test for inputs
//
if (npoints <= 0 | restarts < 1)
{
info = -1;
return;
}
if (mlpbase.mlpissoftmax(ref network))
{
for (i = 0; i <= npoints - 1; i++)
{
if ((int)Math.Round(xy[i, nin]) < 0 | (int)Math.Round(xy[i, nin]) >= nout)
{
info = -2;
return;
}
}
}
decay = Math.Max(decay, mindecay);
info = 2;
//
// Initialize data
//
rep.ngrad = 0;
rep.nhess = 0;
rep.ncholesky = 0;
//
// General case.
// Prepare task and network. Allocate space.
//
mlpbase.mlpinitpreprocessor(ref network, ref xy, npoints);
g = new double[wcount - 1 + 1];
h = new double[wcount - 1 + 1, wcount - 1 + 1];
hmod = new double[wcount - 1 + 1, wcount - 1 + 1];
wbase = new double[wcount - 1 + 1];
wdir = new double[wcount - 1 + 1];
wbest = new double[wcount - 1 + 1];
wt = new double[wcount - 1 + 1];
wx = new double[wcount - 1 + 1];
ebest = AP.Math.MaxRealNumber;
//
// Multiple passes
//
for (pass = 1; pass <= restarts; pass++)
{
//
// Initialize weights
//
mlpbase.mlprandomize(ref network);
//
// First stage of the hybrid algorithm: LBFGS
//
for (i_ = 0; i_ <= wcount - 1; i_++)
{
wbase[i_] = network.weights[i_];
}
minlbfgs.minlbfgscreate(wcount, Math.Min(wcount, 5), ref wbase, ref state);
minlbfgs.minlbfgssetcond(ref state, 0, 0, 0, Math.Max(25, wcount));
while (minlbfgs.minlbfgsiteration(ref state))
{
//
// gradient
//
for (i_ = 0; i_ <= wcount - 1; i_++)
{
network.weights[i_] = state.x[i_];
}
mlpbase.mlpgradbatch(ref network, ref xy, npoints, ref state.f, ref state.g);
//
// weight decay
//
v = 0.0;
for (i_ = 0; i_ <= wcount - 1; i_++)
{
v += network.weights[i_] * network.weights[i_];
}
state.f = state.f + 0.5 * decay * v;
for (i_ = 0; i_ <= wcount - 1; i_++)
{
state.g[i_] = state.g[i_] + decay * network.weights[i_];
}
//
// next iteration
//
rep.ngrad = rep.ngrad + 1;
}
minlbfgs.minlbfgsresults(ref state, ref wbase, ref internalrep);
for (i_ = 0; i_ <= wcount - 1; i_++)
{
network.weights[i_] = wbase[i_];
}
//
// Second stage of the hybrid algorithm: LM
//
// Initialize H with identity matrix,
// G with gradient,
// E with regularized error.
//
mlpbase.mlphessianbatch(ref network, ref xy, npoints, ref e, ref g, ref h);
v = 0.0;
for (i_ = 0; i_ <= wcount - 1; i_++)
{
v += network.weights[i_] * network.weights[i_];
}
e = e + 0.5 * decay * v;
for (i_ = 0; i_ <= wcount - 1; i_++)
{
g[i_] = g[i_] + decay * network.weights[i_];
}
for (k = 0; k <= wcount - 1; k++)
{
h[k, k] = h[k, k] + decay;
}
rep.nhess = rep.nhess + 1;
lambda = 0.001;
nu = 2;
while (true)
{
//
// 1. HMod = H+lambda*I
// 2. Try to solve (H+Lambda*I)*dx = -g.
// Increase lambda if left part is not positive definite.
//
for (i = 0; i <= wcount - 1; i++)
{
for (i_ = 0; i_ <= wcount - 1; i_++)
{
hmod[i, i_] = h[i, i_];
}
hmod[i, i] = hmod[i, i] + lambda;
}
spd = trfac.spdmatrixcholesky(ref hmod, wcount, true);
rep.ncholesky = rep.ncholesky + 1;
if (!spd)
{
lambda = lambda * lambdaup * nu;
nu = nu * 2;
continue;
}
densesolver.spdmatrixcholeskysolve(ref hmod, wcount, true, ref g, ref solverinfo, ref solverrep, ref wdir);
if (solverinfo < 0)
{
lambda = lambda * lambdaup * nu;
nu = nu * 2;
continue;
}
for (i_ = 0; i_ <= wcount - 1; i_++)
{
wdir[i_] = -1 * wdir[i_];
}
//
// Lambda found.
// 1. Save old w in WBase
// 1. Test some stopping criterions
// 2. If error(w+wdir)>error(w), increase lambda
//
for (i_ = 0; i_ <= wcount - 1; i_++)
{
network.weights[i_] = network.weights[i_] + wdir[i_];
}
xnorm2 = 0.0;
for (i_ = 0; i_ <= wcount - 1; i_++)
{
xnorm2 += network.weights[i_] * network.weights[i_];
}
stepnorm = 0.0;
for (i_ = 0; i_ <= wcount - 1; i_++)
{
stepnorm += wdir[i_] * wdir[i_];
}
stepnorm = Math.Sqrt(stepnorm);
enew = mlpbase.mlperror(ref network, ref xy, npoints) + 0.5 * decay * xnorm2;
if ((double)(stepnorm) < (double)(lmsteptol * (1 + Math.Sqrt(xnorm2))))
{
break;
}
if ((double)(enew) > (double)(e))
{
lambda = lambda * lambdaup * nu;
nu = nu * 2;
continue;
}
//
// Optimize using inv(cholesky(H)) as preconditioner
//
matinv.rmatrixtrinverse(ref hmod, wcount, true, false, ref invinfo, ref invrep);
if (invinfo <= 0)
{
//
// if matrix can't be inverted then exit with errors
// TODO: make WCount steps in direction suggested by HMod
//
info = -9;
return;
}
for (i_ = 0; i_ <= wcount - 1; i_++)
{
wbase[i_] = network.weights[i_];
}
for (i = 0; i <= wcount - 1; i++)
{
wt[i] = 0;
}
minlbfgs.minlbfgscreatex(wcount, wcount, ref wt, 1, ref state);
minlbfgs.minlbfgssetcond(ref state, 0, 0, 0, 5);
while (minlbfgs.minlbfgsiteration(ref state))
{
//
// gradient
//
for (i = 0; i <= wcount - 1; i++)
{
v = 0.0;
for (i_ = i; i_ <= wcount - 1; i_++)
{
v += state.x[i_] * hmod[i, i_];
}
network.weights[i] = wbase[i] + v;
}
mlpbase.mlpgradbatch(ref network, ref xy, npoints, ref state.f, ref g);
for (i = 0; i <= wcount - 1; i++)
{
state.g[i] = 0;
}
for (i = 0; i <= wcount - 1; i++)
{
v = g[i];
for (i_ = i; i_ <= wcount - 1; i_++)
{
state.g[i_] = state.g[i_] + v * hmod[i, i_];
}
}
//
// weight decay
// grad(x'*x) = A'*(x0+A*t)
//
v = 0.0;
for (i_ = 0; i_ <= wcount - 1; i_++)
{
v += network.weights[i_] * network.weights[i_];
}
state.f = state.f + 0.5 * decay * v;
for (i = 0; i <= wcount - 1; i++)
{
v = decay * network.weights[i];
for (i_ = i; i_ <= wcount - 1; i_++)
{
state.g[i_] = state.g[i_] + v * hmod[i, i_];
}
}
//
// next iteration
//
rep.ngrad = rep.ngrad + 1;
}
minlbfgs.minlbfgsresults(ref state, ref wt, ref internalrep);
//
// Accept new position.
// Calculate Hessian
//
for (i = 0; i <= wcount - 1; i++)
{
v = 0.0;
for (i_ = i; i_ <= wcount - 1; i_++)
{
v += wt[i_] * hmod[i, i_];
}
network.weights[i] = wbase[i] + v;
}
mlpbase.mlphessianbatch(ref network, ref xy, npoints, ref e, ref g, ref h);
v = 0.0;
for (i_ = 0; i_ <= wcount - 1; i_++)
{
v += network.weights[i_] * network.weights[i_];
}
e = e + 0.5 * decay * v;
for (i_ = 0; i_ <= wcount - 1; i_++)
{
g[i_] = g[i_] + decay * network.weights[i_];
}
for (k = 0; k <= wcount - 1; k++)
{
h[k, k] = h[k, k] + decay;
}
rep.nhess = rep.nhess + 1;
//
// Update lambda
//
lambda = lambda * lambdadown;
nu = 2;
}
//
// update WBest
//
v = 0.0;
for (i_ = 0; i_ <= wcount - 1; i_++)
{
v += network.weights[i_] * network.weights[i_];
}
e = 0.5 * decay * v + mlpbase.mlperror(ref network, ref xy, npoints);
if ((double)(e) < (double)(ebest))
{
ebest = e;
for (i_ = 0; i_ <= wcount - 1; i_++)
{
wbest[i_] = network.weights[i_];
}
}
}
//
// copy WBest to output
//
for (i_ = 0; i_ <= wcount - 1; i_++)
{
network.weights[i_] = wbest[i_];
}
}
/*************************************************************************
* Neural network training using L-BFGS algorithm with regularization.
* Subroutine trains neural network with restarts from random positions.
* Algorithm is well suited for problems of any dimensionality (memory
* requirements and step complexity are linear by weights number).
*
* INPUT PARAMETERS:
* Network - neural network with initialized geometry
* XY - training set
* NPoints - training set size
* Decay - weight decay constant, >=0.001
* Decay term 'Decay*||Weights||^2' is added to error
* function.
* If you don't know what Decay to choose, use 0.001.
* Restarts - number of restarts from random position, >0.
* If you don't know what Restarts to choose, use 2.
* WStep - stopping criterion. Algorithm stops if step size is
* less than WStep. Recommended value - 0.01. Zero step
* size means stopping after MaxIts iterations.
* MaxIts - stopping criterion. Algorithm stops after MaxIts
* iterations (NOT gradient calculations). Zero MaxIts
* means stopping when step is sufficiently small.
*
* OUTPUT PARAMETERS:
* Network - trained neural network.
* Info - return code:
* -8, if both WStep=0 and MaxIts=0
* -2, if there is a point with class number
* outside of [0..NOut-1].
* -1, if wrong parameters specified
* (NPoints<0, Restarts<1).
* 2, if task has been solved.
* Rep - training report
*
* -- ALGLIB --
* Copyright 09.12.2007 by Bochkanov Sergey
*************************************************************************/
public static void mlptrainlbfgs(ref mlpbase.multilayerperceptron network,
ref double[,] xy,
int npoints,
double decay,
int restarts,
double wstep,
int maxits,
ref int info,
ref mlpreport rep)
{
int i = 0;
int pass = 0;
int nin = 0;
int nout = 0;
int wcount = 0;
double[] w = new double[0];
double[] wbest = new double[0];
double e = 0;
double v = 0;
double ebest = 0;
minlbfgs.minlbfgsreport internalrep = new minlbfgs.minlbfgsreport();
minlbfgs.minlbfgsstate state = new minlbfgs.minlbfgsstate();
int i_ = 0;
//
// Test inputs, parse flags, read network geometry
//
if ((double)(wstep) == (double)(0) & maxits == 0)
{
info = -8;
return;
}
if (npoints <= 0 | restarts < 1 | (double)(wstep) < (double)(0) | maxits < 0)
{
info = -1;
return;
}
mlpbase.mlpproperties(ref network, ref nin, ref nout, ref wcount);
if (mlpbase.mlpissoftmax(ref network))
{
for (i = 0; i <= npoints - 1; i++)
{
if ((int)Math.Round(xy[i, nin]) < 0 | (int)Math.Round(xy[i, nin]) >= nout)
{
info = -2;
return;
}
}
}
decay = Math.Max(decay, mindecay);
info = 2;
//
// Prepare
//
mlpbase.mlpinitpreprocessor(ref network, ref xy, npoints);
w = new double[wcount - 1 + 1];
wbest = new double[wcount - 1 + 1];
ebest = AP.Math.MaxRealNumber;
//
// Multiple starts
//
rep.ncholesky = 0;
rep.nhess = 0;
rep.ngrad = 0;
for (pass = 1; pass <= restarts; pass++)
{
//
// Process
//
mlpbase.mlprandomize(ref network);
for (i_ = 0; i_ <= wcount - 1; i_++)
{
w[i_] = network.weights[i_];
}
minlbfgs.minlbfgscreate(wcount, Math.Min(wcount, 10), ref w, ref state);
minlbfgs.minlbfgssetcond(ref state, 0.0, 0.0, wstep, maxits);
while (minlbfgs.minlbfgsiteration(ref state))
{
for (i_ = 0; i_ <= wcount - 1; i_++)
{
network.weights[i_] = state.x[i_];
}
mlpbase.mlpgradnbatch(ref network, ref xy, npoints, ref state.f, ref state.g);
v = 0.0;
for (i_ = 0; i_ <= wcount - 1; i_++)
{
v += network.weights[i_] * network.weights[i_];
}
state.f = state.f + 0.5 * decay * v;
for (i_ = 0; i_ <= wcount - 1; i_++)
{
state.g[i_] = state.g[i_] + decay * network.weights[i_];
}
rep.ngrad = rep.ngrad + 1;
}
minlbfgs.minlbfgsresults(ref state, ref w, ref internalrep);
for (i_ = 0; i_ <= wcount - 1; i_++)
{
network.weights[i_] = w[i_];
}
//
// Compare with best
//
v = 0.0;
for (i_ = 0; i_ <= wcount - 1; i_++)
{
v += network.weights[i_] * network.weights[i_];
}
e = mlpbase.mlperrorn(ref network, ref xy, npoints) + 0.5 * decay * v;
if ((double)(e) < (double)(ebest))
{
for (i_ = 0; i_ <= wcount - 1; i_++)
{
wbest[i_] = network.weights[i_];
}
ebest = e;
}
}
//
// The best network
//
for (i_ = 0; i_ <= wcount - 1; i_++)
{
network.weights[i_] = wbest[i_];
}
}
/*************************************************************************
This function trains neural network passed to this function, using current
dataset (one which was passed to MLPSetDataset() or MLPSetSparseDataset())
and current training settings. Training from NRestarts random starting
positions is performed, best network is chosen.
Training is performed using current training algorithm.
INPUT PARAMETERS:
S - trainer object;
Network - neural network. It must have same number of inputs and
output/classes as was specified during creation of the
trainer object;
TNetwork - the training neural network.
User may look weights in parameter Network while
continue training process.
It has architecture like Network. You have to copy or
create new network with architecture like Network.
State - created LBFGS optimizer;
NRestarts - number of restarts, >=0:
* NRestarts>0 means that specified number of random
restarts are performed, best network is chosen after
training
* NRestarts=0 means that current state of the network
is used for training.
TrnSubset - some subset from training set(it stores row's numbers),
used as trainig set;
TrnSubsetSize- size of subset(if TrnSubsetSize<0 - used full dataset);
when TrnSubsetSize=0, network is filled by zero value,
and ValSubset parameter is IGNORED;
ValSubset - some subset from training set(it stores row's numbers),
used as validation set;
ValSubsetSize- size of subset(if ValSubsetSize<0 - used full dataset);
when ValSubsetSize<>0 this mean that is used early
stopping training algorithm;
BufWBest - buffer for storing interim resuls (BufWBest[0:WCOunt-1]
it has be allocated by user);
BufWFinal - buffer for storing interim resuls(BufWFinal[0:WCOunt-1]
it has be allocated by user).
OUTPUT PARAMETERS:
Network - trained network;
Rep - training report.
NOTE: when no dataset was specified with MLPSetDataset/SetSparseDataset(),
network is filled by zero values. Same behavior for functions
MLPStartTraining and MLPContinueTraining.
NOTE: this method uses sum-of-squares error function for training.
-- ALGLIB --
Copyright 13.08.2012 by Bochkanov Sergey
*************************************************************************/
private static void mlptrainnetworkx(mlptrainer s,
mlpbase.multilayerperceptron network,
mlpbase.multilayerperceptron tnetwork,
minlbfgs.minlbfgsstate state,
int nrestarts,
int[] trnsubset,
int trnsubsetsize,
int[] valsubset,
int valsubsetsize,
double[] bufwbest,
double[] bufwfinal,
mlpreport rep)
{
mlpbase.modelerrors modrep = new mlpbase.modelerrors();
double eval = 0;
double v = 0;
double ebestcur = 0;
double efinal = 0;
int ngradbatch = 0;
int nin = 0;
int nout = 0;
int wcount = 0;
int twcount = 0;
int itbest = 0;
int itcnt = 0;
int ntype = 0;
int ttype = 0;
bool rndstart = new bool();
int pass = 0;
int i = 0;
int i_ = 0;
alglib.ap.assert(s.npoints>=0, "MLPTrainNetworkX: internal error - parameter S is not initialized or is spoiled(S.NPoints<0)");
if( s.rcpar )
{
ttype = 0;
}
else
{
ttype = 1;
}
if( !mlpbase.mlpissoftmax(network) )
{
ntype = 0;
}
else
{
ntype = 1;
}
alglib.ap.assert(ntype==ttype, "MLPTrainNetworkX: internal error - type of the resulting network is not similar to network type in trainer object");
if( !mlpbase.mlpissoftmax(tnetwork) )
{
ntype = 0;
}
else
{
ntype = 1;
}
alglib.ap.assert(ntype==ttype, "MLPTrainNetworkX: internal error - type of the training network is not similar to network type in trainer object");
mlpbase.mlpproperties(network, ref nin, ref nout, ref wcount);
alglib.ap.assert(s.nin==nin, "MLPTrainNetworkX: internal error - number of inputs in trainer is not equal to number of inputs in the network.");
alglib.ap.assert(s.nout==nout, "MLPTrainNetworkX: internal error - number of outputs in trainer is not equal to number of outputs in the network.");
mlpbase.mlpproperties(tnetwork, ref nin, ref nout, ref twcount);
alglib.ap.assert(s.nin==nin, "MLPTrainNetworkX: internal error - number of inputs in trainer is not equal to number of inputs in the training network.");
alglib.ap.assert(s.nout==nout, "MLPTrainNetworkX: internal error - number of outputs in trainer is not equal to number of outputs in the training network.");
alglib.ap.assert(twcount==wcount, "MLPTrainNetworkX: internal error - number of weights the resulting network is not equal to number of weights in the training network.");
alglib.ap.assert(nrestarts>=0, "MLPTrainNetworkX: internal error - NRestarts<0.");
alglib.ap.assert(alglib.ap.len(trnsubset)>=trnsubsetsize, "MLPTrainNetworkX: internal error - parameter TrnSubsetSize more than input subset size(Length(TrnSubset)<TrnSubsetSize)");
for(i=0; i<=trnsubsetsize-1; i++)
{
alglib.ap.assert(trnsubset[i]>=0 && trnsubset[i]<=s.npoints-1, "MLPTrainNetworkX: internal error - parameter TrnSubset contains incorrect index(TrnSubset[I]<0 or TrnSubset[I]>S.NPoints-1)");
}
alglib.ap.assert(alglib.ap.len(valsubset)>=valsubsetsize, "MLPTrainNetworkX: internal error - parameter ValSubsetSize more than input subset size(Length(ValSubset)<ValSubsetSize)");
for(i=0; i<=valsubsetsize-1; i++)
{
alglib.ap.assert(valsubset[i]>=0 && valsubset[i]<=s.npoints-1, "MLPTrainNetworkX: internal error - parameter ValSubset contains incorrect index(ValSubset[I]<0 or ValSubset[I]>S.NPoints-1)");
}
//
// Initialize parameter Rep
//
rep.relclserror = 0;
rep.avgce = 0;
rep.rmserror = 0;
rep.avgerror = 0;
rep.avgrelerror = 0;
rep.ngrad = 0;
rep.nhess = 0;
rep.ncholesky = 0;
if( ((s.datatype==0 || s.datatype==1) && s.npoints>0) && trnsubsetsize!=0 )
{
//
// Prepare
//
efinal = math.maxrealnumber;
if( nrestarts!=0 )
{
rndstart = true;
}
else
{
rndstart = false;
nrestarts = 1;
}
ngradbatch = 0;
eval = 0;
ebestcur = 0;
for(pass=1; pass<=nrestarts; pass++)
{
mlpstarttrainingx(s, network, tnetwork, state, rndstart, trnsubset, trnsubsetsize);
itbest = 0;
itcnt = 0;
if( s.datatype==0 )
{
ebestcur = mlpbase.mlperrorsubset(network, s.densexy, s.npoints, valsubset, valsubsetsize);
}
if( s.datatype==1 )
{
ebestcur = mlpbase.mlperrorsparsesubset(network, s.sparsexy, s.npoints, valsubset, valsubsetsize);
}
for(i_=0; i_<=wcount-1;i_++)
{
bufwbest[i_] = network.weights[i_];
}
while( mlpcontinuetrainingx(s, network, tnetwork, state, trnsubset, trnsubsetsize, ref ngradbatch) )
{
if( s.datatype==0 )
{
eval = mlpbase.mlperrorsubset(network, s.densexy, s.npoints, valsubset, valsubsetsize);
}
if( s.datatype==1 )
{
eval = mlpbase.mlperrorsparsesubset(network, s.sparsexy, s.npoints, valsubset, valsubsetsize);
}
if( (double)(eval)<=(double)(ebestcur) )
{
for(i_=0; i_<=wcount-1;i_++)
{
bufwbest[i_] = network.weights[i_];
}
ebestcur = eval;
itbest = itcnt;
}
if( itcnt>30 && (double)(itcnt)>(double)(1.5*itbest) )
{
break;
}
itcnt = itcnt+1;
}
for(i_=0; i_<=wcount-1;i_++)
{
network.weights[i_] = bufwbest[i_];
}
//
// Compare with final(the best) answer.
//
v = 0.0;
for(i_=0; i_<=wcount-1;i_++)
{
v += bufwbest[i_]*bufwbest[i_];
}
if( s.datatype==0 )
{
ebestcur = mlpbase.mlperrorsubset(network, s.densexy, s.npoints, trnsubset, trnsubsetsize)+0.5*s.decay*v;
}
if( s.datatype==1 )
{
ebestcur = mlpbase.mlperrorsparsesubset(network, s.sparsexy, s.npoints, trnsubset, trnsubsetsize)+0.5*s.decay*v;
}
if( (double)(ebestcur)<(double)(efinal) )
{
for(i_=0; i_<=wcount-1;i_++)
{
bufwfinal[i_] = bufwbest[i_];
}
efinal = ebestcur;
}
}
//
// Final network
//
for(i_=0; i_<=wcount-1;i_++)
{
network.weights[i_] = bufwfinal[i_];
}
rep.ngrad = ngradbatch;
}
else
{
for(i=0; i<=wcount-1; i++)
{
network.weights[i] = 0;
}
}
//
// Calculate errors.
//
if( s.datatype==0 )
{
mlpbase.mlpallerrorssubset(network, s.densexy, s.npoints, trnsubset, trnsubsetsize, modrep);
}
if( s.datatype==1 )
{
mlpbase.mlpallerrorssparsesubset(network, s.sparsexy, s.npoints, trnsubset, trnsubsetsize, modrep);
}
rep.relclserror = modrep.relclserror;
rep.avgce = modrep.avgce;
rep.rmserror = modrep.rmserror;
rep.avgerror = modrep.avgerror;
rep.avgrelerror = modrep.avgrelerror;
}
/*************************************************************************
This function trains neural network ensemble passed to this function using
current dataset and early stopping training algorithm. Each early stopping
round performs NRestarts random restarts (thus, EnsembleSize*NRestarts
training rounds is performed in total).
INPUT PARAMETERS:
S - trainer object;
Ensemble - neural network ensemble. It must have same number of
inputs and outputs/classes as was specified during
creation of the trainer object.
NRestarts - number of restarts, >=0:
* NRestarts>0 means that specified number of random
restarts are performed during each ES round;
* NRestarts=0 is silently replaced by 1.
OUTPUT PARAMETERS:
Ensemble - trained ensemble;
Rep - it contains all type of errors.
NOTE: when no dataset was specified with MLPSetDataset/SetSparseDataset(),
or single-point dataset was passed, ensemble is filled by zero
values.
NOTE: this method uses sum-of-squares error function for training.
-- ALGLIB --
Copyright 22.08.2012 by Bochkanov Sergey
*************************************************************************/
public static void mlptrainensemblees(mlptrainer s,
mlpe.mlpensemble ensemble,
int nrestarts,
mlpreport rep)
{
int pcount = 0;
mlpreport tmprep = new mlpreport();
int nin = 0;
int nout = 0;
int wcount = 0;
int ntype = 0;
int ttype = 0;
int i = 0;
int k = 0;
int i_ = 0;
int i1_ = 0;
alglib.ap.assert(s.npoints>=0, "MLPTrainEnsembleES: parameter S is not initialized or is spoiled(S.NPoints<0)");
if( !mlpe.mlpeissoftmax(ensemble) )
{
ntype = 0;
}
else
{
ntype = 1;
}
if( s.rcpar )
{
ttype = 0;
}
else
{
ttype = 1;
}
alglib.ap.assert(ntype==ttype, "MLPTrainEnsembleES: internal error - type of input network is not similar to network type in trainer object");
nin = mlpbase.mlpgetinputscount(ensemble.network);
alglib.ap.assert(s.nin==nin, "MLPTrainEnsembleES: number of inputs in trainer is not equal to number of inputs in ensemble network");
nout = mlpbase.mlpgetoutputscount(ensemble.network);
alglib.ap.assert(s.nout==nout, "MLPTrainEnsembleES: number of outputs in trainer is not equal to number of outputs in ensemble network");
alglib.ap.assert(nrestarts>=0, "MLPTrainEnsembleES: NRestarts<0.");
wcount = mlpbase.mlpgetweightscount(ensemble.network);
//
// Initialize parameter Rep
//
rep.relclserror = 0;
rep.avgce = 0;
rep.rmserror = 0;
rep.avgerror = 0;
rep.avgrelerror = 0;
rep.ngrad = 0;
rep.nhess = 0;
rep.ncholesky = 0;
//
// Allocate
//
if( mlpbase.mlpissoftmax(ensemble.network) )
{
pcount = nin;
}
else
{
pcount = nin+nout;
}
apserv.ivectorsetlengthatleast(ref s.subset, s.npoints);
apserv.ivectorsetlengthatleast(ref s.valsubset, s.npoints);
apserv.rvectorsetlengthatleast(ref s.wbest, wcount);
apserv.rvectorsetlengthatleast(ref s.wfinal, wcount);
//
// Create LBFGS optimizer
//
minlbfgs.minlbfgscreate(wcount, Math.Min(wcount, s.lbfgsfactor), ensemble.network.weights, s.tstate);
minlbfgs.minlbfgssetcond(s.tstate, 0.0, 0.0, s.wstep, s.maxits);
minlbfgs.minlbfgssetxrep(s.tstate, true);
mlpbase.mlpcopy(ensemble.network, s.tnetwork);
//
// Train networks
//
if( (s.datatype==0 || s.datatype==1) && s.npoints>1 )
{
for(k=0; k<=ensemble.ensemblesize-1; k++)
{
//
// Split set
//
do
{
s.subsetsize = 0;
s.valsubsetsize = 0;
for(i=0; i<=s.npoints-1; i++)
{
if( (double)(math.randomreal())<(double)(0.66) )
{
//
// Assign sample to training set
//
s.subset[s.subsetsize] = i;
s.subsetsize = s.subsetsize+1;
}
else
{
//
// Assign sample to validation set
//
s.valsubset[s.valsubsetsize] = i;
s.valsubsetsize = s.valsubsetsize+1;
}
}
}
while( !(s.subsetsize!=0 && s.valsubsetsize!=0) );
//
// Train
//
mlptrainnetworkx(s, ensemble.network, s.tnetwork, s.tstate, nrestarts, s.subset, s.subsetsize, s.valsubset, s.valsubsetsize, s.wbest, s.wfinal, tmprep);
rep.ngrad = rep.ngrad+tmprep.ngrad;
//
// Save results
//
i1_ = (0) - (k*wcount);
for(i_=k*wcount; i_<=(k+1)*wcount-1;i_++)
{
ensemble.weights[i_] = ensemble.network.weights[i_+i1_];
}
i1_ = (0) - (k*pcount);
for(i_=k*pcount; i_<=(k+1)*pcount-1;i_++)
{
ensemble.columnmeans[i_] = ensemble.network.columnmeans[i_+i1_];
}
i1_ = (0) - (k*pcount);
for(i_=k*pcount; i_<=(k+1)*pcount-1;i_++)
{
ensemble.columnsigmas[i_] = ensemble.network.columnsigmas[i_+i1_];
}
}
}
else
{
for(i=0; i<=ensemble.ensemblesize*wcount-1; i++)
{
ensemble.network.weights[i] = 0.0;
ensemble.columnmeans[i] = 0.0;
ensemble.columnsigmas[i] = 1.0;
}
}
//
// Calculate errors.
//
if( s.datatype==0 )
{
mlpe.mlpeallerrors(ensemble, s.densexy, s.npoints, ref rep.relclserror, ref rep.avgce, ref rep.rmserror, ref rep.avgerror, ref rep.avgrelerror);
}
if( s.datatype==1 )
{
mlpe.mlpeallerrorssparse(ensemble, s.sparsexy, s.npoints, ref rep.relclserror, ref rep.avgce, ref rep.rmserror, ref rep.avgerror, ref rep.avgrelerror);
}
}
/*************************************************************************
Single-threaded stub. HPC ALGLIB replaces it by multithreaded code.
*************************************************************************/
public static void _pexec_mlptrainnetwork(mlptrainer s,
mlpbase.multilayerperceptron network,
int nrestarts,
mlpreport rep)
{
mlptrainnetwork(s,network,nrestarts,rep);
}
/*************************************************************************
* Internal cross-validation subroutine
*************************************************************************/
private static void mlpkfoldcvgeneral(ref mlpbase.multilayerperceptron n,
ref double[,] xy,
int npoints,
double decay,
int restarts,
int foldscount,
bool lmalgorithm,
double wstep,
int maxits,
ref int info,
ref mlpreport rep,
ref mlpcvreport cvrep)
{
int i = 0;
int fold = 0;
int j = 0;
int k = 0;
mlpbase.multilayerperceptron network = new mlpbase.multilayerperceptron();
int nin = 0;
int nout = 0;
int rowlen = 0;
int wcount = 0;
int nclasses = 0;
int tssize = 0;
int cvssize = 0;
double[,] cvset = new double[0, 0];
double[,] testset = new double[0, 0];
int[] folds = new int[0];
int relcnt = 0;
mlpreport internalrep = new mlpreport();
double[] x = new double[0];
double[] y = new double[0];
int i_ = 0;
//
// Read network geometry, test parameters
//
mlpbase.mlpproperties(ref n, ref nin, ref nout, ref wcount);
if (mlpbase.mlpissoftmax(ref n))
{
nclasses = nout;
rowlen = nin + 1;
}
else
{
nclasses = -nout;
rowlen = nin + nout;
}
if (npoints <= 0 | foldscount <2 | foldscount> npoints)
{
info = -1;
return;
}
mlpbase.mlpcopy(ref n, ref network);
//
// K-fold out cross-validation.
// First, estimate generalization error
//
testset = new double[npoints - 1 + 1, rowlen - 1 + 1];
cvset = new double[npoints - 1 + 1, rowlen - 1 + 1];
x = new double[nin - 1 + 1];
y = new double[nout - 1 + 1];
mlpkfoldsplit(ref xy, npoints, nclasses, foldscount, false, ref folds);
cvrep.relclserror = 0;
cvrep.avgce = 0;
cvrep.rmserror = 0;
cvrep.avgerror = 0;
cvrep.avgrelerror = 0;
rep.ngrad = 0;
rep.nhess = 0;
rep.ncholesky = 0;
relcnt = 0;
for (fold = 0; fold <= foldscount - 1; fold++)
{
//
// Separate set
//
tssize = 0;
cvssize = 0;
for (i = 0; i <= npoints - 1; i++)
{
if (folds[i] == fold)
{
for (i_ = 0; i_ <= rowlen - 1; i_++)
{
testset[tssize, i_] = xy[i, i_];
}
tssize = tssize + 1;
}
else
{
for (i_ = 0; i_ <= rowlen - 1; i_++)
{
cvset[cvssize, i_] = xy[i, i_];
}
cvssize = cvssize + 1;
}
}
//
// Train on CV training set
//
if (lmalgorithm)
{
mlptrainlm(ref network, ref cvset, cvssize, decay, restarts, ref info, ref internalrep);
}
else
{
mlptrainlbfgs(ref network, ref cvset, cvssize, decay, restarts, wstep, maxits, ref info, ref internalrep);
}
if (info < 0)
{
cvrep.relclserror = 0;
cvrep.avgce = 0;
cvrep.rmserror = 0;
cvrep.avgerror = 0;
cvrep.avgrelerror = 0;
return;
}
rep.ngrad = rep.ngrad + internalrep.ngrad;
rep.nhess = rep.nhess + internalrep.nhess;
rep.ncholesky = rep.ncholesky + internalrep.ncholesky;
//
// Estimate error using CV test set
//
if (mlpbase.mlpissoftmax(ref network))
{
//
// classification-only code
//
cvrep.relclserror = cvrep.relclserror + mlpbase.mlpclserror(ref network, ref testset, tssize);
cvrep.avgce = cvrep.avgce + mlpbase.mlperrorn(ref network, ref testset, tssize);
}
for (i = 0; i <= tssize - 1; i++)
{
for (i_ = 0; i_ <= nin - 1; i_++)
{
x[i_] = testset[i, i_];
}
mlpbase.mlpprocess(ref network, ref x, ref y);
if (mlpbase.mlpissoftmax(ref network))
{
//
// Classification-specific code
//
k = (int)Math.Round(testset[i, nin]);
for (j = 0; j <= nout - 1; j++)
{
if (j == k)
{
cvrep.rmserror = cvrep.rmserror + AP.Math.Sqr(y[j] - 1);
cvrep.avgerror = cvrep.avgerror + Math.Abs(y[j] - 1);
cvrep.avgrelerror = cvrep.avgrelerror + Math.Abs(y[j] - 1);
relcnt = relcnt + 1;
}
else
{
cvrep.rmserror = cvrep.rmserror + AP.Math.Sqr(y[j]);
cvrep.avgerror = cvrep.avgerror + Math.Abs(y[j]);
}
}
}
else
{
//
// Regression-specific code
//
for (j = 0; j <= nout - 1; j++)
{
cvrep.rmserror = cvrep.rmserror + AP.Math.Sqr(y[j] - testset[i, nin + j]);
cvrep.avgerror = cvrep.avgerror + Math.Abs(y[j] - testset[i, nin + j]);
if ((double)(testset[i, nin + j]) != (double)(0))
{
cvrep.avgrelerror = cvrep.avgrelerror + Math.Abs((y[j] - testset[i, nin + j]) / testset[i, nin + j]);
relcnt = relcnt + 1;
}
}
}
}
}
if (mlpbase.mlpissoftmax(ref network))
{
cvrep.relclserror = cvrep.relclserror / npoints;
cvrep.avgce = cvrep.avgce / (Math.Log(2) * npoints);
}
cvrep.rmserror = Math.Sqrt(cvrep.rmserror / (npoints * nout));
cvrep.avgerror = cvrep.avgerror / (npoints * nout);
cvrep.avgrelerror = cvrep.avgrelerror / relcnt;
info = 1;
}
/*************************************************************************
Training neural networks ensemble using bootstrap aggregating (bagging).
Modified Levenberg-Marquardt algorithm is used as base training method.
INPUT PARAMETERS:
Ensemble - model with initialized geometry
XY - training set
NPoints - training set size
Decay - weight decay coefficient, >=0.001
Restarts - restarts, >0.
OUTPUT PARAMETERS:
Ensemble - trained model
Info - return code:
* -2, if there is a point with class number
outside of [0..NClasses-1].
* -1, if incorrect parameters was passed
(NPoints<0, Restarts<1).
* 2, if task has been solved.
Rep - training report.
OOBErrors - out-of-bag generalization error estimate
-- ALGLIB --
Copyright 17.02.2009 by Bochkanov Sergey
*************************************************************************/
public static void mlpebagginglm(mlpe.mlpensemble ensemble,
double[,] xy,
int npoints,
double decay,
int restarts,
ref int info,
mlpreport rep,
mlpcvreport ooberrors)
{
info = 0;
mlpebagginginternal(ensemble, xy, npoints, decay, restarts, 0.0, 0, true, ref info, rep, ooberrors);
}
public override alglib.apobject make_copy()
{
mlpreport _result = new mlpreport();
_result.relclserror = relclserror;
_result.avgce = avgce;
_result.rmserror = rmserror;
_result.avgerror = avgerror;
_result.avgrelerror = avgrelerror;
_result.ngrad = ngrad;
_result.nhess = nhess;
_result.ncholesky = ncholesky;
return _result;
}
/*************************************************************************
Training neural networks ensemble using early stopping.
INPUT PARAMETERS:
Ensemble - model with initialized geometry
XY - training set
NPoints - training set size
Decay - weight decay coefficient, >=0.001
Restarts - restarts, >0.
OUTPUT PARAMETERS:
Ensemble - trained model
Info - return code:
* -2, if there is a point with class number
outside of [0..NClasses-1].
* -1, if incorrect parameters was passed
(NPoints<0, Restarts<1).
* 6, if task has been solved.
Rep - training report.
OOBErrors - out-of-bag generalization error estimate
-- ALGLIB --
Copyright 10.03.2009 by Bochkanov Sergey
*************************************************************************/
public static void mlpetraines(mlpe.mlpensemble ensemble,
double[,] xy,
int npoints,
double decay,
int restarts,
ref int info,
mlpreport rep)
{
int i = 0;
int k = 0;
int ccount = 0;
int pcount = 0;
double[,] trnxy = new double[0,0];
double[,] valxy = new double[0,0];
int trnsize = 0;
int valsize = 0;
int tmpinfo = 0;
mlpreport tmprep = new mlpreport();
mlpbase.modelerrors moderr = new mlpbase.modelerrors();
int nin = 0;
int nout = 0;
int wcount = 0;
int i_ = 0;
int i1_ = 0;
info = 0;
nin = mlpbase.mlpgetinputscount(ensemble.network);
nout = mlpbase.mlpgetoutputscount(ensemble.network);
wcount = mlpbase.mlpgetweightscount(ensemble.network);
if( (npoints<2 || restarts<1) || (double)(decay)<(double)(0) )
{
info = -1;
return;
}
if( mlpbase.mlpissoftmax(ensemble.network) )
{
for(i=0; i<=npoints-1; i++)
{
if( (int)Math.Round(xy[i,nin])<0 || (int)Math.Round(xy[i,nin])>=nout )
{
info = -2;
return;
}
}
}
info = 6;
//
// allocate
//
if( mlpbase.mlpissoftmax(ensemble.network) )
{
ccount = nin+1;
pcount = nin;
}
else
{
ccount = nin+nout;
pcount = nin+nout;
}
trnxy = new double[npoints, ccount];
valxy = new double[npoints, ccount];
rep.ngrad = 0;
rep.nhess = 0;
rep.ncholesky = 0;
//
// train networks
//
for(k=0; k<=ensemble.ensemblesize-1; k++)
{
//
// Split set
//
do
{
trnsize = 0;
valsize = 0;
for(i=0; i<=npoints-1; i++)
{
if( (double)(math.randomreal())<(double)(0.66) )
{
//
// Assign sample to training set
//
for(i_=0; i_<=ccount-1;i_++)
{
trnxy[trnsize,i_] = xy[i,i_];
}
trnsize = trnsize+1;
}
else
{
//
// Assign sample to validation set
//
for(i_=0; i_<=ccount-1;i_++)
{
valxy[valsize,i_] = xy[i,i_];
}
valsize = valsize+1;
}
}
}
while( !(trnsize!=0 && valsize!=0) );
//
// Train
//
mlptraines(ensemble.network, trnxy, trnsize, valxy, valsize, decay, restarts, ref tmpinfo, tmprep);
if( tmpinfo<0 )
{
info = tmpinfo;
return;
}
//
// save results
//
i1_ = (0) - (k*wcount);
for(i_=k*wcount; i_<=(k+1)*wcount-1;i_++)
{
ensemble.weights[i_] = ensemble.network.weights[i_+i1_];
}
i1_ = (0) - (k*pcount);
for(i_=k*pcount; i_<=(k+1)*pcount-1;i_++)
{
ensemble.columnmeans[i_] = ensemble.network.columnmeans[i_+i1_];
}
i1_ = (0) - (k*pcount);
for(i_=k*pcount; i_<=(k+1)*pcount-1;i_++)
{
ensemble.columnsigmas[i_] = ensemble.network.columnsigmas[i_+i1_];
}
rep.ngrad = rep.ngrad+tmprep.ngrad;
rep.nhess = rep.nhess+tmprep.nhess;
rep.ncholesky = rep.ncholesky+tmprep.ncholesky;
}
mlpe.mlpeallerrorsx(ensemble, xy, ensemble.network.dummysxy, npoints, 0, ensemble.network.dummyidx, 0, npoints, 0, ensemble.network.buf, moderr);
rep.relclserror = moderr.relclserror;
rep.avgce = moderr.avgce;
rep.rmserror = moderr.rmserror;
rep.avgerror = moderr.avgerror;
rep.avgrelerror = moderr.avgrelerror;
}
public override void init()
{
trnsubset = new int[0];
valsubset = new int[0];
mlpsessions = new alglib.smp.shared_pool();
mlprep = new mlpreport();
network = new mlpbase.multilayerperceptron();
}
/*************************************************************************
Single-threaded stub. HPC ALGLIB replaces it by multithreaded code.
*************************************************************************/
public static void _pexec_mlptrainensemblees(mlptrainer s,
mlpe.mlpensemble ensemble,
int nrestarts,
mlpreport rep)
{
mlptrainensemblees(s,ensemble,nrestarts,rep);
}
public override void init()
{
network = new mlpbase.multilayerperceptron();
rep = new mlpreport();
subset = new int[0];
xyrow = new double[0];
y = new double[0];
trnpool = new alglib.smp.shared_pool();
}
/*************************************************************************
Internal bagging subroutine.
-- ALGLIB --
Copyright 19.02.2009 by Bochkanov Sergey
*************************************************************************/
private static void mlpebagginginternal(mlpe.mlpensemble ensemble,
double[,] xy,
int npoints,
double decay,
int restarts,
double wstep,
int maxits,
bool lmalgorithm,
ref int info,
mlpreport rep,
mlpcvreport ooberrors)
{
double[,] xys = new double[0,0];
bool[] s = new bool[0];
double[,] oobbuf = new double[0,0];
int[] oobcntbuf = new int[0];
double[] x = new double[0];
double[] y = new double[0];
double[] dy = new double[0];
double[] dsbuf = new double[0];
int ccnt = 0;
int pcnt = 0;
int i = 0;
int j = 0;
int k = 0;
double v = 0;
mlpreport tmprep = new mlpreport();
int nin = 0;
int nout = 0;
int wcount = 0;
hqrnd.hqrndstate rs = new hqrnd.hqrndstate();
int i_ = 0;
int i1_ = 0;
info = 0;
nin = mlpbase.mlpgetinputscount(ensemble.network);
nout = mlpbase.mlpgetoutputscount(ensemble.network);
wcount = mlpbase.mlpgetweightscount(ensemble.network);
//
// Test for inputs
//
if( (!lmalgorithm && (double)(wstep)==(double)(0)) && maxits==0 )
{
info = -8;
return;
}
if( ((npoints<=0 || restarts<1) || (double)(wstep)<(double)(0)) || maxits<0 )
{
info = -1;
return;
}
if( mlpbase.mlpissoftmax(ensemble.network) )
{
for(i=0; i<=npoints-1; i++)
{
if( (int)Math.Round(xy[i,nin])<0 || (int)Math.Round(xy[i,nin])>=nout )
{
info = -2;
return;
}
}
}
//
// allocate temporaries
//
info = 2;
rep.ngrad = 0;
rep.nhess = 0;
rep.ncholesky = 0;
ooberrors.relclserror = 0;
ooberrors.avgce = 0;
ooberrors.rmserror = 0;
ooberrors.avgerror = 0;
ooberrors.avgrelerror = 0;
if( mlpbase.mlpissoftmax(ensemble.network) )
{
ccnt = nin+1;
pcnt = nin;
}
else
{
ccnt = nin+nout;
pcnt = nin+nout;
}
xys = new double[npoints, ccnt];
s = new bool[npoints];
oobbuf = new double[npoints, nout];
oobcntbuf = new int[npoints];
x = new double[nin];
y = new double[nout];
if( mlpbase.mlpissoftmax(ensemble.network) )
{
dy = new double[1];
}
else
{
dy = new double[nout];
}
for(i=0; i<=npoints-1; i++)
{
for(j=0; j<=nout-1; j++)
{
oobbuf[i,j] = 0;
}
}
for(i=0; i<=npoints-1; i++)
{
oobcntbuf[i] = 0;
}
//
// main bagging cycle
//
hqrnd.hqrndrandomize(rs);
for(k=0; k<=ensemble.ensemblesize-1; k++)
{
//
// prepare dataset
//
for(i=0; i<=npoints-1; i++)
{
s[i] = false;
}
for(i=0; i<=npoints-1; i++)
{
j = hqrnd.hqrnduniformi(rs, npoints);
s[j] = true;
for(i_=0; i_<=ccnt-1;i_++)
{
xys[i,i_] = xy[j,i_];
}
}
//
// train
//
if( lmalgorithm )
{
mlptrainlm(ensemble.network, xys, npoints, decay, restarts, ref info, tmprep);
}
else
{
mlptrainlbfgs(ensemble.network, xys, npoints, decay, restarts, wstep, maxits, ref info, tmprep);
}
if( info<0 )
{
return;
}
//
// save results
//
rep.ngrad = rep.ngrad+tmprep.ngrad;
rep.nhess = rep.nhess+tmprep.nhess;
rep.ncholesky = rep.ncholesky+tmprep.ncholesky;
i1_ = (0) - (k*wcount);
for(i_=k*wcount; i_<=(k+1)*wcount-1;i_++)
{
ensemble.weights[i_] = ensemble.network.weights[i_+i1_];
}
i1_ = (0) - (k*pcnt);
for(i_=k*pcnt; i_<=(k+1)*pcnt-1;i_++)
{
ensemble.columnmeans[i_] = ensemble.network.columnmeans[i_+i1_];
}
i1_ = (0) - (k*pcnt);
for(i_=k*pcnt; i_<=(k+1)*pcnt-1;i_++)
{
ensemble.columnsigmas[i_] = ensemble.network.columnsigmas[i_+i1_];
}
//
// OOB estimates
//
for(i=0; i<=npoints-1; i++)
{
if( !s[i] )
{
for(i_=0; i_<=nin-1;i_++)
{
x[i_] = xy[i,i_];
}
mlpbase.mlpprocess(ensemble.network, x, ref y);
for(i_=0; i_<=nout-1;i_++)
{
oobbuf[i,i_] = oobbuf[i,i_] + y[i_];
}
oobcntbuf[i] = oobcntbuf[i]+1;
}
}
}
//
// OOB estimates
//
if( mlpbase.mlpissoftmax(ensemble.network) )
{
bdss.dserrallocate(nout, ref dsbuf);
}
else
{
bdss.dserrallocate(-nout, ref dsbuf);
}
for(i=0; i<=npoints-1; i++)
{
if( oobcntbuf[i]!=0 )
{
v = (double)1/(double)oobcntbuf[i];
for(i_=0; i_<=nout-1;i_++)
{
y[i_] = v*oobbuf[i,i_];
}
if( mlpbase.mlpissoftmax(ensemble.network) )
{
dy[0] = xy[i,nin];
}
else
{
i1_ = (nin) - (0);
for(i_=0; i_<=nout-1;i_++)
{
dy[i_] = v*xy[i,i_+i1_];
}
}
bdss.dserraccumulate(ref dsbuf, y, dy);
}
}
bdss.dserrfinish(ref dsbuf);
ooberrors.relclserror = dsbuf[0];
ooberrors.avgce = dsbuf[1];
ooberrors.rmserror = dsbuf[2];
ooberrors.avgerror = dsbuf[3];
ooberrors.avgrelerror = dsbuf[4];
}
/*************************************************************************
This function estimates generalization error using cross-validation on the
current dataset with current training settings.
FOR USERS OF COMMERCIAL EDITION:
! Commercial version of ALGLIB includes two important improvements of
! this function:
! * multicore support (C++ and C# computational cores)
! * SSE support (C++ computational core)
!
! Second improvement gives constant speedup (2-3X). First improvement
! gives close-to-linear speedup on multicore systems. Following
! operations can be executed in parallel:
! * FoldsCount cross-validation rounds (always)
! * NRestarts training sessions performed within each of
! cross-validation rounds (if NRestarts>1)
! * gradient calculation over large dataset (if dataset is large enough)
!
! In order to use multicore features you have to:
! * use commercial version of ALGLIB
! * call this function with "smp_" prefix, which indicates that
! multicore code will be used (for multicore support)
!
! In order to use SSE features you have to:
! * use commercial version of ALGLIB on Intel processors
! * use C++ computational core
!
! This note is given for users of commercial edition; if you use GPL
! edition, you still will be able to call smp-version of this function,
! but all computations will be done serially.
!
! We recommend you to carefully read ALGLIB Reference Manual, section
! called 'SMP support', before using parallel version of this function.
INPUT PARAMETERS:
S - trainer object
Network - neural network. It must have same number of inputs and
output/classes as was specified during creation of the
trainer object. Network is not changed during cross-
validation and is not trained - it is used only as
representative of its architecture. I.e., we estimate
generalization properties of ARCHITECTURE, not some
specific network.
NRestarts - number of restarts, >=0:
* NRestarts>0 means that for each cross-validation
round specified number of random restarts is
performed, with best network being chosen after
training.
* NRestarts=0 is same as NRestarts=1
FoldsCount - number of folds in k-fold cross-validation:
* 2<=FoldsCount<=size of dataset
* recommended value: 10.
* values larger than dataset size will be silently
truncated down to dataset size
OUTPUT PARAMETERS:
Rep - structure which contains cross-validation estimates:
* Rep.RelCLSError - fraction of misclassified cases.
* Rep.AvgCE - acerage cross-entropy
* Rep.RMSError - root-mean-square error
* Rep.AvgError - average error
* Rep.AvgRelError - average relative error
NOTE: when no dataset was specified with MLPSetDataset/SetSparseDataset(),
or subset with only one point was given, zeros are returned as
estimates.
NOTE: this method performs FoldsCount cross-validation rounds, each one
with NRestarts random starts. Thus, FoldsCount*NRestarts networks
are trained in total.
NOTE: Rep.RelCLSError/Rep.AvgCE are zero on regression problems.
NOTE: on classification problems Rep.RMSError/Rep.AvgError/Rep.AvgRelError
contain errors in prediction of posterior probabilities.
-- ALGLIB --
Copyright 23.07.2012 by Bochkanov Sergey
*************************************************************************/
public static void mlpkfoldcv(mlptrainer s,
mlpbase.multilayerperceptron network,
int nrestarts,
int foldscount,
mlpreport rep)
{
alglib.smp.shared_pool pooldatacv = new alglib.smp.shared_pool();
mlpparallelizationcv datacv = new mlpparallelizationcv();
mlpparallelizationcv sdatacv = null;
double[,] cvy = new double[0,0];
int[] folds = new int[0];
double[] buf = new double[0];
double[] dy = new double[0];
int nin = 0;
int nout = 0;
int wcount = 0;
int rowsize = 0;
int ntype = 0;
int ttype = 0;
int i = 0;
int j = 0;
int k = 0;
hqrnd.hqrndstate rs = new hqrnd.hqrndstate();
int i_ = 0;
int i1_ = 0;
if( !mlpbase.mlpissoftmax(network) )
{
ntype = 0;
}
else
{
ntype = 1;
}
if( s.rcpar )
{
ttype = 0;
}
else
{
ttype = 1;
}
alglib.ap.assert(ntype==ttype, "MLPKFoldCV: type of input network is not similar to network type in trainer object");
alglib.ap.assert(s.npoints>=0, "MLPKFoldCV: possible trainer S is not initialized(S.NPoints<0)");
mlpbase.mlpproperties(network, ref nin, ref nout, ref wcount);
alglib.ap.assert(s.nin==nin, "MLPKFoldCV: number of inputs in trainer is not equal to number of inputs in network");
alglib.ap.assert(s.nout==nout, "MLPKFoldCV: number of outputs in trainer is not equal to number of outputs in network");
alglib.ap.assert(nrestarts>=0, "MLPKFoldCV: NRestarts<0");
alglib.ap.assert(foldscount>=2, "MLPKFoldCV: FoldsCount<2");
if( foldscount>s.npoints )
{
foldscount = s.npoints;
}
rep.relclserror = 0;
rep.avgce = 0;
rep.rmserror = 0;
rep.avgerror = 0;
rep.avgrelerror = 0;
hqrnd.hqrndrandomize(rs);
rep.ngrad = 0;
rep.nhess = 0;
rep.ncholesky = 0;
if( s.npoints==0 || s.npoints==1 )
{
return;
}
//
// Read network geometry, test parameters
//
if( s.rcpar )
{
rowsize = nin+nout;
dy = new double[nout];
bdss.dserrallocate(-nout, ref buf);
}
else
{
rowsize = nin+1;
dy = new double[1];
bdss.dserrallocate(nout, ref buf);
}
//
// Folds
//
folds = new int[s.npoints];
for(i=0; i<=s.npoints-1; i++)
{
folds[i] = i*foldscount/s.npoints;
}
for(i=0; i<=s.npoints-2; i++)
{
j = i+hqrnd.hqrnduniformi(rs, s.npoints-i);
if( j!=i )
{
k = folds[i];
folds[i] = folds[j];
folds[j] = k;
}
}
cvy = new double[s.npoints, nout];
//
// Initialize SEED-value for shared pool
//
datacv.ngrad = 0;
mlpbase.mlpcopy(network, datacv.network);
datacv.subset = new int[s.npoints];
datacv.xyrow = new double[rowsize];
datacv.y = new double[nout];
//
// Create shared pool
//
alglib.smp.ae_shared_pool_set_seed(pooldatacv, datacv);
//
// Parallelization
//
mthreadcv(s, rowsize, nrestarts, folds, 0, foldscount, cvy, pooldatacv);
//
// Calculate value for NGrad
//
alglib.smp.ae_shared_pool_first_recycled(pooldatacv, ref sdatacv);
while( sdatacv!=null )
{
rep.ngrad = rep.ngrad+sdatacv.ngrad;
alglib.smp.ae_shared_pool_next_recycled(pooldatacv, ref sdatacv);
}
//
// Connect of results and calculate cross-validation error
//
for(i=0; i<=s.npoints-1; i++)
{
if( s.datatype==0 )
{
for(i_=0; i_<=rowsize-1;i_++)
{
datacv.xyrow[i_] = s.densexy[i,i_];
}
}
if( s.datatype==1 )
{
sparse.sparsegetrow(s.sparsexy, i, ref datacv.xyrow);
}
for(i_=0; i_<=nout-1;i_++)
{
datacv.y[i_] = cvy[i,i_];
}
if( s.rcpar )
{
i1_ = (nin) - (0);
for(i_=0; i_<=nout-1;i_++)
{
dy[i_] = datacv.xyrow[i_+i1_];
}
}
else
{
dy[0] = datacv.xyrow[nin];
}
bdss.dserraccumulate(ref buf, datacv.y, dy);
}
bdss.dserrfinish(ref buf);
rep.relclserror = buf[0];
rep.avgce = buf[1];
rep.rmserror = buf[2];
rep.avgerror = buf[3];
rep.avgrelerror = buf[4];
}
/*************************************************************************
Neural network training using L-BFGS algorithm with regularization.
Subroutine trains neural network with restarts from random positions.
Algorithm is well suited for problems of any dimensionality (memory
requirements and step complexity are linear by weights number).
INPUT PARAMETERS:
Network - neural network with initialized geometry
XY - training set
NPoints - training set size
Decay - weight decay constant, >=0.001
Decay term 'Decay*||Weights||^2' is added to error
function.
If you don't know what Decay to choose, use 0.001.
Restarts - number of restarts from random position, >0.
If you don't know what Restarts to choose, use 2.
WStep - stopping criterion. Algorithm stops if step size is
less than WStep. Recommended value - 0.01. Zero step
size means stopping after MaxIts iterations.
MaxIts - stopping criterion. Algorithm stops after MaxIts
iterations (NOT gradient calculations). Zero MaxIts
means stopping when step is sufficiently small.
OUTPUT PARAMETERS:
Network - trained neural network.
Info - return code:
* -8, if both WStep=0 and MaxIts=0
* -2, if there is a point with class number
outside of [0..NOut-1].
* -1, if wrong parameters specified
(NPoints<0, Restarts<1).
* 2, if task has been solved.
Rep - training report
-- ALGLIB --
Copyright 09.12.2007 by Bochkanov Sergey
*************************************************************************/
public static void mlptrainlbfgs(mlpbase.multilayerperceptron network,
double[,] xy,
int npoints,
double decay,
int restarts,
double wstep,
int maxits,
ref int info,
mlpreport rep)
{
int i = 0;
int pass = 0;
int nin = 0;
int nout = 0;
int wcount = 0;
double[] w = new double[0];
double[] wbest = new double[0];
double e = 0;
double v = 0;
double ebest = 0;
minlbfgs.minlbfgsreport internalrep = new minlbfgs.minlbfgsreport();
minlbfgs.minlbfgsstate state = new minlbfgs.minlbfgsstate();
int i_ = 0;
info = 0;
//
// Test inputs, parse flags, read network geometry
//
if( (double)(wstep)==(double)(0) && maxits==0 )
{
info = -8;
return;
}
if( ((npoints<=0 || restarts<1) || (double)(wstep)<(double)(0)) || maxits<0 )
{
info = -1;
return;
}
mlpbase.mlpproperties(network, ref nin, ref nout, ref wcount);
if( mlpbase.mlpissoftmax(network) )
{
for(i=0; i<=npoints-1; i++)
{
if( (int)Math.Round(xy[i,nin])<0 || (int)Math.Round(xy[i,nin])>=nout )
{
info = -2;
return;
}
}
}
decay = Math.Max(decay, mindecay);
info = 2;
//
// Prepare
//
mlpbase.mlpinitpreprocessor(network, xy, npoints);
w = new double[wcount-1+1];
wbest = new double[wcount-1+1];
ebest = math.maxrealnumber;
//
// Multiple starts
//
rep.ncholesky = 0;
rep.nhess = 0;
rep.ngrad = 0;
for(pass=1; pass<=restarts; pass++)
{
//
// Process
//
mlpbase.mlprandomize(network);
for(i_=0; i_<=wcount-1;i_++)
{
w[i_] = network.weights[i_];
}
minlbfgs.minlbfgscreate(wcount, Math.Min(wcount, 10), w, state);
minlbfgs.minlbfgssetcond(state, 0.0, 0.0, wstep, maxits);
while( minlbfgs.minlbfgsiteration(state) )
{
for(i_=0; i_<=wcount-1;i_++)
{
network.weights[i_] = state.x[i_];
}
mlpbase.mlpgradnbatch(network, xy, npoints, ref state.f, ref state.g);
v = 0.0;
for(i_=0; i_<=wcount-1;i_++)
{
v += network.weights[i_]*network.weights[i_];
}
state.f = state.f+0.5*decay*v;
for(i_=0; i_<=wcount-1;i_++)
{
state.g[i_] = state.g[i_] + decay*network.weights[i_];
}
rep.ngrad = rep.ngrad+1;
}
minlbfgs.minlbfgsresults(state, ref w, internalrep);
for(i_=0; i_<=wcount-1;i_++)
{
network.weights[i_] = w[i_];
}
//
// Compare with best
//
v = 0.0;
for(i_=0; i_<=wcount-1;i_++)
{
v += network.weights[i_]*network.weights[i_];
}
e = mlpbase.mlperrorn(network, xy, npoints)+0.5*decay*v;
if( (double)(e)<(double)(ebest) )
{
for(i_=0; i_<=wcount-1;i_++)
{
wbest[i_] = network.weights[i_];
}
ebest = e;
}
}
//
// The best network
//
for(i_=0; i_<=wcount-1;i_++)
{
network.weights[i_] = wbest[i_];
}
}
/*************************************************************************
Single-threaded stub. HPC ALGLIB replaces it by multithreaded code.
*************************************************************************/
public static void _pexec_mlpkfoldcv(mlptrainer s,
mlpbase.multilayerperceptron network,
int nrestarts,
int foldscount,
mlpreport rep)
{
mlpkfoldcv(s,network,nrestarts,foldscount,rep);
}
/*************************************************************************
Cross-validation estimate of generalization error.
Base algorithm - L-BFGS.
INPUT PARAMETERS:
Network - neural network with initialized geometry. Network is
not changed during cross-validation - it is used only
as a representative of its architecture.
XY - training set.
SSize - training set size
Decay - weight decay, same as in MLPTrainLBFGS
Restarts - number of restarts, >0.
restarts are counted for each partition separately, so
total number of restarts will be Restarts*FoldsCount.
WStep - stopping criterion, same as in MLPTrainLBFGS
MaxIts - stopping criterion, same as in MLPTrainLBFGS
FoldsCount - number of folds in k-fold cross-validation,
2<=FoldsCount<=SSize.
recommended value: 10.
OUTPUT PARAMETERS:
Info - return code, same as in MLPTrainLBFGS
Rep - report, same as in MLPTrainLM/MLPTrainLBFGS
CVRep - generalization error estimates
-- ALGLIB --
Copyright 09.12.2007 by Bochkanov Sergey
*************************************************************************/
public static void mlpkfoldcvlbfgs(mlpbase.multilayerperceptron network,
double[,] xy,
int npoints,
double decay,
int restarts,
double wstep,
int maxits,
int foldscount,
ref int info,
mlpreport rep,
mlpcvreport cvrep)
{
info = 0;
mlpkfoldcvgeneral(network, xy, npoints, decay, restarts, foldscount, false, wstep, maxits, ref info, rep, cvrep);
}
/*************************************************************************
This function trains neural network passed to this function, using current
dataset (one which was passed to MLPSetDataset() or MLPSetSparseDataset())
and current training settings. Training from NRestarts random starting
positions is performed, best network is chosen.
Training is performed using current training algorithm.
FOR USERS OF COMMERCIAL EDITION:
! Commercial version of ALGLIB includes two important improvements of
! this function:
! * multicore support (C++ and C# computational cores)
! * SSE support (C++ computational core)
!
! Second improvement gives constant speedup (2-3X). First improvement
! gives close-to-linear speedup on multicore systems. Following
! operations can be executed in parallel:
! * NRestarts training sessions performed within each of
! cross-validation rounds (if NRestarts>1)
! * gradient calculation over large dataset (if dataset is large enough)
!
! In order to use multicore features you have to:
! * use commercial version of ALGLIB
! * call this function with "smp_" prefix, which indicates that
! multicore code will be used (for multicore support)
!
! In order to use SSE features you have to:
! * use commercial version of ALGLIB on Intel processors
! * use C++ computational core
!
! This note is given for users of commercial edition; if you use GPL
! edition, you still will be able to call smp-version of this function,
! but all computations will be done serially.
!
! We recommend you to carefully read ALGLIB Reference Manual, section
! called 'SMP support', before using parallel version of this function.
INPUT PARAMETERS:
S - trainer object
Network - neural network. It must have same number of inputs and
output/classes as was specified during creation of the
trainer object.
NRestarts - number of restarts, >=0:
* NRestarts>0 means that specified number of random
restarts are performed, best network is chosen after
training
* NRestarts=0 means that current state of the network
is used for training.
OUTPUT PARAMETERS:
Network - trained network
NOTE: when no dataset was specified with MLPSetDataset/SetSparseDataset(),
network is filled by zero values. Same behavior for functions
MLPStartTraining and MLPContinueTraining.
NOTE: this method uses sum-of-squares error function for training.
-- ALGLIB --
Copyright 23.07.2012 by Bochkanov Sergey
*************************************************************************/
public static void mlptrainnetwork(mlptrainer s,
mlpbase.multilayerperceptron network,
int nrestarts,
mlpreport rep)
{
int nin = 0;
int nout = 0;
int wcount = 0;
int ntype = 0;
int ttype = 0;
alglib.smp.shared_pool trnpool = new alglib.smp.shared_pool();
alglib.ap.assert(s.npoints>=0, "MLPTrainNetwork: parameter S is not initialized or is spoiled(S.NPoints<0)");
if( !mlpbase.mlpissoftmax(network) )
{
ntype = 0;
}
else
{
ntype = 1;
}
if( s.rcpar )
{
ttype = 0;
}
else
{
ttype = 1;
}
alglib.ap.assert(ntype==ttype, "MLPTrainNetwork: type of input network is not similar to network type in trainer object");
mlpbase.mlpproperties(network, ref nin, ref nout, ref wcount);
alglib.ap.assert(s.nin==nin, "MLPTrainNetwork: number of inputs in trainer is not equal to number of inputs in network");
alglib.ap.assert(s.nout==nout, "MLPTrainNetwork: number of outputs in trainer is not equal to number of outputs in network");
alglib.ap.assert(nrestarts>=0, "MLPTrainNetwork: NRestarts<0.");
//
// Train
//
mlptrainnetworkx(s, nrestarts, -1, s.subset, -1, s.subset, 0, network, rep, true, trnpool);
}
/*************************************************************************
Internal cross-validation subroutine
*************************************************************************/
private static void mlpkfoldcvgeneral(mlpbase.multilayerperceptron n,
double[,] xy,
int npoints,
double decay,
int restarts,
int foldscount,
bool lmalgorithm,
double wstep,
int maxits,
ref int info,
mlpreport rep,
mlpcvreport cvrep)
{
int i = 0;
int fold = 0;
int j = 0;
int k = 0;
mlpbase.multilayerperceptron network = new mlpbase.multilayerperceptron();
int nin = 0;
int nout = 0;
int rowlen = 0;
int wcount = 0;
int nclasses = 0;
int tssize = 0;
int cvssize = 0;
double[,] cvset = new double[0,0];
double[,] testset = new double[0,0];
int[] folds = new int[0];
int relcnt = 0;
mlpreport internalrep = new mlpreport();
double[] x = new double[0];
double[] y = new double[0];
int i_ = 0;
info = 0;
//
// Read network geometry, test parameters
//
mlpbase.mlpproperties(n, ref nin, ref nout, ref wcount);
if( mlpbase.mlpissoftmax(n) )
{
nclasses = nout;
rowlen = nin+1;
}
else
{
nclasses = -nout;
rowlen = nin+nout;
}
if( (npoints<=0 || foldscount<2) || foldscount>npoints )
{
info = -1;
return;
}
mlpbase.mlpcopy(n, network);
//
// K-fold out cross-validation.
// First, estimate generalization error
//
testset = new double[npoints-1+1, rowlen-1+1];
cvset = new double[npoints-1+1, rowlen-1+1];
x = new double[nin-1+1];
y = new double[nout-1+1];
mlpkfoldsplit(xy, npoints, nclasses, foldscount, false, ref folds);
cvrep.relclserror = 0;
cvrep.avgce = 0;
cvrep.rmserror = 0;
cvrep.avgerror = 0;
cvrep.avgrelerror = 0;
rep.ngrad = 0;
rep.nhess = 0;
rep.ncholesky = 0;
relcnt = 0;
for(fold=0; fold<=foldscount-1; fold++)
{
//
// Separate set
//
tssize = 0;
cvssize = 0;
for(i=0; i<=npoints-1; i++)
{
if( folds[i]==fold )
{
for(i_=0; i_<=rowlen-1;i_++)
{
testset[tssize,i_] = xy[i,i_];
}
tssize = tssize+1;
}
else
{
for(i_=0; i_<=rowlen-1;i_++)
{
cvset[cvssize,i_] = xy[i,i_];
}
cvssize = cvssize+1;
}
}
//
// Train on CV training set
//
if( lmalgorithm )
{
mlptrainlm(network, cvset, cvssize, decay, restarts, ref info, internalrep);
}
else
{
mlptrainlbfgs(network, cvset, cvssize, decay, restarts, wstep, maxits, ref info, internalrep);
}
if( info<0 )
{
cvrep.relclserror = 0;
cvrep.avgce = 0;
cvrep.rmserror = 0;
cvrep.avgerror = 0;
cvrep.avgrelerror = 0;
return;
}
rep.ngrad = rep.ngrad+internalrep.ngrad;
rep.nhess = rep.nhess+internalrep.nhess;
rep.ncholesky = rep.ncholesky+internalrep.ncholesky;
//
// Estimate error using CV test set
//
if( mlpbase.mlpissoftmax(network) )
{
//
// classification-only code
//
cvrep.relclserror = cvrep.relclserror+mlpbase.mlpclserror(network, testset, tssize);
cvrep.avgce = cvrep.avgce+mlpbase.mlperrorn(network, testset, tssize);
}
for(i=0; i<=tssize-1; i++)
{
for(i_=0; i_<=nin-1;i_++)
{
x[i_] = testset[i,i_];
}
mlpbase.mlpprocess(network, x, ref y);
if( mlpbase.mlpissoftmax(network) )
{
//
// Classification-specific code
//
k = (int)Math.Round(testset[i,nin]);
for(j=0; j<=nout-1; j++)
{
if( j==k )
{
cvrep.rmserror = cvrep.rmserror+math.sqr(y[j]-1);
cvrep.avgerror = cvrep.avgerror+Math.Abs(y[j]-1);
cvrep.avgrelerror = cvrep.avgrelerror+Math.Abs(y[j]-1);
relcnt = relcnt+1;
}
else
{
cvrep.rmserror = cvrep.rmserror+math.sqr(y[j]);
cvrep.avgerror = cvrep.avgerror+Math.Abs(y[j]);
}
}
}
else
{
//
// Regression-specific code
//
for(j=0; j<=nout-1; j++)
{
cvrep.rmserror = cvrep.rmserror+math.sqr(y[j]-testset[i,nin+j]);
cvrep.avgerror = cvrep.avgerror+Math.Abs(y[j]-testset[i,nin+j]);
if( (double)(testset[i,nin+j])!=(double)(0) )
{
cvrep.avgrelerror = cvrep.avgrelerror+Math.Abs((y[j]-testset[i,nin+j])/testset[i,nin+j]);
relcnt = relcnt+1;
}
}
}
}
}
if( mlpbase.mlpissoftmax(network) )
{
cvrep.relclserror = cvrep.relclserror/npoints;
cvrep.avgce = cvrep.avgce/(Math.Log(2)*npoints);
}
cvrep.rmserror = Math.Sqrt(cvrep.rmserror/(npoints*nout));
cvrep.avgerror = cvrep.avgerror/(npoints*nout);
cvrep.avgrelerror = cvrep.avgrelerror/relcnt;
info = 1;
}
public override void init()
{
network = new mlpbase.multilayerperceptron();
tnetwork = new mlpbase.multilayerperceptron();
state = new minlbfgs.minlbfgsstate();
rep = new mlpreport();
subset = new int[0];
xyrow = new double[0];
y = new double[0];
bufwbest = new double[0];
bufwfinal = new double[0];
}
