/*************************************************************************
Unsets network (initialize it to smallest network possible
*************************************************************************/
private static void unsetensemble(mlpe.mlpensemble ensemble)
{
mlpe.mlpecreate0(1, 1, 1, ensemble);
}
public mlpensemble(mlpe.mlpensemble obj)
{
_innerobj = obj;
}
/*************************************************************************
Internal bagging subroutine.
-- ALGLIB --
Copyright 19.02.2009 by Bochkanov Sergey
*************************************************************************/
private static void mlpebagginginternal(mlpe.mlpensemble ensemble,
double[,] xy,
int npoints,
double decay,
int restarts,
double wstep,
int maxits,
bool lmalgorithm,
ref int info,
mlpreport rep,
mlpcvreport ooberrors)
{
double[,] xys = new double[0,0];
bool[] s = new bool[0];
double[,] oobbuf = new double[0,0];
int[] oobcntbuf = new int[0];
double[] x = new double[0];
double[] y = new double[0];
double[] dy = new double[0];
double[] dsbuf = new double[0];
int ccnt = 0;
int pcnt = 0;
int i = 0;
int j = 0;
int k = 0;
double v = 0;
mlpreport tmprep = new mlpreport();
int nin = 0;
int nout = 0;
int wcount = 0;
hqrnd.hqrndstate rs = new hqrnd.hqrndstate();
int i_ = 0;
int i1_ = 0;
info = 0;
nin = mlpbase.mlpgetinputscount(ensemble.network);
nout = mlpbase.mlpgetoutputscount(ensemble.network);
wcount = mlpbase.mlpgetweightscount(ensemble.network);
//
// Test for inputs
//
if( (!lmalgorithm && (double)(wstep)==(double)(0)) && maxits==0 )
{
info = -8;
return;
}
if( ((npoints<=0 || restarts<1) || (double)(wstep)<(double)(0)) || maxits<0 )
{
info = -1;
return;
}
if( mlpbase.mlpissoftmax(ensemble.network) )
{
for(i=0; i<=npoints-1; i++)
{
if( (int)Math.Round(xy[i,nin])<0 || (int)Math.Round(xy[i,nin])>=nout )
{
info = -2;
return;
}
}
}
//
// allocate temporaries
//
info = 2;
rep.ngrad = 0;
rep.nhess = 0;
rep.ncholesky = 0;
ooberrors.relclserror = 0;
ooberrors.avgce = 0;
ooberrors.rmserror = 0;
ooberrors.avgerror = 0;
ooberrors.avgrelerror = 0;
if( mlpbase.mlpissoftmax(ensemble.network) )
{
ccnt = nin+1;
pcnt = nin;
}
else
{
ccnt = nin+nout;
pcnt = nin+nout;
}
xys = new double[npoints, ccnt];
s = new bool[npoints];
oobbuf = new double[npoints, nout];
oobcntbuf = new int[npoints];
x = new double[nin];
y = new double[nout];
if( mlpbase.mlpissoftmax(ensemble.network) )
{
dy = new double[1];
}
else
{
dy = new double[nout];
}
for(i=0; i<=npoints-1; i++)
{
for(j=0; j<=nout-1; j++)
{
oobbuf[i,j] = 0;
}
}
for(i=0; i<=npoints-1; i++)
{
oobcntbuf[i] = 0;
}
//
// main bagging cycle
//
hqrnd.hqrndrandomize(rs);
for(k=0; k<=ensemble.ensemblesize-1; k++)
{
//
// prepare dataset
//
for(i=0; i<=npoints-1; i++)
{
s[i] = false;
}
for(i=0; i<=npoints-1; i++)
{
j = hqrnd.hqrnduniformi(rs, npoints);
s[j] = true;
for(i_=0; i_<=ccnt-1;i_++)
{
xys[i,i_] = xy[j,i_];
}
}
//
// train
//
if( lmalgorithm )
{
mlptrainlm(ensemble.network, xys, npoints, decay, restarts, ref info, tmprep);
}
else
{
mlptrainlbfgs(ensemble.network, xys, npoints, decay, restarts, wstep, maxits, ref info, tmprep);
}
if( info<0 )
{
return;
}
//
// save results
//
rep.ngrad = rep.ngrad+tmprep.ngrad;
rep.nhess = rep.nhess+tmprep.nhess;
rep.ncholesky = rep.ncholesky+tmprep.ncholesky;
i1_ = (0) - (k*wcount);
for(i_=k*wcount; i_<=(k+1)*wcount-1;i_++)
{
ensemble.weights[i_] = ensemble.network.weights[i_+i1_];
}
i1_ = (0) - (k*pcnt);
for(i_=k*pcnt; i_<=(k+1)*pcnt-1;i_++)
{
ensemble.columnmeans[i_] = ensemble.network.columnmeans[i_+i1_];
}
i1_ = (0) - (k*pcnt);
for(i_=k*pcnt; i_<=(k+1)*pcnt-1;i_++)
{
ensemble.columnsigmas[i_] = ensemble.network.columnsigmas[i_+i1_];
}
//
// OOB estimates
//
for(i=0; i<=npoints-1; i++)
{
if( !s[i] )
{
for(i_=0; i_<=nin-1;i_++)
{
x[i_] = xy[i,i_];
}
mlpbase.mlpprocess(ensemble.network, x, ref y);
for(i_=0; i_<=nout-1;i_++)
{
oobbuf[i,i_] = oobbuf[i,i_] + y[i_];
}
oobcntbuf[i] = oobcntbuf[i]+1;
}
}
}
//
// OOB estimates
//
if( mlpbase.mlpissoftmax(ensemble.network) )
{
bdss.dserrallocate(nout, ref dsbuf);
}
else
{
bdss.dserrallocate(-nout, ref dsbuf);
}
for(i=0; i<=npoints-1; i++)
{
if( oobcntbuf[i]!=0 )
{
v = (double)1/(double)oobcntbuf[i];
for(i_=0; i_<=nout-1;i_++)
{
y[i_] = v*oobbuf[i,i_];
}
if( mlpbase.mlpissoftmax(ensemble.network) )
{
dy[0] = xy[i,nin];
}
else
{
i1_ = (nin) - (0);
for(i_=0; i_<=nout-1;i_++)
{
dy[i_] = v*xy[i,i_+i1_];
}
}
bdss.dserraccumulate(ref dsbuf, y, dy);
}
}
bdss.dserrfinish(ref dsbuf);
ooberrors.relclserror = dsbuf[0];
ooberrors.avgce = dsbuf[1];
ooberrors.rmserror = dsbuf[2];
ooberrors.avgerror = dsbuf[3];
ooberrors.avgrelerror = dsbuf[4];
}
/*************************************************************************
Network creation
*************************************************************************/
private static void createensemble(mlpe.mlpensemble ensemble,
int nkind,
double a1,
double a2,
int nin,
int nhid1,
int nhid2,
int nout,
int ec)
{
ap.assert(((nin>0 & nhid1>=0) & nhid2>=0) & nout>0, "CreateNetwork error");
ap.assert(nhid1!=0 | nhid2==0, "CreateNetwork error");
ap.assert(nkind!=1 | nout>=2, "CreateNetwork error");
if( nhid1==0 )
{
//
// No hidden layers
//
if( nkind==0 )
{
mlpe.mlpecreate0(nin, nout, ec, ensemble);
}
else
{
if( nkind==1 )
{
mlpe.mlpecreatec0(nin, nout, ec, ensemble);
}
else
{
if( nkind==2 )
{
mlpe.mlpecreateb0(nin, nout, a1, a2, ec, ensemble);
}
else
{
if( nkind==3 )
{
mlpe.mlpecreater0(nin, nout, a1, a2, ec, ensemble);
}
}
}
}
return;
}
if( nhid2==0 )
{
//
// One hidden layer
//
if( nkind==0 )
{
mlpe.mlpecreate1(nin, nhid1, nout, ec, ensemble);
}
else
{
if( nkind==1 )
{
mlpe.mlpecreatec1(nin, nhid1, nout, ec, ensemble);
}
else
{
if( nkind==2 )
{
mlpe.mlpecreateb1(nin, nhid1, nout, a1, a2, ec, ensemble);
}
else
{
if( nkind==3 )
{
mlpe.mlpecreater1(nin, nhid1, nout, a1, a2, ec, ensemble);
}
}
}
}
return;
}
//
// Two hidden layers
//
if( nkind==0 )
{
mlpe.mlpecreate2(nin, nhid1, nhid2, nout, ec, ensemble);
}
else
{
if( nkind==1 )
{
mlpe.mlpecreatec2(nin, nhid1, nhid2, nout, ec, ensemble);
}
else
{
if( nkind==2 )
{
mlpe.mlpecreateb2(nin, nhid1, nhid2, nout, a1, a2, ec, ensemble);
}
else
{
if( nkind==3 )
{
mlpe.mlpecreater2(nin, nhid1, nhid2, nout, a1, a2, ec, ensemble);
}
}
}
}
}
/*************************************************************************
Single-threaded stub. HPC ALGLIB replaces it by multithreaded code.
*************************************************************************/
public static void _pexec_mlptrainensemblees(mlptrainer s,
mlpe.mlpensemble ensemble,
int nrestarts,
mlpreport rep)
{
mlptrainensemblees(s,ensemble,nrestarts,rep);
}
/*************************************************************************
This function trains neural network ensemble passed to this function using
current dataset and early stopping training algorithm. Each early stopping
round performs NRestarts random restarts (thus, EnsembleSize*NRestarts
training rounds is performed in total).
-- ALGLIB --
Copyright 22.08.2012 by Bochkanov Sergey
*************************************************************************/
private static void mlptrainensemblex(mlptrainer s,
mlpe.mlpensemble ensemble,
int idx0,
int idx1,
int nrestarts,
int trainingmethod,
apserv.sinteger ngrad,
bool isrootcall,
alglib.smp.shared_pool esessions)
{
int pcount = 0;
int nin = 0;
int nout = 0;
int wcount = 0;
int i = 0;
int j = 0;
int k = 0;
int trnsubsetsize = 0;
int valsubsetsize = 0;
int k0 = 0;
apserv.sinteger ngrad0 = new apserv.sinteger();
apserv.sinteger ngrad1 = new apserv.sinteger();
mlpetrnsession psession = null;
hqrnd.hqrndstate rs = new hqrnd.hqrndstate();
int i_ = 0;
int i1_ = 0;
nin = mlpbase.mlpgetinputscount(ensemble.network);
nout = mlpbase.mlpgetoutputscount(ensemble.network);
wcount = mlpbase.mlpgetweightscount(ensemble.network);
if( mlpbase.mlpissoftmax(ensemble.network) )
{
pcount = nin;
}
else
{
pcount = nin+nout;
}
if( nrestarts<=0 )
{
nrestarts = 1;
}
//
// Handle degenerate case
//
if( s.npoints<2 )
{
for(i=idx0; i<=idx1-1; i++)
{
for(j=0; j<=wcount-1; j++)
{
ensemble.weights[i*wcount+j] = 0.0;
}
for(j=0; j<=pcount-1; j++)
{
ensemble.columnmeans[i*pcount+j] = 0.0;
ensemble.columnsigmas[i*pcount+j] = 1.0;
}
}
return;
}
//
// Process root call
//
if( isrootcall )
{
//
// Prepare:
// * prepare MLPETrnSessions
// * fill ensemble by zeros (helps to detect errors)
//
initmlpetrnsessions(ensemble.network, s, esessions);
for(i=idx0; i<=idx1-1; i++)
{
for(j=0; j<=wcount-1; j++)
{
ensemble.weights[i*wcount+j] = 0.0;
}
for(j=0; j<=pcount-1; j++)
{
ensemble.columnmeans[i*pcount+j] = 0.0;
ensemble.columnsigmas[i*pcount+j] = 0.0;
}
}
//
// Train in non-root mode and exit
//
mlptrainensemblex(s, ensemble, idx0, idx1, nrestarts, trainingmethod, ngrad, false, esessions);
return;
}
//
// Split problem
//
if( idx1-idx0>=2 )
{
k0 = (idx1-idx0)/2;
ngrad0.val = 0;
ngrad1.val = 0;
mlptrainensemblex(s, ensemble, idx0, idx0+k0, nrestarts, trainingmethod, ngrad0, false, esessions);
mlptrainensemblex(s, ensemble, idx0+k0, idx1, nrestarts, trainingmethod, ngrad1, false, esessions);
ngrad.val = ngrad0.val+ngrad1.val;
return;
}
//
// Retrieve and prepare session
//
alglib.smp.ae_shared_pool_retrieve(esessions, ref psession);
//
// Train
//
hqrnd.hqrndrandomize(rs);
for(k=idx0; k<=idx1-1; k++)
{
//
// Split set
//
trnsubsetsize = 0;
valsubsetsize = 0;
if( trainingmethod==0 )
{
do
{
trnsubsetsize = 0;
valsubsetsize = 0;
for(i=0; i<=s.npoints-1; i++)
{
if( (double)(math.randomreal())<(double)(0.66) )
{
//
// Assign sample to training set
//
psession.trnsubset[trnsubsetsize] = i;
trnsubsetsize = trnsubsetsize+1;
}
else
{
//
// Assign sample to validation set
//
psession.valsubset[valsubsetsize] = i;
valsubsetsize = valsubsetsize+1;
}
}
}
while( !(trnsubsetsize!=0 && valsubsetsize!=0) );
}
if( trainingmethod==1 )
{
valsubsetsize = 0;
trnsubsetsize = s.npoints;
for(i=0; i<=s.npoints-1; i++)
{
psession.trnsubset[i] = hqrnd.hqrnduniformi(rs, s.npoints);
}
}
//
// Train
//
mlptrainnetworkx(s, nrestarts, -1, psession.trnsubset, trnsubsetsize, psession.valsubset, valsubsetsize, psession.network, psession.mlprep, true, psession.mlpsessions);
ngrad.val = ngrad.val+psession.mlprep.ngrad;
//
// Save results
//
i1_ = (0) - (k*wcount);
for(i_=k*wcount; i_<=(k+1)*wcount-1;i_++)
{
ensemble.weights[i_] = psession.network.weights[i_+i1_];
}
i1_ = (0) - (k*pcount);
for(i_=k*pcount; i_<=(k+1)*pcount-1;i_++)
{
ensemble.columnmeans[i_] = psession.network.columnmeans[i_+i1_];
}
i1_ = (0) - (k*pcount);
for(i_=k*pcount; i_<=(k+1)*pcount-1;i_++)
{
ensemble.columnsigmas[i_] = psession.network.columnsigmas[i_+i1_];
}
}
//
// Recycle session
//
alglib.smp.ae_shared_pool_recycle(esessions, ref psession);
}
/*************************************************************************
This function trains neural network ensemble passed to this function using
current dataset and early stopping training algorithm. Each early stopping
round performs NRestarts random restarts (thus, EnsembleSize*NRestarts
training rounds is performed in total).
FOR USERS OF COMMERCIAL EDITION:
! Commercial version of ALGLIB includes two important improvements of
! this function:
! * multicore support (C++ and C# computational cores)
! * SSE support (C++ computational core)
!
! Second improvement gives constant speedup (2-3X). First improvement
! gives close-to-linear speedup on multicore systems. Following
! operations can be executed in parallel:
! * EnsembleSize training sessions performed for each of ensemble
! members (always parallelized)
! * NRestarts training sessions performed within each of training
! sessions (if NRestarts>1)
! * gradient calculation over large dataset (if dataset is large enough)
!
! In order to use multicore features you have to:
! * use commercial version of ALGLIB
! * call this function with "smp_" prefix, which indicates that
! multicore code will be used (for multicore support)
!
! In order to use SSE features you have to:
! * use commercial version of ALGLIB on Intel processors
! * use C++ computational core
!
! This note is given for users of commercial edition; if you use GPL
! edition, you still will be able to call smp-version of this function,
! but all computations will be done serially.
!
! We recommend you to carefully read ALGLIB Reference Manual, section
! called 'SMP support', before using parallel version of this function.
INPUT PARAMETERS:
S - trainer object;
Ensemble - neural network ensemble. It must have same number of
inputs and outputs/classes as was specified during
creation of the trainer object.
NRestarts - number of restarts, >=0:
* NRestarts>0 means that specified number of random
restarts are performed during each ES round;
* NRestarts=0 is silently replaced by 1.
OUTPUT PARAMETERS:
Ensemble - trained ensemble;
Rep - it contains all type of errors.
NOTE: this training method uses BOTH early stopping and weight decay! So,
you should select weight decay before starting training just as you
select it before training "conventional" networks.
NOTE: when no dataset was specified with MLPSetDataset/SetSparseDataset(),
or single-point dataset was passed, ensemble is filled by zero
values.
NOTE: this method uses sum-of-squares error function for training.
-- ALGLIB --
Copyright 22.08.2012 by Bochkanov Sergey
*************************************************************************/
public static void mlptrainensemblees(mlptrainer s,
mlpe.mlpensemble ensemble,
int nrestarts,
mlpreport rep)
{
int nin = 0;
int nout = 0;
int ntype = 0;
int ttype = 0;
alglib.smp.shared_pool esessions = new alglib.smp.shared_pool();
apserv.sinteger sgrad = new apserv.sinteger();
mlpbase.modelerrors tmprep = new mlpbase.modelerrors();
alglib.ap.assert(s.npoints>=0, "MLPTrainEnsembleES: parameter S is not initialized or is spoiled(S.NPoints<0)");
if( !mlpe.mlpeissoftmax(ensemble) )
{
ntype = 0;
}
else
{
ntype = 1;
}
if( s.rcpar )
{
ttype = 0;
}
else
{
ttype = 1;
}
alglib.ap.assert(ntype==ttype, "MLPTrainEnsembleES: internal error - type of input network is not similar to network type in trainer object");
nin = mlpbase.mlpgetinputscount(ensemble.network);
alglib.ap.assert(s.nin==nin, "MLPTrainEnsembleES: number of inputs in trainer is not equal to number of inputs in ensemble network");
nout = mlpbase.mlpgetoutputscount(ensemble.network);
alglib.ap.assert(s.nout==nout, "MLPTrainEnsembleES: number of outputs in trainer is not equal to number of outputs in ensemble network");
alglib.ap.assert(nrestarts>=0, "MLPTrainEnsembleES: NRestarts<0.");
//
// Initialize parameter Rep
//
rep.relclserror = 0;
rep.avgce = 0;
rep.rmserror = 0;
rep.avgerror = 0;
rep.avgrelerror = 0;
rep.ngrad = 0;
rep.nhess = 0;
rep.ncholesky = 0;
//
// Allocate
//
apserv.ivectorsetlengthatleast(ref s.subset, s.npoints);
apserv.ivectorsetlengthatleast(ref s.valsubset, s.npoints);
//
// Start training
//
// NOTE: ESessions is not initialized because MLPTrainEnsembleX
// needs uninitialized pool.
//
sgrad.val = 0;
mlptrainensemblex(s, ensemble, 0, ensemble.ensemblesize, nrestarts, 0, sgrad, true, esessions);
rep.ngrad = sgrad.val;
//
// Calculate errors.
//
if( s.datatype==0 )
{
mlpe.mlpeallerrorsx(ensemble, s.densexy, s.sparsexy, s.npoints, 0, ensemble.network.dummyidx, 0, s.npoints, 0, ensemble.network.buf, tmprep);
}
if( s.datatype==1 )
{
mlpe.mlpeallerrorsx(ensemble, s.densexy, s.sparsexy, s.npoints, 1, ensemble.network.dummyidx, 0, s.npoints, 0, ensemble.network.buf, tmprep);
}
rep.relclserror = tmprep.relclserror;
rep.avgce = tmprep.avgce;
rep.rmserror = tmprep.rmserror;
rep.avgerror = tmprep.avgerror;
rep.avgrelerror = tmprep.avgrelerror;
}
/*************************************************************************
Training neural networks ensemble using early stopping.
INPUT PARAMETERS:
Ensemble - model with initialized geometry
XY - training set
NPoints - training set size
Decay - weight decay coefficient, >=0.001
Restarts - restarts, >0.
OUTPUT PARAMETERS:
Ensemble - trained model
Info - return code:
* -2, if there is a point with class number
outside of [0..NClasses-1].
* -1, if incorrect parameters was passed
(NPoints<0, Restarts<1).
* 6, if task has been solved.
Rep - training report.
OOBErrors - out-of-bag generalization error estimate
-- ALGLIB --
Copyright 10.03.2009 by Bochkanov Sergey
*************************************************************************/
public static void mlpetraines(mlpe.mlpensemble ensemble,
double[,] xy,
int npoints,
double decay,
int restarts,
ref int info,
mlpreport rep)
{
int i = 0;
int k = 0;
int ccount = 0;
int pcount = 0;
double[,] trnxy = new double[0,0];
double[,] valxy = new double[0,0];
int trnsize = 0;
int valsize = 0;
int tmpinfo = 0;
mlpreport tmprep = new mlpreport();
mlpbase.modelerrors moderr = new mlpbase.modelerrors();
int nin = 0;
int nout = 0;
int wcount = 0;
int i_ = 0;
int i1_ = 0;
info = 0;
nin = mlpbase.mlpgetinputscount(ensemble.network);
nout = mlpbase.mlpgetoutputscount(ensemble.network);
wcount = mlpbase.mlpgetweightscount(ensemble.network);
if( (npoints<2 || restarts<1) || (double)(decay)<(double)(0) )
{
info = -1;
return;
}
if( mlpbase.mlpissoftmax(ensemble.network) )
{
for(i=0; i<=npoints-1; i++)
{
if( (int)Math.Round(xy[i,nin])<0 || (int)Math.Round(xy[i,nin])>=nout )
{
info = -2;
return;
}
}
}
info = 6;
//
// allocate
//
if( mlpbase.mlpissoftmax(ensemble.network) )
{
ccount = nin+1;
pcount = nin;
}
else
{
ccount = nin+nout;
pcount = nin+nout;
}
trnxy = new double[npoints, ccount];
valxy = new double[npoints, ccount];
rep.ngrad = 0;
rep.nhess = 0;
rep.ncholesky = 0;
//
// train networks
//
for(k=0; k<=ensemble.ensemblesize-1; k++)
{
//
// Split set
//
do
{
trnsize = 0;
valsize = 0;
for(i=0; i<=npoints-1; i++)
{
if( (double)(math.randomreal())<(double)(0.66) )
{
//
// Assign sample to training set
//
for(i_=0; i_<=ccount-1;i_++)
{
trnxy[trnsize,i_] = xy[i,i_];
}
trnsize = trnsize+1;
}
else
{
//
// Assign sample to validation set
//
for(i_=0; i_<=ccount-1;i_++)
{
valxy[valsize,i_] = xy[i,i_];
}
valsize = valsize+1;
}
}
}
while( !(trnsize!=0 && valsize!=0) );
//
// Train
//
mlptraines(ensemble.network, trnxy, trnsize, valxy, valsize, decay, restarts, ref tmpinfo, tmprep);
if( tmpinfo<0 )
{
info = tmpinfo;
return;
}
//
// save results
//
i1_ = (0) - (k*wcount);
for(i_=k*wcount; i_<=(k+1)*wcount-1;i_++)
{
ensemble.weights[i_] = ensemble.network.weights[i_+i1_];
}
i1_ = (0) - (k*pcount);
for(i_=k*pcount; i_<=(k+1)*pcount-1;i_++)
{
ensemble.columnmeans[i_] = ensemble.network.columnmeans[i_+i1_];
}
i1_ = (0) - (k*pcount);
for(i_=k*pcount; i_<=(k+1)*pcount-1;i_++)
{
ensemble.columnsigmas[i_] = ensemble.network.columnsigmas[i_+i1_];
}
rep.ngrad = rep.ngrad+tmprep.ngrad;
rep.nhess = rep.nhess+tmprep.nhess;
rep.ncholesky = rep.ncholesky+tmprep.ncholesky;
}
mlpe.mlpeallerrorsx(ensemble, xy, ensemble.network.dummysxy, npoints, 0, ensemble.network.dummyidx, 0, npoints, 0, ensemble.network.buf, moderr);
rep.relclserror = moderr.relclserror;
rep.avgce = moderr.avgce;
rep.rmserror = moderr.rmserror;
rep.avgerror = moderr.avgerror;
rep.avgrelerror = moderr.avgrelerror;
}
/*************************************************************************
Training neural networks ensemble using bootstrap aggregating (bagging).
L-BFGS algorithm is used as base training method.
INPUT PARAMETERS:
Ensemble - model with initialized geometry
XY - training set
NPoints - training set size
Decay - weight decay coefficient, >=0.001
Restarts - restarts, >0.
WStep - stopping criterion, same as in MLPTrainLBFGS
MaxIts - stopping criterion, same as in MLPTrainLBFGS
OUTPUT PARAMETERS:
Ensemble - trained model
Info - return code:
* -8, if both WStep=0 and MaxIts=0
* -2, if there is a point with class number
outside of [0..NClasses-1].
* -1, if incorrect parameters was passed
(NPoints<0, Restarts<1).
* 2, if task has been solved.
Rep - training report.
OOBErrors - out-of-bag generalization error estimate
-- ALGLIB --
Copyright 17.02.2009 by Bochkanov Sergey
*************************************************************************/
public static void mlpebagginglbfgs(mlpe.mlpensemble ensemble,
double[,] xy,
int npoints,
double decay,
int restarts,
double wstep,
int maxits,
ref int info,
mlpreport rep,
mlpcvreport ooberrors)
{
info = 0;
mlpebagginginternal(ensemble, xy, npoints, decay, restarts, wstep, maxits, false, ref info, rep, ooberrors);
}
/*************************************************************************
Training neural networks ensemble using bootstrap aggregating (bagging).
Modified Levenberg-Marquardt algorithm is used as base training method.
INPUT PARAMETERS:
Ensemble - model with initialized geometry
XY - training set
NPoints - training set size
Decay - weight decay coefficient, >=0.001
Restarts - restarts, >0.
OUTPUT PARAMETERS:
Ensemble - trained model
Info - return code:
* -2, if there is a point with class number
outside of [0..NClasses-1].
* -1, if incorrect parameters was passed
(NPoints<0, Restarts<1).
* 2, if task has been solved.
Rep - training report.
OOBErrors - out-of-bag generalization error estimate
-- ALGLIB --
Copyright 17.02.2009 by Bochkanov Sergey
*************************************************************************/
public static void mlpebagginglm(mlpe.mlpensemble ensemble,
double[,] xy,
int npoints,
double decay,
int restarts,
ref int info,
mlpreport rep,
mlpcvreport ooberrors)
{
info = 0;
mlpebagginginternal(ensemble, xy, npoints, decay, restarts, 0.0, 0, true, ref info, rep, ooberrors);
}
/*************************************************************************
This function trains neural network ensemble passed to this function using
current dataset and early stopping training algorithm. Each early stopping
round performs NRestarts random restarts (thus, EnsembleSize*NRestarts
training rounds is performed in total).
INPUT PARAMETERS:
S - trainer object;
Ensemble - neural network ensemble. It must have same number of
inputs and outputs/classes as was specified during
creation of the trainer object.
NRestarts - number of restarts, >=0:
* NRestarts>0 means that specified number of random
restarts are performed during each ES round;
* NRestarts=0 is silently replaced by 1.
OUTPUT PARAMETERS:
Ensemble - trained ensemble;
Rep - it contains all type of errors.
NOTE: when no dataset was specified with MLPSetDataset/SetSparseDataset(),
or single-point dataset was passed, ensemble is filled by zero
values.
NOTE: this method uses sum-of-squares error function for training.
-- ALGLIB --
Copyright 22.08.2012 by Bochkanov Sergey
*************************************************************************/
public static void mlptrainensemblees(mlptrainer s,
mlpe.mlpensemble ensemble,
int nrestarts,
mlpreport rep)
{
int pcount = 0;
mlpreport tmprep = new mlpreport();
int nin = 0;
int nout = 0;
int wcount = 0;
int ntype = 0;
int ttype = 0;
int i = 0;
int k = 0;
int i_ = 0;
int i1_ = 0;
alglib.ap.assert(s.npoints>=0, "MLPTrainEnsembleES: parameter S is not initialized or is spoiled(S.NPoints<0)");
if( !mlpe.mlpeissoftmax(ensemble) )
{
ntype = 0;
}
else
{
ntype = 1;
}
if( s.rcpar )
{
ttype = 0;
}
else
{
ttype = 1;
}
alglib.ap.assert(ntype==ttype, "MLPTrainEnsembleES: internal error - type of input network is not similar to network type in trainer object");
nin = mlpbase.mlpgetinputscount(ensemble.network);
alglib.ap.assert(s.nin==nin, "MLPTrainEnsembleES: number of inputs in trainer is not equal to number of inputs in ensemble network");
nout = mlpbase.mlpgetoutputscount(ensemble.network);
alglib.ap.assert(s.nout==nout, "MLPTrainEnsembleES: number of outputs in trainer is not equal to number of outputs in ensemble network");
alglib.ap.assert(nrestarts>=0, "MLPTrainEnsembleES: NRestarts<0.");
wcount = mlpbase.mlpgetweightscount(ensemble.network);
//
// Initialize parameter Rep
//
rep.relclserror = 0;
rep.avgce = 0;
rep.rmserror = 0;
rep.avgerror = 0;
rep.avgrelerror = 0;
rep.ngrad = 0;
rep.nhess = 0;
rep.ncholesky = 0;
//
// Allocate
//
if( mlpbase.mlpissoftmax(ensemble.network) )
{
pcount = nin;
}
else
{
pcount = nin+nout;
}
apserv.ivectorsetlengthatleast(ref s.subset, s.npoints);
apserv.ivectorsetlengthatleast(ref s.valsubset, s.npoints);
apserv.rvectorsetlengthatleast(ref s.wbest, wcount);
apserv.rvectorsetlengthatleast(ref s.wfinal, wcount);
//
// Create LBFGS optimizer
//
minlbfgs.minlbfgscreate(wcount, Math.Min(wcount, s.lbfgsfactor), ensemble.network.weights, s.tstate);
minlbfgs.minlbfgssetcond(s.tstate, 0.0, 0.0, s.wstep, s.maxits);
minlbfgs.minlbfgssetxrep(s.tstate, true);
mlpbase.mlpcopy(ensemble.network, s.tnetwork);
//
// Train networks
//
if( (s.datatype==0 || s.datatype==1) && s.npoints>1 )
{
for(k=0; k<=ensemble.ensemblesize-1; k++)
{
//
// Split set
//
do
{
s.subsetsize = 0;
s.valsubsetsize = 0;
for(i=0; i<=s.npoints-1; i++)
{
if( (double)(math.randomreal())<(double)(0.66) )
{
//
// Assign sample to training set
//
s.subset[s.subsetsize] = i;
s.subsetsize = s.subsetsize+1;
}
else
{
//
// Assign sample to validation set
//
s.valsubset[s.valsubsetsize] = i;
s.valsubsetsize = s.valsubsetsize+1;
}
}
}
while( !(s.subsetsize!=0 && s.valsubsetsize!=0) );
//
// Train
//
mlptrainnetworkx(s, ensemble.network, s.tnetwork, s.tstate, nrestarts, s.subset, s.subsetsize, s.valsubset, s.valsubsetsize, s.wbest, s.wfinal, tmprep);
rep.ngrad = rep.ngrad+tmprep.ngrad;
//
// Save results
//
i1_ = (0) - (k*wcount);
for(i_=k*wcount; i_<=(k+1)*wcount-1;i_++)
{
ensemble.weights[i_] = ensemble.network.weights[i_+i1_];
}
i1_ = (0) - (k*pcount);
for(i_=k*pcount; i_<=(k+1)*pcount-1;i_++)
{
ensemble.columnmeans[i_] = ensemble.network.columnmeans[i_+i1_];
}
i1_ = (0) - (k*pcount);
for(i_=k*pcount; i_<=(k+1)*pcount-1;i_++)
{
ensemble.columnsigmas[i_] = ensemble.network.columnsigmas[i_+i1_];
}
}
}
else
{
for(i=0; i<=ensemble.ensemblesize*wcount-1; i++)
{
ensemble.network.weights[i] = 0.0;
ensemble.columnmeans[i] = 0.0;
ensemble.columnsigmas[i] = 1.0;
}
}
//
// Calculate errors.
//
if( s.datatype==0 )
{
mlpe.mlpeallerrors(ensemble, s.densexy, s.npoints, ref rep.relclserror, ref rep.avgce, ref rep.rmserror, ref rep.avgerror, ref rep.avgrelerror);
}
if( s.datatype==1 )
{
mlpe.mlpeallerrorssparse(ensemble, s.sparsexy, s.npoints, ref rep.relclserror, ref rep.avgce, ref rep.rmserror, ref rep.avgerror, ref rep.avgrelerror);
}
}
