/*************************************************************************
This function trains neural network passed to this function, using current
dataset (one which was passed to MLPSetDataset() or MLPSetSparseDataset())
and current training settings. Training from NRestarts random starting
positions is performed, best network is chosen.
Training is performed using current training algorithm.
INPUT PARAMETERS:
S - trainer object;
Network - neural network. It must have same number of inputs and
output/classes as was specified during creation of the
trainer object;
TNetwork - the training neural network.
User may look weights in parameter Network while
continue training process.
It has architecture like Network. You have to copy or
create new network with architecture like Network.
State - created LBFGS optimizer;
NRestarts - number of restarts, >=0:
* NRestarts>0 means that specified number of random
restarts are performed, best network is chosen after
training
* NRestarts=0 means that current state of the network
is used for training.
TrnSubset - some subset from training set(it stores row's numbers),
used as trainig set;
TrnSubsetSize- size of subset(if TrnSubsetSize<0 - used full dataset);
when TrnSubsetSize=0, network is filled by zero value,
and ValSubset parameter is IGNORED;
ValSubset - some subset from training set(it stores row's numbers),
used as validation set;
ValSubsetSize- size of subset(if ValSubsetSize<0 - used full dataset);
when ValSubsetSize<>0 this mean that is used early
stopping training algorithm;
BufWBest - buffer for storing interim resuls (BufWBest[0:WCOunt-1]
it has be allocated by user);
BufWFinal - buffer for storing interim resuls(BufWFinal[0:WCOunt-1]
it has be allocated by user).
OUTPUT PARAMETERS:
Network - trained network;
Rep - training report.
NOTE: when no dataset was specified with MLPSetDataset/SetSparseDataset(),
network is filled by zero values. Same behavior for functions
MLPStartTraining and MLPContinueTraining.
NOTE: this method uses sum-of-squares error function for training.
-- ALGLIB --
Copyright 13.08.2012 by Bochkanov Sergey
*************************************************************************/
private static void mlptrainnetworkx(mlptrainer s,
mlpbase.multilayerperceptron network,
mlpbase.multilayerperceptron tnetwork,
minlbfgs.minlbfgsstate state,
int nrestarts,
int[] trnsubset,
int trnsubsetsize,
int[] valsubset,
int valsubsetsize,
double[] bufwbest,
double[] bufwfinal,
mlpreport rep)
{
mlpbase.modelerrors modrep = new mlpbase.modelerrors();
double eval = 0;
double v = 0;
double ebestcur = 0;
double efinal = 0;
int ngradbatch = 0;
int nin = 0;
int nout = 0;
int wcount = 0;
int twcount = 0;
int itbest = 0;
int itcnt = 0;
int ntype = 0;
int ttype = 0;
bool rndstart = new bool();
int pass = 0;
int i = 0;
int i_ = 0;
alglib.ap.assert(s.npoints>=0, "MLPTrainNetworkX: internal error - parameter S is not initialized or is spoiled(S.NPoints<0)");
if( s.rcpar )
{
ttype = 0;
}
else
{
ttype = 1;
}
if( !mlpbase.mlpissoftmax(network) )
{
ntype = 0;
}
else
{
ntype = 1;
}
alglib.ap.assert(ntype==ttype, "MLPTrainNetworkX: internal error - type of the resulting network is not similar to network type in trainer object");
if( !mlpbase.mlpissoftmax(tnetwork) )
{
ntype = 0;
}
else
{
ntype = 1;
}
alglib.ap.assert(ntype==ttype, "MLPTrainNetworkX: internal error - type of the training network is not similar to network type in trainer object");
mlpbase.mlpproperties(network, ref nin, ref nout, ref wcount);
alglib.ap.assert(s.nin==nin, "MLPTrainNetworkX: internal error - number of inputs in trainer is not equal to number of inputs in the network.");
alglib.ap.assert(s.nout==nout, "MLPTrainNetworkX: internal error - number of outputs in trainer is not equal to number of outputs in the network.");
mlpbase.mlpproperties(tnetwork, ref nin, ref nout, ref twcount);
alglib.ap.assert(s.nin==nin, "MLPTrainNetworkX: internal error - number of inputs in trainer is not equal to number of inputs in the training network.");
alglib.ap.assert(s.nout==nout, "MLPTrainNetworkX: internal error - number of outputs in trainer is not equal to number of outputs in the training network.");
alglib.ap.assert(twcount==wcount, "MLPTrainNetworkX: internal error - number of weights the resulting network is not equal to number of weights in the training network.");
alglib.ap.assert(nrestarts>=0, "MLPTrainNetworkX: internal error - NRestarts<0.");
alglib.ap.assert(alglib.ap.len(trnsubset)>=trnsubsetsize, "MLPTrainNetworkX: internal error - parameter TrnSubsetSize more than input subset size(Length(TrnSubset)<TrnSubsetSize)");
for(i=0; i<=trnsubsetsize-1; i++)
{
alglib.ap.assert(trnsubset[i]>=0 && trnsubset[i]<=s.npoints-1, "MLPTrainNetworkX: internal error - parameter TrnSubset contains incorrect index(TrnSubset[I]<0 or TrnSubset[I]>S.NPoints-1)");
}
alglib.ap.assert(alglib.ap.len(valsubset)>=valsubsetsize, "MLPTrainNetworkX: internal error - parameter ValSubsetSize more than input subset size(Length(ValSubset)<ValSubsetSize)");
for(i=0; i<=valsubsetsize-1; i++)
{
alglib.ap.assert(valsubset[i]>=0 && valsubset[i]<=s.npoints-1, "MLPTrainNetworkX: internal error - parameter ValSubset contains incorrect index(ValSubset[I]<0 or ValSubset[I]>S.NPoints-1)");
}
//
// Initialize parameter Rep
//
rep.relclserror = 0;
rep.avgce = 0;
rep.rmserror = 0;
rep.avgerror = 0;
rep.avgrelerror = 0;
rep.ngrad = 0;
rep.nhess = 0;
rep.ncholesky = 0;
if( ((s.datatype==0 || s.datatype==1) && s.npoints>0) && trnsubsetsize!=0 )
{
//
// Prepare
//
efinal = math.maxrealnumber;
if( nrestarts!=0 )
{
rndstart = true;
}
else
{
rndstart = false;
nrestarts = 1;
}
ngradbatch = 0;
eval = 0;
ebestcur = 0;
for(pass=1; pass<=nrestarts; pass++)
{
mlpstarttrainingx(s, network, tnetwork, state, rndstart, trnsubset, trnsubsetsize);
itbest = 0;
itcnt = 0;
if( s.datatype==0 )
{
ebestcur = mlpbase.mlperrorsubset(network, s.densexy, s.npoints, valsubset, valsubsetsize);
}
if( s.datatype==1 )
{
ebestcur = mlpbase.mlperrorsparsesubset(network, s.sparsexy, s.npoints, valsubset, valsubsetsize);
}
for(i_=0; i_<=wcount-1;i_++)
{
bufwbest[i_] = network.weights[i_];
}
while( mlpcontinuetrainingx(s, network, tnetwork, state, trnsubset, trnsubsetsize, ref ngradbatch) )
{
if( s.datatype==0 )
{
eval = mlpbase.mlperrorsubset(network, s.densexy, s.npoints, valsubset, valsubsetsize);
}
if( s.datatype==1 )
{
eval = mlpbase.mlperrorsparsesubset(network, s.sparsexy, s.npoints, valsubset, valsubsetsize);
}
if( (double)(eval)<=(double)(ebestcur) )
{
for(i_=0; i_<=wcount-1;i_++)
{
bufwbest[i_] = network.weights[i_];
}
ebestcur = eval;
itbest = itcnt;
}
if( itcnt>30 && (double)(itcnt)>(double)(1.5*itbest) )
{
break;
}
itcnt = itcnt+1;
}
for(i_=0; i_<=wcount-1;i_++)
{
network.weights[i_] = bufwbest[i_];
}
//
// Compare with final(the best) answer.
//
v = 0.0;
for(i_=0; i_<=wcount-1;i_++)
{
v += bufwbest[i_]*bufwbest[i_];
}
if( s.datatype==0 )
{
ebestcur = mlpbase.mlperrorsubset(network, s.densexy, s.npoints, trnsubset, trnsubsetsize)+0.5*s.decay*v;
}
if( s.datatype==1 )
{
ebestcur = mlpbase.mlperrorsparsesubset(network, s.sparsexy, s.npoints, trnsubset, trnsubsetsize)+0.5*s.decay*v;
}
if( (double)(ebestcur)<(double)(efinal) )
{
for(i_=0; i_<=wcount-1;i_++)
{
bufwfinal[i_] = bufwbest[i_];
}
efinal = ebestcur;
}
}
//
// Final network
//
for(i_=0; i_<=wcount-1;i_++)
{
network.weights[i_] = bufwfinal[i_];
}
rep.ngrad = ngradbatch;
}
else
{
for(i=0; i<=wcount-1; i++)
{
network.weights[i] = 0;
}
}
//
// Calculate errors.
//
if( s.datatype==0 )
{
mlpbase.mlpallerrorssubset(network, s.densexy, s.npoints, trnsubset, trnsubsetsize, modrep);
}
if( s.datatype==1 )
{
mlpbase.mlpallerrorssparsesubset(network, s.sparsexy, s.npoints, trnsubset, trnsubsetsize, modrep);
}
rep.relclserror = modrep.relclserror;
rep.avgce = modrep.avgce;
rep.rmserror = modrep.rmserror;
rep.avgerror = modrep.avgerror;
rep.avgrelerror = modrep.avgrelerror;
}
/*************************************************************************
This function initializes temporaries needed for training session.
-- ALGLIB --
Copyright 01.07.2013 by Bochkanov Sergey
*************************************************************************/
private static void initmlptrnsession(mlpbase.multilayerperceptron networktrained,
bool randomizenetwork,
mlptrainer trainer,
smlptrnsession session)
{
int nin = 0;
int nout = 0;
int wcount = 0;
int pcount = 0;
int[] dummysubset = new int[0];
//
// Prepare network:
// * copy input network to Session.Network
// * re-initialize preprocessor and weights if RandomizeNetwork=True
//
mlpbase.mlpcopy(networktrained, session.network);
if( randomizenetwork )
{
alglib.ap.assert(trainer.datatype==0 || trainer.datatype==1, "InitTemporaries: unexpected Trainer.DataType");
if( trainer.datatype==0 )
{
mlpbase.mlpinitpreprocessorsubset(session.network, trainer.densexy, trainer.npoints, dummysubset, -1);
}
if( trainer.datatype==1 )
{
mlpbase.mlpinitpreprocessorsparsesubset(session.network, trainer.sparsexy, trainer.npoints, dummysubset, -1);
}
mlpbase.mlprandomize(session.network);
session.randomizenetwork = true;
}
else
{
session.randomizenetwork = false;
}
//
// Determine network geometry and initialize optimizer
//
mlpbase.mlpproperties(session.network, ref nin, ref nout, ref wcount);
minlbfgs.minlbfgscreate(wcount, Math.Min(wcount, trainer.lbfgsfactor), session.network.weights, session.optimizer);
minlbfgs.minlbfgssetxrep(session.optimizer, true);
//
// Create buffers
//
session.wbuf0 = new double[wcount];
session.wbuf1 = new double[wcount];
//
// Initialize session result
//
mlpbase.mlpexporttunableparameters(session.network, ref session.bestparameters, ref pcount);
session.bestrmserror = math.maxrealnumber;
}
/*************************************************************************
This function initializes temporaries needed for ensemble training.
*************************************************************************/
private static void initmlpetrnsession(mlpbase.multilayerperceptron individualnetwork,
mlptrainer trainer,
mlpetrnsession session)
{
int[] dummysubset = new int[0];
//
// Prepare network:
// * copy input network to Session.Network
// * re-initialize preprocessor and weights if RandomizeNetwork=True
//
mlpbase.mlpcopy(individualnetwork, session.network);
initmlptrnsessions(individualnetwork, true, trainer, session.mlpsessions);
apserv.ivectorsetlengthatleast(ref session.trnsubset, trainer.npoints);
apserv.ivectorsetlengthatleast(ref session.valsubset, trainer.npoints);
}
/*************************************************************************
This function trains neural network passed to this function, using current
dataset (one which was passed to MLPSetDataset() or MLPSetSparseDataset())
and current training settings. Training from NRestarts random starting
positions is performed, best network is chosen.
This function is inteded to be used internally. It may be used in several
settings:
* training with ValSubsetSize=0, corresponds to "normal" training with
termination criteria based on S.MaxIts (steps count) and S.WStep (step
size). Training sample is given by TrnSubset/TrnSubsetSize.
* training with ValSubsetSize>0, corresponds to early stopping training
with additional MaxIts/WStep stopping criteria. Training sample is given
by TrnSubset/TrnSubsetSize, validation sample is given by ValSubset/
ValSubsetSize.
-- ALGLIB --
Copyright 13.08.2012 by Bochkanov Sergey
*************************************************************************/
private static void mlptrainnetworkx(mlptrainer s,
int nrestarts,
int algokind,
int[] trnsubset,
int trnsubsetsize,
int[] valsubset,
int valsubsetsize,
mlpbase.multilayerperceptron network,
mlpreport rep,
bool isrootcall,
alglib.smp.shared_pool sessions)
{
mlpbase.modelerrors modrep = new mlpbase.modelerrors();
double eval = 0;
double ebest = 0;
int ngradbatch = 0;
int nin = 0;
int nout = 0;
int wcount = 0;
int pcount = 0;
int itbest = 0;
int itcnt = 0;
int ntype = 0;
int ttype = 0;
bool rndstart = new bool();
int i = 0;
int nr0 = 0;
int nr1 = 0;
mlpreport rep0 = new mlpreport();
mlpreport rep1 = new mlpreport();
bool randomizenetwork = new bool();
double bestrmserror = 0;
smlptrnsession psession = null;
int i_ = 0;
mlpbase.mlpproperties(network, ref nin, ref nout, ref wcount);
//
// Process root call
//
if( isrootcall )
{
//
// Check correctness of parameters
//
alglib.ap.assert(algokind==0 || algokind==-1, "MLPTrainNetworkX: unexpected AlgoKind");
alglib.ap.assert(s.npoints>=0, "MLPTrainNetworkX: internal error - parameter S is not initialized or is spoiled(S.NPoints<0)");
if( s.rcpar )
{
ttype = 0;
}
else
{
ttype = 1;
}
if( !mlpbase.mlpissoftmax(network) )
{
ntype = 0;
}
else
{
ntype = 1;
}
alglib.ap.assert(ntype==ttype, "MLPTrainNetworkX: internal error - type of the training network is not similar to network type in trainer object");
alglib.ap.assert(s.nin==nin, "MLPTrainNetworkX: internal error - number of inputs in trainer is not equal to number of inputs in the training network.");
alglib.ap.assert(s.nout==nout, "MLPTrainNetworkX: internal error - number of outputs in trainer is not equal to number of outputs in the training network.");
alglib.ap.assert(nrestarts>=0, "MLPTrainNetworkX: internal error - NRestarts<0.");
alglib.ap.assert(alglib.ap.len(trnsubset)>=trnsubsetsize, "MLPTrainNetworkX: internal error - parameter TrnSubsetSize more than input subset size(Length(TrnSubset)<TrnSubsetSize)");
for(i=0; i<=trnsubsetsize-1; i++)
{
alglib.ap.assert(trnsubset[i]>=0 && trnsubset[i]<=s.npoints-1, "MLPTrainNetworkX: internal error - parameter TrnSubset contains incorrect index(TrnSubset[I]<0 or TrnSubset[I]>S.NPoints-1)");
}
alglib.ap.assert(alglib.ap.len(valsubset)>=valsubsetsize, "MLPTrainNetworkX: internal error - parameter ValSubsetSize more than input subset size(Length(ValSubset)<ValSubsetSize)");
for(i=0; i<=valsubsetsize-1; i++)
{
alglib.ap.assert(valsubset[i]>=0 && valsubset[i]<=s.npoints-1, "MLPTrainNetworkX: internal error - parameter ValSubset contains incorrect index(ValSubset[I]<0 or ValSubset[I]>S.NPoints-1)");
}
//
// Train
//
randomizenetwork = nrestarts>0;
initmlptrnsessions(network, randomizenetwork, s, sessions);
mlptrainnetworkx(s, nrestarts, algokind, trnsubset, trnsubsetsize, valsubset, valsubsetsize, network, rep, false, sessions);
//
// Choose best network
//
bestrmserror = math.maxrealnumber;
alglib.smp.ae_shared_pool_first_recycled(sessions, ref psession);
while( psession!=null )
{
if( (double)(psession.bestrmserror)<(double)(bestrmserror) )
{
mlpbase.mlpimporttunableparameters(network, psession.bestparameters);
bestrmserror = psession.bestrmserror;
}
alglib.smp.ae_shared_pool_next_recycled(sessions, ref psession);
}
//
// Calculate errors
//
if( s.datatype==0 )
{
mlpbase.mlpallerrorssubset(network, s.densexy, s.npoints, trnsubset, trnsubsetsize, modrep);
}
if( s.datatype==1 )
{
mlpbase.mlpallerrorssparsesubset(network, s.sparsexy, s.npoints, trnsubset, trnsubsetsize, modrep);
}
rep.relclserror = modrep.relclserror;
rep.avgce = modrep.avgce;
rep.rmserror = modrep.rmserror;
rep.avgerror = modrep.avgerror;
rep.avgrelerror = modrep.avgrelerror;
//
// Done
//
return;
}
//
// Split problem, if we have more than 1 restart
//
if( nrestarts>=2 )
{
//
// Divide problem with NRestarts into two: NR0 and NR1.
//
nr0 = nrestarts/2;
nr1 = nrestarts-nr0;
mlptrainnetworkx(s, nr0, algokind, trnsubset, trnsubsetsize, valsubset, valsubsetsize, network, rep0, false, sessions);
mlptrainnetworkx(s, nr1, algokind, trnsubset, trnsubsetsize, valsubset, valsubsetsize, network, rep1, false, sessions);
//
// Aggregate results
//
rep.ngrad = rep0.ngrad+rep1.ngrad;
rep.nhess = rep0.nhess+rep1.nhess;
rep.ncholesky = rep0.ncholesky+rep1.ncholesky;
//
// Done :)
//
return;
}
//
// Execution with NRestarts=1 or NRestarts=0:
// * NRestarts=1 means that network is restarted from random position
// * NRestarts=0 means that network is not randomized
//
alglib.ap.assert(nrestarts==0 || nrestarts==1, "MLPTrainNetworkX: internal error");
rep.ngrad = 0;
rep.nhess = 0;
rep.ncholesky = 0;
alglib.smp.ae_shared_pool_retrieve(sessions, ref psession);
if( ((s.datatype==0 || s.datatype==1) && s.npoints>0) && trnsubsetsize!=0 )
{
//
// Train network using combination of early stopping and step-size
// and step-count based criteria. Network state with best value of
// validation set error is stored in WBuf0. When validation set is
// zero, most recent state of network is stored.
//
rndstart = nrestarts!=0;
ngradbatch = 0;
eval = 0;
ebest = 0;
itbest = 0;
itcnt = 0;
mlpstarttrainingx(s, rndstart, algokind, trnsubset, trnsubsetsize, psession);
if( s.datatype==0 )
{
ebest = mlpbase.mlperrorsubset(psession.network, s.densexy, s.npoints, valsubset, valsubsetsize);
}
if( s.datatype==1 )
{
ebest = mlpbase.mlperrorsparsesubset(psession.network, s.sparsexy, s.npoints, valsubset, valsubsetsize);
}
for(i_=0; i_<=wcount-1;i_++)
{
psession.wbuf0[i_] = psession.network.weights[i_];
}
while( mlpcontinuetrainingx(s, trnsubset, trnsubsetsize, ref ngradbatch, psession) )
{
if( s.datatype==0 )
{
eval = mlpbase.mlperrorsubset(psession.network, s.densexy, s.npoints, valsubset, valsubsetsize);
}
if( s.datatype==1 )
{
eval = mlpbase.mlperrorsparsesubset(psession.network, s.sparsexy, s.npoints, valsubset, valsubsetsize);
}
if( (double)(eval)<=(double)(ebest) || valsubsetsize==0 )
{
for(i_=0; i_<=wcount-1;i_++)
{
psession.wbuf0[i_] = psession.network.weights[i_];
}
ebest = eval;
itbest = itcnt;
}
if( itcnt>30 && (double)(itcnt)>(double)(1.5*itbest) )
{
break;
}
itcnt = itcnt+1;
}
for(i_=0; i_<=wcount-1;i_++)
{
psession.network.weights[i_] = psession.wbuf0[i_];
}
rep.ngrad = ngradbatch;
}
else
{
for(i=0; i<=wcount-1; i++)
{
psession.network.weights[i] = 0;
}
}
//
// Evaluate network performance and update PSession.BestParameters/BestRMSError
// (if needed).
//
if( s.datatype==0 )
{
mlpbase.mlpallerrorssubset(psession.network, s.densexy, s.npoints, trnsubset, trnsubsetsize, modrep);
}
if( s.datatype==1 )
{
mlpbase.mlpallerrorssparsesubset(psession.network, s.sparsexy, s.npoints, trnsubset, trnsubsetsize, modrep);
}
if( (double)(modrep.rmserror)<(double)(psession.bestrmserror) )
{
mlpbase.mlpexporttunableparameters(psession.network, ref psession.bestparameters, ref pcount);
psession.bestrmserror = modrep.rmserror;
}
//
// Move session back to pool
//
alglib.smp.ae_shared_pool_recycle(sessions, ref psession);
}
/*************************************************************************
This function performs step-by-step training of the neural network. Here
"step-by-step" means that training starts with MLPStartTrainingX call,
and then user subsequently calls MLPContinueTrainingX to perform one more
iteration of the training.
After call to this function trainer object remembers network and is ready
to train it. However, no training is performed until first call to
MLPContinueTraining() function. Subsequent calls to MLPContinueTraining()
will advance traing progress one iteration further.
-- ALGLIB --
Copyright 13.08.2012 by Bochkanov Sergey
*************************************************************************/
private static void mlpstarttrainingx(mlptrainer s,
bool randomstart,
int algokind,
int[] subset,
int subsetsize,
smlptrnsession session)
{
int nin = 0;
int nout = 0;
int wcount = 0;
int ntype = 0;
int ttype = 0;
int i = 0;
//
// Check parameters
//
alglib.ap.assert(s.npoints>=0, "MLPStartTrainingX: internal error - parameter S is not initialized or is spoiled(S.NPoints<0)");
alglib.ap.assert(algokind==0 || algokind==-1, "MLPStartTrainingX: unexpected AlgoKind");
if( s.rcpar )
{
ttype = 0;
}
else
{
ttype = 1;
}
if( !mlpbase.mlpissoftmax(session.network) )
{
ntype = 0;
}
else
{
ntype = 1;
}
alglib.ap.assert(ntype==ttype, "MLPStartTrainingX: internal error - type of the resulting network is not similar to network type in trainer object");
mlpbase.mlpproperties(session.network, ref nin, ref nout, ref wcount);
alglib.ap.assert(s.nin==nin, "MLPStartTrainingX: number of inputs in trainer is not equal to number of inputs in the network.");
alglib.ap.assert(s.nout==nout, "MLPStartTrainingX: number of outputs in trainer is not equal to number of outputs in the network.");
alglib.ap.assert(alglib.ap.len(subset)>=subsetsize, "MLPStartTrainingX: internal error - parameter SubsetSize more than input subset size(Length(Subset)<SubsetSize)");
for(i=0; i<=subsetsize-1; i++)
{
alglib.ap.assert(subset[i]>=0 && subset[i]<=s.npoints-1, "MLPStartTrainingX: internal error - parameter Subset contains incorrect index(Subset[I]<0 or Subset[I]>S.NPoints-1)");
}
//
// Prepare session
//
minlbfgs.minlbfgssetcond(session.optimizer, 0.0, 0.0, s.wstep, s.maxits);
if( s.npoints>0 && subsetsize!=0 )
{
if( randomstart )
{
mlpbase.mlprandomize(session.network);
}
minlbfgs.minlbfgsrestartfrom(session.optimizer, session.network.weights);
}
else
{
for(i=0; i<=wcount-1; i++)
{
session.network.weights[i] = 0;
}
}
if( algokind==-1 )
{
session.algoused = s.algokind;
if( s.algokind==1 )
{
session.minibatchsize = s.minibatchsize;
}
}
else
{
session.algoused = 0;
}
hqrnd.hqrndrandomize(session.generator);
session.rstate.ia = new int[15+1];
session.rstate.ra = new double[1+1];
session.rstate.stage = -1;
}
/*************************************************************************
Single-threaded stub. HPC ALGLIB replaces it by multithreaded code.
*************************************************************************/
public static bool _pexec_mlpcontinuetraining(mlptrainer s,
mlpbase.multilayerperceptron network)
{
return mlpcontinuetraining(s,network);
}
/*************************************************************************
Single-threaded stub. HPC ALGLIB replaces it by multithreaded code.
*************************************************************************/
public static void _pexec_mlptrainensemblees(mlptrainer s,
mlpe.mlpensemble ensemble,
int nrestarts,
mlpreport rep)
{
mlptrainensemblees(s,ensemble,nrestarts,rep);
}
/*************************************************************************
Creation of the network trainer object for regression networks
INPUT PARAMETERS:
NIn - number of inputs, NIn>=1
NOut - number of outputs, NOut>=1
OUTPUT PARAMETERS:
S - neural network trainer object.
This structure can be used to train any regression
network with NIn inputs and NOut outputs.
-- ALGLIB --
Copyright 23.07.2012 by Bochkanov Sergey
*************************************************************************/
public static void mlpcreatetrainer(int nin,
int nout,
mlptrainer s)
{
alglib.ap.assert(nin>=1, "MLPCreateTrainer: NIn<1.");
alglib.ap.assert(nout>=1, "MLPCreateTrainer: NOut<1.");
s.nin = nin;
s.nout = nout;
s.rcpar = true;
s.lbfgsfactor = defaultlbfgsfactor;
s.decay = 1.0E-6;
mlpsetcond(s, 0, 0);
s.datatype = 0;
s.npoints = 0;
mlpsetalgobatch(s);
}
/*************************************************************************
Creation of the network trainer object for classification networks
INPUT PARAMETERS:
NIn - number of inputs, NIn>=1
NClasses - number of classes, NClasses>=2
OUTPUT PARAMETERS:
S - neural network trainer object.
This structure can be used to train any classification
network with NIn inputs and NOut outputs.
-- ALGLIB --
Copyright 23.07.2012 by Bochkanov Sergey
*************************************************************************/
public static void mlpcreatetrainercls(int nin,
int nclasses,
mlptrainer s)
{
alglib.ap.assert(nin>=1, "MLPCreateTrainerCls: NIn<1.");
alglib.ap.assert(nclasses>=2, "MLPCreateTrainerCls: NClasses<2.");
s.nin = nin;
s.nout = nclasses;
s.rcpar = false;
s.lbfgsfactor = defaultlbfgsfactor;
s.decay = 1.0E-6;
mlpsetcond(s, 0, 0);
s.datatype = 0;
s.npoints = 0;
mlpsetalgobatch(s);
}
/*************************************************************************
This function estimates generalization error using cross-validation on the
current dataset with current training settings.
FOR USERS OF COMMERCIAL EDITION:
! Commercial version of ALGLIB includes two important improvements of
! this function:
! * multicore support (C++ and C# computational cores)
! * SSE support (C++ computational core)
!
! Second improvement gives constant speedup (2-3X). First improvement
! gives close-to-linear speedup on multicore systems. Following
! operations can be executed in parallel:
! * FoldsCount cross-validation rounds (always)
! * NRestarts training sessions performed within each of
! cross-validation rounds (if NRestarts>1)
! * gradient calculation over large dataset (if dataset is large enough)
!
! In order to use multicore features you have to:
! * use commercial version of ALGLIB
! * call this function with "smp_" prefix, which indicates that
! multicore code will be used (for multicore support)
!
! In order to use SSE features you have to:
! * use commercial version of ALGLIB on Intel processors
! * use C++ computational core
!
! This note is given for users of commercial edition; if you use GPL
! edition, you still will be able to call smp-version of this function,
! but all computations will be done serially.
!
! We recommend you to carefully read ALGLIB Reference Manual, section
! called 'SMP support', before using parallel version of this function.
INPUT PARAMETERS:
S - trainer object
Network - neural network. It must have same number of inputs and
output/classes as was specified during creation of the
trainer object. Network is not changed during cross-
validation and is not trained - it is used only as
representative of its architecture. I.e., we estimate
generalization properties of ARCHITECTURE, not some
specific network.
NRestarts - number of restarts, >=0:
* NRestarts>0 means that for each cross-validation
round specified number of random restarts is
performed, with best network being chosen after
training.
* NRestarts=0 is same as NRestarts=1
FoldsCount - number of folds in k-fold cross-validation:
* 2<=FoldsCount<=size of dataset
* recommended value: 10.
* values larger than dataset size will be silently
truncated down to dataset size
OUTPUT PARAMETERS:
Rep - structure which contains cross-validation estimates:
* Rep.RelCLSError - fraction of misclassified cases.
* Rep.AvgCE - acerage cross-entropy
* Rep.RMSError - root-mean-square error
* Rep.AvgError - average error
* Rep.AvgRelError - average relative error
NOTE: when no dataset was specified with MLPSetDataset/SetSparseDataset(),
or subset with only one point was given, zeros are returned as
estimates.
NOTE: this method performs FoldsCount cross-validation rounds, each one
with NRestarts random starts. Thus, FoldsCount*NRestarts networks
are trained in total.
NOTE: Rep.RelCLSError/Rep.AvgCE are zero on regression problems.
NOTE: on classification problems Rep.RMSError/Rep.AvgError/Rep.AvgRelError
contain errors in prediction of posterior probabilities.
-- ALGLIB --
Copyright 23.07.2012 by Bochkanov Sergey
*************************************************************************/
public static void mlpkfoldcv(mlptrainer s,
mlpbase.multilayerperceptron network,
int nrestarts,
int foldscount,
mlpreport rep)
{
alglib.smp.shared_pool pooldatacv = new alglib.smp.shared_pool();
mlpparallelizationcv datacv = new mlpparallelizationcv();
mlpparallelizationcv sdatacv = null;
double[,] cvy = new double[0,0];
int[] folds = new int[0];
double[] buf = new double[0];
double[] dy = new double[0];
int nin = 0;
int nout = 0;
int wcount = 0;
int rowsize = 0;
int ntype = 0;
int ttype = 0;
int i = 0;
int j = 0;
int k = 0;
hqrnd.hqrndstate rs = new hqrnd.hqrndstate();
int i_ = 0;
int i1_ = 0;
if( !mlpbase.mlpissoftmax(network) )
{
ntype = 0;
}
else
{
ntype = 1;
}
if( s.rcpar )
{
ttype = 0;
}
else
{
ttype = 1;
}
alglib.ap.assert(ntype==ttype, "MLPKFoldCV: type of input network is not similar to network type in trainer object");
alglib.ap.assert(s.npoints>=0, "MLPKFoldCV: possible trainer S is not initialized(S.NPoints<0)");
mlpbase.mlpproperties(network, ref nin, ref nout, ref wcount);
alglib.ap.assert(s.nin==nin, "MLPKFoldCV: number of inputs in trainer is not equal to number of inputs in network");
alglib.ap.assert(s.nout==nout, "MLPKFoldCV: number of outputs in trainer is not equal to number of outputs in network");
alglib.ap.assert(nrestarts>=0, "MLPKFoldCV: NRestarts<0");
alglib.ap.assert(foldscount>=2, "MLPKFoldCV: FoldsCount<2");
if( foldscount>s.npoints )
{
foldscount = s.npoints;
}
rep.relclserror = 0;
rep.avgce = 0;
rep.rmserror = 0;
rep.avgerror = 0;
rep.avgrelerror = 0;
hqrnd.hqrndrandomize(rs);
rep.ngrad = 0;
rep.nhess = 0;
rep.ncholesky = 0;
if( s.npoints==0 || s.npoints==1 )
{
return;
}
//
// Read network geometry, test parameters
//
if( s.rcpar )
{
rowsize = nin+nout;
dy = new double[nout];
bdss.dserrallocate(-nout, ref buf);
}
else
{
rowsize = nin+1;
dy = new double[1];
bdss.dserrallocate(nout, ref buf);
}
//
// Folds
//
folds = new int[s.npoints];
for(i=0; i<=s.npoints-1; i++)
{
folds[i] = i*foldscount/s.npoints;
}
for(i=0; i<=s.npoints-2; i++)
{
j = i+hqrnd.hqrnduniformi(rs, s.npoints-i);
if( j!=i )
{
k = folds[i];
folds[i] = folds[j];
folds[j] = k;
}
}
cvy = new double[s.npoints, nout];
//
// Initialize SEED-value for shared pool
//
datacv.ngrad = 0;
mlpbase.mlpcopy(network, datacv.network);
datacv.subset = new int[s.npoints];
datacv.xyrow = new double[rowsize];
datacv.y = new double[nout];
//
// Create shared pool
//
alglib.smp.ae_shared_pool_set_seed(pooldatacv, datacv);
//
// Parallelization
//
mthreadcv(s, rowsize, nrestarts, folds, 0, foldscount, cvy, pooldatacv);
//
// Calculate value for NGrad
//
alglib.smp.ae_shared_pool_first_recycled(pooldatacv, ref sdatacv);
while( sdatacv!=null )
{
rep.ngrad = rep.ngrad+sdatacv.ngrad;
alglib.smp.ae_shared_pool_next_recycled(pooldatacv, ref sdatacv);
}
//
// Connect of results and calculate cross-validation error
//
for(i=0; i<=s.npoints-1; i++)
{
if( s.datatype==0 )
{
for(i_=0; i_<=rowsize-1;i_++)
{
datacv.xyrow[i_] = s.densexy[i,i_];
}
}
if( s.datatype==1 )
{
sparse.sparsegetrow(s.sparsexy, i, ref datacv.xyrow);
}
for(i_=0; i_<=nout-1;i_++)
{
datacv.y[i_] = cvy[i,i_];
}
if( s.rcpar )
{
i1_ = (nin) - (0);
for(i_=0; i_<=nout-1;i_++)
{
dy[i_] = datacv.xyrow[i_+i1_];
}
}
else
{
dy[0] = datacv.xyrow[nin];
}
bdss.dserraccumulate(ref buf, datacv.y, dy);
}
bdss.dserrfinish(ref buf);
rep.relclserror = buf[0];
rep.avgce = buf[1];
rep.rmserror = buf[2];
rep.avgerror = buf[3];
rep.avgrelerror = buf[4];
}
/*************************************************************************
Single-threaded stub. HPC ALGLIB replaces it by multithreaded code.
*************************************************************************/
public static void _pexec_mlpkfoldcv(mlptrainer s,
mlpbase.multilayerperceptron network,
int nrestarts,
int foldscount,
mlpreport rep)
{
mlpkfoldcv(s,network,nrestarts,foldscount,rep);
}
public override alglib.apobject make_copy()
{
mlptrainer _result = new mlptrainer();
_result.nin = nin;
_result.nout = nout;
_result.rcpar = rcpar;
_result.lbfgsfactor = lbfgsfactor;
_result.decay = decay;
_result.wstep = wstep;
_result.maxits = maxits;
_result.datatype = datatype;
_result.npoints = npoints;
_result.densexy = (double[,])densexy.Clone();
_result.sparsexy = (sparse.sparsematrix)sparsexy.make_copy();
_result.session = (smlptrnsession)session.make_copy();
_result.ngradbatch = ngradbatch;
_result.subset = (int[])subset.Clone();
_result.subsetsize = subsetsize;
_result.valsubset = (int[])valsubset.Clone();
_result.valsubsetsize = valsubsetsize;
_result.algokind = algokind;
_result.minibatchsize = minibatchsize;
return _result;
}
/*************************************************************************
This function performs step-by-step training of the neural network. Here
"step-by-step" means that training starts with MLPStartTrainingX call,
and then user subsequently calls MLPContinueTrainingX to perform one more
iteration of the training.
This function performs one more iteration of the training and returns
either True (training continues) or False (training stopped). In case True
was returned, Network weights are updated according to the current state
of the optimization progress. In case False was returned, no additional
updates is performed (previous update of the network weights moved us to
the final point, and no additional updates is needed).
EXAMPLE:
>
> [initialize network and trainer object]
>
> MLPStartTraining(Trainer, Network, True)
> while MLPContinueTraining(Trainer, Network) do
> [visualize training progress]
>
INPUT PARAMETERS:
S - trainer object
Network - neural network which receives A COPY of the actual
network which is trained by the algorithm. After each
training roung state of the network being trained is
copied to this variable.
It must have same number of inputs and output/classes
as was specified during creation of the trainer object
and it must have exactly same architecture as the
second network (TNetwork).
TNetwork - neural network being trained.
State - LBFGS optimizer, already initialized, number of
dimensions must be equal to number of weights in the
networks.
Subset - some subset from training set(it stores row's numbers);
SubsetSize - size of subset(if SubsetSize<0 - used full dataset).
NGradBatch - number of calls MLPGradBatch function. Initial value
is zero;
OUTPUT PARAMETERS:
Network - weights of the neural network are rewritten by the
current approximation;
NGradBatch - number of calls MLPGradBatch function after training.
NOTE: this method uses sum-of-squares error function for training.
NOTE: it is expected that trainer object settings are NOT changed during
step-by-step training, i.e. no one changes stopping criteria or
training set during training. It is possible and there is no defense
against such actions, but algorithm behavior in such cases is
undefined and can be unpredictable.
NOTE: It is expected that Network is the same one which was passed to
MLPStartTraining() function. However, THIS function checks only
following:
* that number of network inputs is consistent with trainer object
settings
* that number of network outputs/classes is consistent with trainer
object settings
* that number of network weights is the same as number of weights in
the network passed to MLPStartTraining() function
Exception is thrown when these conditions are violated.
It is also expected that you do not change state of the network on
your own - the only party who has right to change network during its
training is a trainer object. Any attempt to interfere with trainer
may lead to unpredictable results.
-- ALGLIB --
Copyright 13.08.2012 by Bochkanov Sergey
*************************************************************************/
private static bool mlpcontinuetrainingx(mlptrainer s,
mlpbase.multilayerperceptron network,
mlpbase.multilayerperceptron tnetwork,
minlbfgs.minlbfgsstate state,
int[] subset,
int subsetsize,
ref int ngradbatch)
{
bool result = new bool();
int nin = 0;
int nout = 0;
int wcount = 0;
int twcount = 0;
int ntype = 0;
int ttype = 0;
double decay = 0;
double v = 0;
int i = 0;
int i_ = 0;
alglib.ap.assert(s.npoints>=0, "MLPContinueTrainingX: internal error - parameter S is not initialized or is spoiled(S.NPoints<0).");
if( s.rcpar )
{
ttype = 0;
}
else
{
ttype = 1;
}
if( !mlpbase.mlpissoftmax(network) )
{
ntype = 0;
}
else
{
ntype = 1;
}
alglib.ap.assert(ntype==ttype, "MLPContinueTrainingX: internal error - type of the resulting network is not similar to network type in trainer object.");
if( !mlpbase.mlpissoftmax(tnetwork) )
{
ntype = 0;
}
else
{
ntype = 1;
}
alglib.ap.assert(ntype==ttype, "MLPContinueTrainingX: internal error - type of the training network is not similar to network type in trainer object.");
mlpbase.mlpproperties(network, ref nin, ref nout, ref wcount);
alglib.ap.assert(s.nin==nin, "MLPContinueTrainingX: internal error - number of inputs in trainer is not equal to number of inputs in the network.");
alglib.ap.assert(s.nout==nout, "MLPContinueTrainingX: internal error - number of outputs in trainer is not equal to number of outputs in the network.");
mlpbase.mlpproperties(tnetwork, ref nin, ref nout, ref twcount);
alglib.ap.assert(s.nin==nin, "MLPContinueTrainingX: internal error - number of inputs in trainer is not equal to number of inputs in the training network.");
alglib.ap.assert(s.nout==nout, "MLPContinueTrainingX: internal error - number of outputs in trainer is not equal to number of outputs in the training network.");
alglib.ap.assert(twcount==wcount, "MLPContinueTrainingX: internal error - number of weights the resulting network is not equal to number of weights in the training network.");
alglib.ap.assert(alglib.ap.len(subset)>=subsetsize, "MLPContinueTrainingX: internal error - parameter SubsetSize more than input subset size(Length(Subset)<SubsetSize).");
for(i=0; i<=subsetsize-1; i++)
{
alglib.ap.assert(subset[i]>=0 && subset[i]<=s.npoints-1, "MLPContinueTrainingX: internal error - parameter Subset contains incorrect index(Subset[I]<0 or Subset[I]>S.NPoints-1).");
}
if( ((s.datatype==0 || s.datatype==1) && s.npoints>0) && subsetsize!=0 )
{
decay = s.decay;
while( minlbfgs.minlbfgsiteration(state) )
{
if( state.xupdated )
{
for(i_=0; i_<=wcount-1;i_++)
{
network.weights[i_] = tnetwork.weights[i_];
}
result = true;
return result;
}
for(i_=0; i_<=wcount-1;i_++)
{
tnetwork.weights[i_] = state.x[i_];
}
if( s.datatype==0 )
{
mlpbase.mlpgradbatchsubset(tnetwork, s.densexy, s.npoints, subset, subsetsize, ref state.f, ref state.g);
}
if( s.datatype==1 )
{
mlpbase.mlpgradbatchsparsesubset(tnetwork, s.sparsexy, s.npoints, subset, subsetsize, ref state.f, ref state.g);
}
//
// Increment number of operations performed on batch gradient
//
ngradbatch = ngradbatch+1;
v = 0.0;
for(i_=0; i_<=wcount-1;i_++)
{
v += tnetwork.weights[i_]*tnetwork.weights[i_];
}
state.f = state.f+0.5*decay*v;
for(i_=0; i_<=wcount-1;i_++)
{
state.g[i_] = state.g[i_] + decay*tnetwork.weights[i_];
}
}
for(i_=0; i_<=wcount-1;i_++)
{
network.weights[i_] = tnetwork.weights[i_];
}
}
result = false;
return result;
}
/*************************************************************************
This function performs step-by-step training of the neural network. Here
"step-by-step" means that training starts with MLPStartTrainingX call,
and then user subsequently calls MLPContinueTrainingX to perform one more
iteration of the training.
After call to this function trainer object remembers network and is ready
to train it. However, no training is performed until first call to
MLPContinueTraining() function. Subsequent calls to MLPContinueTraining()
will advance traing progress one iteration further.
EXAMPLE:
>
> ...initialize network and trainer object....
>
> MLPStartTraining(Trainer, Network, True)
> while MLPContinueTraining(Trainer, Network) do
> ...visualize training progress...
>
INPUT PARAMETERS:
S - trainer object;
Network - neural network which receives A COPY of the actual
network which is trained by the algorithm. After each
training roung state of the network being trained is
copied to this variable.
It must have same number of inputs and output/classes
as was specified during creation of the trainer object
and it must have exactly same architecture as the
second network (TNetwork).
TNetwork - neural network being trained.
State - LBFGS optimizer, already initialized, number of
dimensions must be equal to number of weights in the
networks.
RandomStart - randomize network before training or not:
* True means that network is randomized and its
initial state (one which was passed to the trainer
object) is lost;
* False means that training is started from the
current state of the network.
Subset - some subset from training set(it stores row's numbers);
SubsetSize - size of subset(if SubsetSize<0 - used full dataset).
OUTPUT PARAMETERS:
Network - neural network which is ready to training (weights are
initialized, preprocessor is initialized using current
training set)
NOTE: this method uses sum-of-squares error function for training.
NOTE: it is expected that trainer object settings are NOT changed during
step-by-step training, i.e. no one changes stopping criteria or
training set during training. It is possible and there is no defense
against such actions, but algorithm behavior in such cases is
undefined and can be unpredictable.
-- ALGLIB --
Copyright 13.08.2012 by Bochkanov Sergey
*************************************************************************/
private static void mlpstarttrainingx(mlptrainer s,
mlpbase.multilayerperceptron network,
mlpbase.multilayerperceptron tnetwork,
minlbfgs.minlbfgsstate state,
bool randomstart,
int[] subset,
int subsetsize)
{
int nin = 0;
int nout = 0;
int wcount = 0;
int twcount = 0;
int ntype = 0;
int ttype = 0;
int i = 0;
int i_ = 0;
alglib.ap.assert(s.npoints>=0, "MLPStartTrainingX: internal error - parameter S is not initialized or is spoiled(S.NPoints<0)");
if( s.rcpar )
{
ttype = 0;
}
else
{
ttype = 1;
}
if( !mlpbase.mlpissoftmax(network) )
{
ntype = 0;
}
else
{
ntype = 1;
}
alglib.ap.assert(ntype==ttype, "MLPStartTrainingX: internal error - type of the resulting network is not similar to network type in trainer object");
if( !mlpbase.mlpissoftmax(tnetwork) )
{
ntype = 0;
}
else
{
ntype = 1;
}
alglib.ap.assert(ntype==ttype, "MLPStartTrainingX: internal error - type of the training network is not similar to network type in trainer object");
mlpbase.mlpproperties(network, ref nin, ref nout, ref wcount);
alglib.ap.assert(s.nin==nin, "MLPStartTrainingX: number of inputs in trainer is not equal to number of inputs in the network.");
alglib.ap.assert(s.nout==nout, "MLPStartTrainingX: number of outputs in trainer is not equal to number of outputs in the network.");
mlpbase.mlpproperties(tnetwork, ref nin, ref nout, ref twcount);
alglib.ap.assert(s.nin==nin, "MLPStartTrainingX: number of inputs in trainer is not equal to number of inputs in the training network.");
alglib.ap.assert(s.nout==nout, "MLPStartTrainingX: number of outputs in trainer is not equal to number of outputs in the training network.");
alglib.ap.assert(twcount==wcount, "MLPStartTrainingX: number of weights the resulting network is not equal to number of weights in the training network.");
alglib.ap.assert(alglib.ap.len(subset)>=subsetsize, "MLPStartTrainingX: internal error - parameter SubsetSize more than input subset size(Length(Subset)<SubsetSize)");
for(i=0; i<=subsetsize-1; i++)
{
alglib.ap.assert(subset[i]>=0 && subset[i]<=s.npoints-1, "MLPStartTrainingX: internal error - parameter Subset contains incorrect index(Subset[I]<0 or Subset[I]>S.NPoints-1)");
}
if( ((s.datatype==0 || s.datatype==1) && s.npoints>0) && subsetsize!=0 )
{
//
// Prepare
//
if( s.datatype==0 )
{
mlpbase.mlpinitpreprocessorsubset(network, s.densexy, s.npoints, subset, subsetsize);
mlpbase.mlpinitpreprocessorsubset(tnetwork, s.densexy, s.npoints, subset, subsetsize);
}
if( s.datatype==1 )
{
mlpbase.mlpinitpreprocessorsparsesubset(network, s.sparsexy, s.npoints, subset, subsetsize);
mlpbase.mlpinitpreprocessorsparsesubset(tnetwork, s.sparsexy, s.npoints, subset, subsetsize);
}
//
// Process
//
if( randomstart )
{
mlpbase.mlprandomize(network);
}
minlbfgs.minlbfgsrestartfrom(state, network.weights);
}
else
{
for(i=0; i<=wcount-1; i++)
{
network.weights[i] = 0;
}
}
//
// Copy weights
//
for(i_=0; i_<=wcount-1;i_++)
{
tnetwork.weights[i_] = network.weights[i_];
}
}
/*************************************************************************
IMPORTANT: this is an "expert" version of the MLPTrain() function. We do
not recommend you to use it unless you are pretty sure that you
need ability to monitor training progress.
This function performs step-by-step training of the neural network. Here
"step-by-step" means that training starts with MLPStartTraining() call,
and then user subsequently calls MLPContinueTraining() to perform one more
iteration of the training.
After call to this function trainer object remembers network and is ready
to train it. However, no training is performed until first call to
MLPContinueTraining() function. Subsequent calls to MLPContinueTraining()
will advance training progress one iteration further.
EXAMPLE:
>
> ...initialize network and trainer object....
>
> MLPStartTraining(Trainer, Network, True)
> while MLPContinueTraining(Trainer, Network) do
> ...visualize training progress...
>
INPUT PARAMETERS:
S - trainer object
Network - neural network. It must have same number of inputs and
output/classes as was specified during creation of the
trainer object.
RandomStart - randomize network before training or not:
* True means that network is randomized and its
initial state (one which was passed to the trainer
object) is lost.
* False means that training is started from the
current state of the network
OUTPUT PARAMETERS:
Network - neural network which is ready to training (weights are
initialized, preprocessor is initialized using current
training set)
NOTE: this method uses sum-of-squares error function for training.
NOTE: it is expected that trainer object settings are NOT changed during
step-by-step training, i.e. no one changes stopping criteria or
training set during training. It is possible and there is no defense
against such actions, but algorithm behavior in such cases is
undefined and can be unpredictable.
-- ALGLIB --
Copyright 23.07.2012 by Bochkanov Sergey
*************************************************************************/
public static void mlpstarttraining(mlptrainer s,
mlpbase.multilayerperceptron network,
bool randomstart)
{
int nin = 0;
int nout = 0;
int wcount = 0;
int ntype = 0;
int ttype = 0;
alglib.ap.assert(s.npoints>=0, "MLPStartTraining: parameter S is not initialized or is spoiled(S.NPoints<0)");
if( !mlpbase.mlpissoftmax(network) )
{
ntype = 0;
}
else
{
ntype = 1;
}
if( s.rcpar )
{
ttype = 0;
}
else
{
ttype = 1;
}
alglib.ap.assert(ntype==ttype, "MLPStartTraining: type of input network is not similar to network type in trainer object");
mlpbase.mlpproperties(network, ref nin, ref nout, ref wcount);
alglib.ap.assert(s.nin==nin, "MLPStartTraining: number of inputs in trainer is not equal to number of inputs in the network.");
alglib.ap.assert(s.nout==nout, "MLPStartTraining: number of outputs in trainer is not equal to number of outputs in the network.");
//
// Initialize temporaries
//
initmlptrnsession(network, randomstart, s, s.session);
//
// Train network
//
mlpstarttrainingx(s, randomstart, -1, s.subset, -1, s.session);
//
// Update network
//
mlpbase.mlpcopytunableparameters(s.session.network, network);
}
/*************************************************************************
This function sets "current dataset" of the trainer object to one passed
by user.
INPUT PARAMETERS:
S - trainer object
XY - training set, see below for information on the
training set format. This function checks correctness
of the dataset (no NANs/INFs, class numbers are
correct) and throws exception when incorrect dataset
is passed.
NPoints - points count, >=0.
DATASET FORMAT:
This function uses two different dataset formats - one for regression
networks, another one for classification networks.
For regression networks with NIn inputs and NOut outputs following dataset
format is used:
* dataset is given by NPoints*(NIn+NOut) matrix
* each row corresponds to one example
* first NIn columns are inputs, next NOut columns are outputs
For classification networks with NIn inputs and NClasses clases following
datasetformat is used:
* dataset is given by NPoints*(NIn+1) matrix
* each row corresponds to one example
* first NIn columns are inputs, last column stores class number (from 0 to
NClasses-1).
-- ALGLIB --
Copyright 23.07.2012 by Bochkanov Sergey
*************************************************************************/
public static void mlpsetdataset(mlptrainer s,
double[,] xy,
int npoints)
{
int ndim = 0;
int i = 0;
int j = 0;
alglib.ap.assert(s.nin>=1, "MLPSetDataset: possible parameter S is not initialized or spoiled(S.NIn<=0).");
alglib.ap.assert(npoints>=0, "MLPSetDataset: NPoint<0");
alglib.ap.assert(npoints<=alglib.ap.rows(xy), "MLPSetDataset: invalid size of matrix XY(NPoint more then rows of matrix XY)");
s.datatype = 0;
s.npoints = npoints;
if( npoints==0 )
{
return;
}
if( s.rcpar )
{
alglib.ap.assert(s.nout>=1, "MLPSetDataset: possible parameter S is not initialized or is spoiled(NOut<1 for regression).");
ndim = s.nin+s.nout;
alglib.ap.assert(ndim<=alglib.ap.cols(xy), "MLPSetDataset: invalid size of matrix XY(too few columns in matrix XY).");
alglib.ap.assert(apserv.apservisfinitematrix(xy, npoints, ndim), "MLPSetDataset: parameter XY contains Infinite or NaN.");
}
else
{
alglib.ap.assert(s.nout>=2, "MLPSetDataset: possible parameter S is not initialized or is spoiled(NClasses<2 for classifier).");
ndim = s.nin+1;
alglib.ap.assert(ndim<=alglib.ap.cols(xy), "MLPSetDataset: invalid size of matrix XY(too few columns in matrix XY).");
alglib.ap.assert(apserv.apservisfinitematrix(xy, npoints, ndim), "MLPSetDataset: parameter XY contains Infinite or NaN.");
for(i=0; i<=npoints-1; i++)
{
alglib.ap.assert((int)Math.Round(xy[i,s.nin])>=0 && (int)Math.Round(xy[i,s.nin])<s.nout, "MLPSetDataset: invalid parameter XY(in classifier used nonexistent class number: either XY[.,NIn]<0 or XY[.,NIn]>=NClasses).");
}
}
apserv.rmatrixsetlengthatleast(ref s.densexy, npoints, ndim);
for(i=0; i<=npoints-1; i++)
{
for(j=0; j<=ndim-1; j++)
{
s.densexy[i,j] = xy[i,j];
}
}
}
/*************************************************************************
IMPORTANT: this is an "expert" version of the MLPTrain() function. We do
not recommend you to use it unless you are pretty sure that you
need ability to monitor training progress.
FOR USERS OF COMMERCIAL EDITION:
! Commercial version of ALGLIB includes two important improvements of
! this function:
! * multicore support (C++ and C# computational cores)
! * SSE support (C++ computational core)
!
! Second improvement gives constant speedup (2-3X). First improvement
! gives close-to-linear speedup on multicore systems. Following
! operations can be executed in parallel:
! * gradient calculation over large dataset (if dataset is large enough)
!
! In order to use multicore features you have to:
! * use commercial version of ALGLIB
! * call this function with "smp_" prefix, which indicates that
! multicore code will be used (for multicore support)
!
! In order to use SSE features you have to:
! * use commercial version of ALGLIB on Intel processors
! * use C++ computational core
!
! This note is given for users of commercial edition; if you use GPL
! edition, you still will be able to call smp-version of this function,
! but all computations will be done serially.
!
! We recommend you to carefully read ALGLIB Reference Manual, section
! called 'SMP support', before using parallel version of this function.
This function performs step-by-step training of the neural network. Here
"step-by-step" means that training starts with MLPStartTraining() call,
and then user subsequently calls MLPContinueTraining() to perform one more
iteration of the training.
This function performs one more iteration of the training and returns
either True (training continues) or False (training stopped). In case True
was returned, Network weights are updated according to the current state
of the optimization progress. In case False was returned, no additional
updates is performed (previous update of the network weights moved us to
the final point, and no additional updates is needed).
EXAMPLE:
>
> [initialize network and trainer object]
>
> MLPStartTraining(Trainer, Network, True)
> while MLPContinueTraining(Trainer, Network) do
> [visualize training progress]
>
INPUT PARAMETERS:
S - trainer object
Network - neural network structure, which is used to store
current state of the training process.
OUTPUT PARAMETERS:
Network - weights of the neural network are rewritten by the
current approximation.
NOTE: this method uses sum-of-squares error function for training.
NOTE: it is expected that trainer object settings are NOT changed during
step-by-step training, i.e. no one changes stopping criteria or
training set during training. It is possible and there is no defense
against such actions, but algorithm behavior in such cases is
undefined and can be unpredictable.
NOTE: It is expected that Network is the same one which was passed to
MLPStartTraining() function. However, THIS function checks only
following:
* that number of network inputs is consistent with trainer object
settings
* that number of network outputs/classes is consistent with trainer
object settings
* that number of network weights is the same as number of weights in
the network passed to MLPStartTraining() function
Exception is thrown when these conditions are violated.
It is also expected that you do not change state of the network on
your own - the only party who has right to change network during its
training is a trainer object. Any attempt to interfere with trainer
may lead to unpredictable results.
-- ALGLIB --
Copyright 23.07.2012 by Bochkanov Sergey
*************************************************************************/
public static bool mlpcontinuetraining(mlptrainer s,
mlpbase.multilayerperceptron network)
{
bool result = new bool();
int nin = 0;
int nout = 0;
int wcount = 0;
int ntype = 0;
int ttype = 0;
int i_ = 0;
alglib.ap.assert(s.npoints>=0, "MLPContinueTraining: parameter S is not initialized or is spoiled(S.NPoints<0)");
if( s.rcpar )
{
ttype = 0;
}
else
{
ttype = 1;
}
if( !mlpbase.mlpissoftmax(network) )
{
ntype = 0;
}
else
{
ntype = 1;
}
alglib.ap.assert(ntype==ttype, "MLPContinueTraining: type of input network is not similar to network type in trainer object.");
mlpbase.mlpproperties(network, ref nin, ref nout, ref wcount);
alglib.ap.assert(s.nin==nin, "MLPContinueTraining: number of inputs in trainer is not equal to number of inputs in the network.");
alglib.ap.assert(s.nout==nout, "MLPContinueTraining: number of outputs in trainer is not equal to number of outputs in the network.");
result = mlpcontinuetrainingx(s, s.subset, -1, ref s.ngradbatch, s.session);
if( result )
{
for(i_=0; i_<=wcount-1;i_++)
{
network.weights[i_] = s.session.network.weights[i_];
}
}
return result;
}
/*************************************************************************
This function sets "current dataset" of the trainer object to one passed
by user (sparse matrix is used to store dataset).
INPUT PARAMETERS:
S - trainer object
XY - training set, see below for information on the
training set format. This function checks correctness
of the dataset (no NANs/INFs, class numbers are
correct) and throws exception when incorrect dataset
is passed. Any sparse storage format can be used:
Hash-table, CRS...
NPoints - points count, >=0
DATASET FORMAT:
This function uses two different dataset formats - one for regression
networks, another one for classification networks.
For regression networks with NIn inputs and NOut outputs following dataset
format is used:
* dataset is given by NPoints*(NIn+NOut) matrix
* each row corresponds to one example
* first NIn columns are inputs, next NOut columns are outputs
For classification networks with NIn inputs and NClasses clases following
datasetformat is used:
* dataset is given by NPoints*(NIn+1) matrix
* each row corresponds to one example
* first NIn columns are inputs, last column stores class number (from 0 to
NClasses-1).
-- ALGLIB --
Copyright 23.07.2012 by Bochkanov Sergey
*************************************************************************/
public static void mlpsetsparsedataset(mlptrainer s,
sparse.sparsematrix xy,
int npoints)
{
double v = 0;
int t0 = 0;
int t1 = 0;
int i = 0;
int j = 0;
//
// Check correctness of the data
//
alglib.ap.assert(s.nin>0, "MLPSetSparseDataset: possible parameter S is not initialized or spoiled(S.NIn<=0).");
alglib.ap.assert(npoints>=0, "MLPSetSparseDataset: NPoint<0");
alglib.ap.assert(npoints<=sparse.sparsegetnrows(xy), "MLPSetSparseDataset: invalid size of sparse matrix XY(NPoint more then rows of matrix XY)");
if( npoints>0 )
{
t0 = 0;
t1 = 0;
if( s.rcpar )
{
alglib.ap.assert(s.nout>=1, "MLPSetSparseDataset: possible parameter S is not initialized or is spoiled(NOut<1 for regression).");
alglib.ap.assert(s.nin+s.nout<=sparse.sparsegetncols(xy), "MLPSetSparseDataset: invalid size of sparse matrix XY(too few columns in sparse matrix XY).");
while( sparse.sparseenumerate(xy, ref t0, ref t1, ref i, ref j, ref v) )
{
if( i<npoints && j<s.nin+s.nout )
{
alglib.ap.assert(math.isfinite(v), "MLPSetSparseDataset: sparse matrix XY contains Infinite or NaN.");
}
}
}
else
{
alglib.ap.assert(s.nout>=2, "MLPSetSparseDataset: possible parameter S is not initialized or is spoiled(NClasses<2 for classifier).");
alglib.ap.assert(s.nin+1<=sparse.sparsegetncols(xy), "MLPSetSparseDataset: invalid size of sparse matrix XY(too few columns in sparse matrix XY).");
while( sparse.sparseenumerate(xy, ref t0, ref t1, ref i, ref j, ref v) )
{
if( i<npoints && j<=s.nin )
{
if( j!=s.nin )
{
alglib.ap.assert(math.isfinite(v), "MLPSetSparseDataset: sparse matrix XY contains Infinite or NaN.");
}
else
{
alglib.ap.assert((math.isfinite(v) && (int)Math.Round(v)>=0) && (int)Math.Round(v)<s.nout, "MLPSetSparseDataset: invalid sparse matrix XY(in classifier used nonexistent class number: either XY[.,NIn]<0 or XY[.,NIn]>=NClasses).");
}
}
}
}
}
//
// Set dataset
//
s.datatype = 1;
s.npoints = npoints;
sparse.sparsecopytocrs(xy, s.sparsexy);
}
/*************************************************************************
This function trains neural network ensemble passed to this function using
current dataset and early stopping training algorithm. Each early stopping
round performs NRestarts random restarts (thus, EnsembleSize*NRestarts
training rounds is performed in total).
FOR USERS OF COMMERCIAL EDITION:
! Commercial version of ALGLIB includes two important improvements of
! this function:
! * multicore support (C++ and C# computational cores)
! * SSE support (C++ computational core)
!
! Second improvement gives constant speedup (2-3X). First improvement
! gives close-to-linear speedup on multicore systems. Following
! operations can be executed in parallel:
! * EnsembleSize training sessions performed for each of ensemble
! members (always parallelized)
! * NRestarts training sessions performed within each of training
! sessions (if NRestarts>1)
! * gradient calculation over large dataset (if dataset is large enough)
!
! In order to use multicore features you have to:
! * use commercial version of ALGLIB
! * call this function with "smp_" prefix, which indicates that
! multicore code will be used (for multicore support)
!
! In order to use SSE features you have to:
! * use commercial version of ALGLIB on Intel processors
! * use C++ computational core
!
! This note is given for users of commercial edition; if you use GPL
! edition, you still will be able to call smp-version of this function,
! but all computations will be done serially.
!
! We recommend you to carefully read ALGLIB Reference Manual, section
! called 'SMP support', before using parallel version of this function.
INPUT PARAMETERS:
S - trainer object;
Ensemble - neural network ensemble. It must have same number of
inputs and outputs/classes as was specified during
creation of the trainer object.
NRestarts - number of restarts, >=0:
* NRestarts>0 means that specified number of random
restarts are performed during each ES round;
* NRestarts=0 is silently replaced by 1.
OUTPUT PARAMETERS:
Ensemble - trained ensemble;
Rep - it contains all type of errors.
NOTE: this training method uses BOTH early stopping and weight decay! So,
you should select weight decay before starting training just as you
select it before training "conventional" networks.
NOTE: when no dataset was specified with MLPSetDataset/SetSparseDataset(),
or single-point dataset was passed, ensemble is filled by zero
values.
NOTE: this method uses sum-of-squares error function for training.
-- ALGLIB --
Copyright 22.08.2012 by Bochkanov Sergey
*************************************************************************/
public static void mlptrainensemblees(mlptrainer s,
mlpe.mlpensemble ensemble,
int nrestarts,
mlpreport rep)
{
int nin = 0;
int nout = 0;
int ntype = 0;
int ttype = 0;
alglib.smp.shared_pool esessions = new alglib.smp.shared_pool();
apserv.sinteger sgrad = new apserv.sinteger();
mlpbase.modelerrors tmprep = new mlpbase.modelerrors();
alglib.ap.assert(s.npoints>=0, "MLPTrainEnsembleES: parameter S is not initialized or is spoiled(S.NPoints<0)");
if( !mlpe.mlpeissoftmax(ensemble) )
{
ntype = 0;
}
else
{
ntype = 1;
}
if( s.rcpar )
{
ttype = 0;
}
else
{
ttype = 1;
}
alglib.ap.assert(ntype==ttype, "MLPTrainEnsembleES: internal error - type of input network is not similar to network type in trainer object");
nin = mlpbase.mlpgetinputscount(ensemble.network);
alglib.ap.assert(s.nin==nin, "MLPTrainEnsembleES: number of inputs in trainer is not equal to number of inputs in ensemble network");
nout = mlpbase.mlpgetoutputscount(ensemble.network);
alglib.ap.assert(s.nout==nout, "MLPTrainEnsembleES: number of outputs in trainer is not equal to number of outputs in ensemble network");
alglib.ap.assert(nrestarts>=0, "MLPTrainEnsembleES: NRestarts<0.");
//
// Initialize parameter Rep
//
rep.relclserror = 0;
rep.avgce = 0;
rep.rmserror = 0;
rep.avgerror = 0;
rep.avgrelerror = 0;
rep.ngrad = 0;
rep.nhess = 0;
rep.ncholesky = 0;
//
// Allocate
//
apserv.ivectorsetlengthatleast(ref s.subset, s.npoints);
apserv.ivectorsetlengthatleast(ref s.valsubset, s.npoints);
//
// Start training
//
// NOTE: ESessions is not initialized because MLPTrainEnsembleX
// needs uninitialized pool.
//
sgrad.val = 0;
mlptrainensemblex(s, ensemble, 0, ensemble.ensemblesize, nrestarts, 0, sgrad, true, esessions);
rep.ngrad = sgrad.val;
//
// Calculate errors.
//
if( s.datatype==0 )
{
mlpe.mlpeallerrorsx(ensemble, s.densexy, s.sparsexy, s.npoints, 0, ensemble.network.dummyidx, 0, s.npoints, 0, ensemble.network.buf, tmprep);
}
if( s.datatype==1 )
{
mlpe.mlpeallerrorsx(ensemble, s.densexy, s.sparsexy, s.npoints, 1, ensemble.network.dummyidx, 0, s.npoints, 0, ensemble.network.buf, tmprep);
}
rep.relclserror = tmprep.relclserror;
rep.avgce = tmprep.avgce;
rep.rmserror = tmprep.rmserror;
rep.avgerror = tmprep.avgerror;
rep.avgrelerror = tmprep.avgrelerror;
}
/*************************************************************************
This function sets weight decay coefficient which is used for training.
INPUT PARAMETERS:
S - trainer object
Decay - weight decay coefficient, >=0. Weight decay term
'Decay*||Weights||^2' is added to error function. If
you don't know what Decay to choose, use 1.0E-3.
Weight decay can be set to zero, in this case network
is trained without weight decay.
NOTE: by default network uses some small nonzero value for weight decay.
-- ALGLIB --
Copyright 23.07.2012 by Bochkanov Sergey
*************************************************************************/
public static void mlpsetdecay(mlptrainer s,
double decay)
{
alglib.ap.assert(math.isfinite(decay), "MLPSetDecay: parameter Decay contains Infinite or NaN.");
alglib.ap.assert((double)(decay)>=(double)(0), "MLPSetDecay: Decay<0.");
s.decay = decay;
}
/*************************************************************************
Internal subroutine for parallelization function MLPFoldCV.
INPUT PARAMETERS:
S - trainer object;
RowSize - row size(eitherNIn+NOut or NIn+1);
NRestarts - number of restarts(>=0);
Folds - cross-validation set;
Fold - the number of first cross-validation(>=0);
DFold - the number of second cross-validation(>=Fold+1);
CVY - parameter which stores the result is returned by network,
training on I-th cross-validation set.
It has to be preallocated.
PoolDataCV- parameter for parallelization.
NOTE: There are no checks on the parameters correctness.
-- ALGLIB --
Copyright 25.09.2012 by Bochkanov Sergey
*************************************************************************/
private static void mthreadcv(mlptrainer s,
int rowsize,
int nrestarts,
int[] folds,
int fold,
int dfold,
double[,] cvy,
alglib.smp.shared_pool pooldatacv)
{
mlpparallelizationcv datacv = null;
int i = 0;
int i_ = 0;
if( fold==dfold-1 )
{
//
// Separate set
//
alglib.smp.ae_shared_pool_retrieve(pooldatacv, ref datacv);
datacv.subsetsize = 0;
for(i=0; i<=s.npoints-1; i++)
{
if( folds[i]!=fold )
{
datacv.subset[datacv.subsetsize] = i;
datacv.subsetsize = datacv.subsetsize+1;
}
}
//
// Train on CV training set
//
mlptrainnetworkx(s, nrestarts, -1, datacv.subset, datacv.subsetsize, datacv.subset, 0, datacv.network, datacv.rep, true, datacv.trnpool);
datacv.ngrad = datacv.ngrad+datacv.rep.ngrad;
//
// Estimate error using CV test set
//
for(i=0; i<=s.npoints-1; i++)
{
if( folds[i]==fold )
{
if( s.datatype==0 )
{
for(i_=0; i_<=rowsize-1;i_++)
{
datacv.xyrow[i_] = s.densexy[i,i_];
}
}
if( s.datatype==1 )
{
sparse.sparsegetrow(s.sparsexy, i, ref datacv.xyrow);
}
mlpbase.mlpprocess(datacv.network, datacv.xyrow, ref datacv.y);
for(i_=0; i_<=s.nout-1;i_++)
{
cvy[i,i_] = datacv.y[i_];
}
}
}
alglib.smp.ae_shared_pool_recycle(pooldatacv, ref datacv);
}
else
{
alglib.ap.assert(fold<dfold-1, "MThreadCV: internal error(Fold>DFold-1).");
mthreadcv(s, rowsize, nrestarts, folds, fold, (fold+dfold)/2, cvy, pooldatacv);
mthreadcv(s, rowsize, nrestarts, folds, (fold+dfold)/2, dfold, cvy, pooldatacv);
}
}
/*************************************************************************
This function sets stopping criteria for the optimizer.
INPUT PARAMETERS:
S - trainer object
WStep - stopping criterion. Algorithm stops if step size is
less than WStep. Recommended value - 0.01. Zero step
size means stopping after MaxIts iterations.
WStep>=0.
MaxIts - stopping criterion. Algorithm stops after MaxIts
epochs (full passes over entire dataset). Zero MaxIts
means stopping when step is sufficiently small.
MaxIts>=0.
NOTE: by default, WStep=0.005 and MaxIts=0 are used. These values are also
used when MLPSetCond() is called with WStep=0 and MaxIts=0.
NOTE: these stopping criteria are used for all kinds of neural training -
from "conventional" networks to early stopping ensembles. When used
for "conventional" networks, they are used as the only stopping
criteria. When combined with early stopping, they used as ADDITIONAL
stopping criteria which can terminate early stopping algorithm.
-- ALGLIB --
Copyright 23.07.2012 by Bochkanov Sergey
*************************************************************************/
public static void mlpsetcond(mlptrainer s,
double wstep,
int maxits)
{
alglib.ap.assert(math.isfinite(wstep), "MLPSetCond: parameter WStep contains Infinite or NaN.");
alglib.ap.assert((double)(wstep)>=(double)(0), "MLPSetCond: WStep<0.");
alglib.ap.assert(maxits>=0, "MLPSetCond: MaxIts<0.");
if( (double)(wstep)!=(double)(0) || maxits!=0 )
{
s.wstep = wstep;
s.maxits = maxits;
}
else
{
s.wstep = 0.005;
s.maxits = 0;
}
}
/*************************************************************************
This function trains neural network ensemble passed to this function using
current dataset and early stopping training algorithm. Each early stopping
round performs NRestarts random restarts (thus, EnsembleSize*NRestarts
training rounds is performed in total).
-- ALGLIB --
Copyright 22.08.2012 by Bochkanov Sergey
*************************************************************************/
private static void mlptrainensemblex(mlptrainer s,
mlpe.mlpensemble ensemble,
int idx0,
int idx1,
int nrestarts,
int trainingmethod,
apserv.sinteger ngrad,
bool isrootcall,
alglib.smp.shared_pool esessions)
{
int pcount = 0;
int nin = 0;
int nout = 0;
int wcount = 0;
int i = 0;
int j = 0;
int k = 0;
int trnsubsetsize = 0;
int valsubsetsize = 0;
int k0 = 0;
apserv.sinteger ngrad0 = new apserv.sinteger();
apserv.sinteger ngrad1 = new apserv.sinteger();
mlpetrnsession psession = null;
hqrnd.hqrndstate rs = new hqrnd.hqrndstate();
int i_ = 0;
int i1_ = 0;
nin = mlpbase.mlpgetinputscount(ensemble.network);
nout = mlpbase.mlpgetoutputscount(ensemble.network);
wcount = mlpbase.mlpgetweightscount(ensemble.network);
if( mlpbase.mlpissoftmax(ensemble.network) )
{
pcount = nin;
}
else
{
pcount = nin+nout;
}
if( nrestarts<=0 )
{
nrestarts = 1;
}
//
// Handle degenerate case
//
if( s.npoints<2 )
{
for(i=idx0; i<=idx1-1; i++)
{
for(j=0; j<=wcount-1; j++)
{
ensemble.weights[i*wcount+j] = 0.0;
}
for(j=0; j<=pcount-1; j++)
{
ensemble.columnmeans[i*pcount+j] = 0.0;
ensemble.columnsigmas[i*pcount+j] = 1.0;
}
}
return;
}
//
// Process root call
//
if( isrootcall )
{
//
// Prepare:
// * prepare MLPETrnSessions
// * fill ensemble by zeros (helps to detect errors)
//
initmlpetrnsessions(ensemble.network, s, esessions);
for(i=idx0; i<=idx1-1; i++)
{
for(j=0; j<=wcount-1; j++)
{
ensemble.weights[i*wcount+j] = 0.0;
}
for(j=0; j<=pcount-1; j++)
{
ensemble.columnmeans[i*pcount+j] = 0.0;
ensemble.columnsigmas[i*pcount+j] = 0.0;
}
}
//
// Train in non-root mode and exit
//
mlptrainensemblex(s, ensemble, idx0, idx1, nrestarts, trainingmethod, ngrad, false, esessions);
return;
}
//
// Split problem
//
if( idx1-idx0>=2 )
{
k0 = (idx1-idx0)/2;
ngrad0.val = 0;
ngrad1.val = 0;
mlptrainensemblex(s, ensemble, idx0, idx0+k0, nrestarts, trainingmethod, ngrad0, false, esessions);
mlptrainensemblex(s, ensemble, idx0+k0, idx1, nrestarts, trainingmethod, ngrad1, false, esessions);
ngrad.val = ngrad0.val+ngrad1.val;
return;
}
//
// Retrieve and prepare session
//
alglib.smp.ae_shared_pool_retrieve(esessions, ref psession);
//
// Train
//
hqrnd.hqrndrandomize(rs);
for(k=idx0; k<=idx1-1; k++)
{
//
// Split set
//
trnsubsetsize = 0;
valsubsetsize = 0;
if( trainingmethod==0 )
{
do
{
trnsubsetsize = 0;
valsubsetsize = 0;
for(i=0; i<=s.npoints-1; i++)
{
if( (double)(math.randomreal())<(double)(0.66) )
{
//
// Assign sample to training set
//
psession.trnsubset[trnsubsetsize] = i;
trnsubsetsize = trnsubsetsize+1;
}
else
{
//
// Assign sample to validation set
//
psession.valsubset[valsubsetsize] = i;
valsubsetsize = valsubsetsize+1;
}
}
}
while( !(trnsubsetsize!=0 && valsubsetsize!=0) );
}
if( trainingmethod==1 )
{
valsubsetsize = 0;
trnsubsetsize = s.npoints;
for(i=0; i<=s.npoints-1; i++)
{
psession.trnsubset[i] = hqrnd.hqrnduniformi(rs, s.npoints);
}
}
//
// Train
//
mlptrainnetworkx(s, nrestarts, -1, psession.trnsubset, trnsubsetsize, psession.valsubset, valsubsetsize, psession.network, psession.mlprep, true, psession.mlpsessions);
ngrad.val = ngrad.val+psession.mlprep.ngrad;
//
// Save results
//
i1_ = (0) - (k*wcount);
for(i_=k*wcount; i_<=(k+1)*wcount-1;i_++)
{
ensemble.weights[i_] = psession.network.weights[i_+i1_];
}
i1_ = (0) - (k*pcount);
for(i_=k*pcount; i_<=(k+1)*pcount-1;i_++)
{
ensemble.columnmeans[i_] = psession.network.columnmeans[i_+i1_];
}
i1_ = (0) - (k*pcount);
for(i_=k*pcount; i_<=(k+1)*pcount-1;i_++)
{
ensemble.columnsigmas[i_] = psession.network.columnsigmas[i_+i1_];
}
}
//
// Recycle session
//
alglib.smp.ae_shared_pool_recycle(esessions, ref psession);
}
/*************************************************************************
This function sets training algorithm: batch training using L-BFGS will be
used.
This algorithm:
* the most robust for small-scale problems, but may be too slow for large
scale ones.
* perfoms full pass through the dataset before performing step
* uses conditions specified by MLPSetCond() for stopping
* is default one used by trainer object
INPUT PARAMETERS:
S - trainer object
-- ALGLIB --
Copyright 23.07.2012 by Bochkanov Sergey
*************************************************************************/
public static void mlpsetalgobatch(mlptrainer s)
{
s.algokind = 0;
}
/*************************************************************************
This function performs step-by-step training of the neural network. Here
"step-by-step" means that training starts with MLPStartTrainingX call,
and then user subsequently calls MLPContinueTrainingX to perform one more
iteration of the training.
This function performs one more iteration of the training and returns
either True (training continues) or False (training stopped). In case True
was returned, Network weights are updated according to the current state
of the optimization progress. In case False was returned, no additional
updates is performed (previous update of the network weights moved us to
the final point, and no additional updates is needed).
EXAMPLE:
>
> [initialize network and trainer object]
>
> MLPStartTraining(Trainer, Network, True)
> while MLPContinueTraining(Trainer, Network) do
> [visualize training progress]
>
-- ALGLIB --
Copyright 13.08.2012 by Bochkanov Sergey
*************************************************************************/
private static bool mlpcontinuetrainingx(mlptrainer s,
int[] subset,
int subsetsize,
ref int ngradbatch,
smlptrnsession session)
{
bool result = new bool();
int nin = 0;
int nout = 0;
int wcount = 0;
int twcount = 0;
int ntype = 0;
int ttype = 0;
double decay = 0;
double v = 0;
int i = 0;
int j = 0;
int k = 0;
int trnsetsize = 0;
int epoch = 0;
int minibatchcount = 0;
int minibatchidx = 0;
int cursize = 0;
int idx0 = 0;
int idx1 = 0;
int i_ = 0;
//
// Reverse communication preparations
// I know it looks ugly, but it works the same way
// anywhere from C++ to Python.
//
// This code initializes locals by:
// * random values determined during code
// generation - on first subroutine call
// * values from previous call - on subsequent calls
//
if( session.rstate.stage>=0 )
{
nin = session.rstate.ia[0];
nout = session.rstate.ia[1];
wcount = session.rstate.ia[2];
twcount = session.rstate.ia[3];
ntype = session.rstate.ia[4];
ttype = session.rstate.ia[5];
i = session.rstate.ia[6];
j = session.rstate.ia[7];
k = session.rstate.ia[8];
trnsetsize = session.rstate.ia[9];
epoch = session.rstate.ia[10];
minibatchcount = session.rstate.ia[11];
minibatchidx = session.rstate.ia[12];
cursize = session.rstate.ia[13];
idx0 = session.rstate.ia[14];
idx1 = session.rstate.ia[15];
decay = session.rstate.ra[0];
v = session.rstate.ra[1];
}
else
{
nin = -983;
nout = -989;
wcount = -834;
twcount = 900;
ntype = -287;
ttype = 364;
i = 214;
j = -338;
k = -686;
trnsetsize = 912;
epoch = 585;
minibatchcount = 497;
minibatchidx = -271;
cursize = -581;
idx0 = 745;
idx1 = -533;
decay = -77;
v = 678;
}
if( session.rstate.stage==0 )
{
goto lbl_0;
}
//
// Routine body
//
//
// Check correctness of inputs
//
alglib.ap.assert(s.npoints>=0, "MLPContinueTrainingX: internal error - parameter S is not initialized or is spoiled(S.NPoints<0).");
if( s.rcpar )
{
ttype = 0;
}
else
{
ttype = 1;
}
if( !mlpbase.mlpissoftmax(session.network) )
{
ntype = 0;
}
else
{
ntype = 1;
}
alglib.ap.assert(ntype==ttype, "MLPContinueTrainingX: internal error - type of the resulting network is not similar to network type in trainer object.");
mlpbase.mlpproperties(session.network, ref nin, ref nout, ref wcount);
alglib.ap.assert(s.nin==nin, "MLPContinueTrainingX: internal error - number of inputs in trainer is not equal to number of inputs in the network.");
alglib.ap.assert(s.nout==nout, "MLPContinueTrainingX: internal error - number of outputs in trainer is not equal to number of outputs in the network.");
alglib.ap.assert(alglib.ap.len(subset)>=subsetsize, "MLPContinueTrainingX: internal error - parameter SubsetSize more than input subset size(Length(Subset)<SubsetSize).");
for(i=0; i<=subsetsize-1; i++)
{
alglib.ap.assert(subset[i]>=0 && subset[i]<=s.npoints-1, "MLPContinueTrainingX: internal error - parameter Subset contains incorrect index(Subset[I]<0 or Subset[I]>S.NPoints-1).");
}
//
// Quick exit on empty training set
//
if( s.npoints==0 || subsetsize==0 )
{
result = false;
return result;
}
//
// Minibatch training
//
if( session.algoused==1 )
{
alglib.ap.assert(false, "MINIBATCH TRAINING IS NOT IMPLEMENTED YET");
}
//
// Last option: full batch training
//
decay = s.decay;
lbl_1:
if( !minlbfgs.minlbfgsiteration(session.optimizer) )
{
goto lbl_2;
}
if( !session.optimizer.xupdated )
{
goto lbl_3;
}
for(i_=0; i_<=wcount-1;i_++)
{
session.network.weights[i_] = session.optimizer.x[i_];
}
session.rstate.stage = 0;
goto lbl_rcomm;
lbl_0:
lbl_3:
for(i_=0; i_<=wcount-1;i_++)
{
session.network.weights[i_] = session.optimizer.x[i_];
}
if( s.datatype==0 )
{
mlpbase.mlpgradbatchsubset(session.network, s.densexy, s.npoints, subset, subsetsize, ref session.optimizer.f, ref session.optimizer.g);
}
if( s.datatype==1 )
{
mlpbase.mlpgradbatchsparsesubset(session.network, s.sparsexy, s.npoints, subset, subsetsize, ref session.optimizer.f, ref session.optimizer.g);
}
//
// Increment number of operations performed on batch gradient
//
ngradbatch = ngradbatch+1;
v = 0.0;
for(i_=0; i_<=wcount-1;i_++)
{
v += session.network.weights[i_]*session.network.weights[i_];
}
session.optimizer.f = session.optimizer.f+0.5*decay*v;
for(i_=0; i_<=wcount-1;i_++)
{
session.optimizer.g[i_] = session.optimizer.g[i_] + decay*session.network.weights[i_];
}
goto lbl_1;
lbl_2:
minlbfgs.minlbfgsresultsbuf(session.optimizer, ref session.network.weights, session.optimizerrep);
result = false;
return result;
//
// Saving state
//
lbl_rcomm:
result = true;
session.rstate.ia[0] = nin;
session.rstate.ia[1] = nout;
session.rstate.ia[2] = wcount;
session.rstate.ia[3] = twcount;
session.rstate.ia[4] = ntype;
session.rstate.ia[5] = ttype;
session.rstate.ia[6] = i;
session.rstate.ia[7] = j;
session.rstate.ia[8] = k;
session.rstate.ia[9] = trnsetsize;
session.rstate.ia[10] = epoch;
session.rstate.ia[11] = minibatchcount;
session.rstate.ia[12] = minibatchidx;
session.rstate.ia[13] = cursize;
session.rstate.ia[14] = idx0;
session.rstate.ia[15] = idx1;
session.rstate.ra[0] = decay;
session.rstate.ra[1] = v;
return result;
}
/*************************************************************************
This function trains neural network passed to this function, using current
dataset (one which was passed to MLPSetDataset() or MLPSetSparseDataset())
and current training settings. Training from NRestarts random starting
positions is performed, best network is chosen.
Training is performed using current training algorithm.
FOR USERS OF COMMERCIAL EDITION:
! Commercial version of ALGLIB includes two important improvements of
! this function:
! * multicore support (C++ and C# computational cores)
! * SSE support (C++ computational core)
!
! Second improvement gives constant speedup (2-3X). First improvement
! gives close-to-linear speedup on multicore systems. Following
! operations can be executed in parallel:
! * NRestarts training sessions performed within each of
! cross-validation rounds (if NRestarts>1)
! * gradient calculation over large dataset (if dataset is large enough)
!
! In order to use multicore features you have to:
! * use commercial version of ALGLIB
! * call this function with "smp_" prefix, which indicates that
! multicore code will be used (for multicore support)
!
! In order to use SSE features you have to:
! * use commercial version of ALGLIB on Intel processors
! * use C++ computational core
!
! This note is given for users of commercial edition; if you use GPL
! edition, you still will be able to call smp-version of this function,
! but all computations will be done serially.
!
! We recommend you to carefully read ALGLIB Reference Manual, section
! called 'SMP support', before using parallel version of this function.
INPUT PARAMETERS:
S - trainer object
Network - neural network. It must have same number of inputs and
output/classes as was specified during creation of the
trainer object.
NRestarts - number of restarts, >=0:
* NRestarts>0 means that specified number of random
restarts are performed, best network is chosen after
training
* NRestarts=0 means that current state of the network
is used for training.
OUTPUT PARAMETERS:
Network - trained network
NOTE: when no dataset was specified with MLPSetDataset/SetSparseDataset(),
network is filled by zero values. Same behavior for functions
MLPStartTraining and MLPContinueTraining.
NOTE: this method uses sum-of-squares error function for training.
-- ALGLIB --
Copyright 23.07.2012 by Bochkanov Sergey
*************************************************************************/
public static void mlptrainnetwork(mlptrainer s,
mlpbase.multilayerperceptron network,
int nrestarts,
mlpreport rep)
{
int nin = 0;
int nout = 0;
int wcount = 0;
int ntype = 0;
int ttype = 0;
alglib.smp.shared_pool trnpool = new alglib.smp.shared_pool();
alglib.ap.assert(s.npoints>=0, "MLPTrainNetwork: parameter S is not initialized or is spoiled(S.NPoints<0)");
if( !mlpbase.mlpissoftmax(network) )
{
ntype = 0;
}
else
{
ntype = 1;
}
if( s.rcpar )
{
ttype = 0;
}
else
{
ttype = 1;
}
alglib.ap.assert(ntype==ttype, "MLPTrainNetwork: type of input network is not similar to network type in trainer object");
mlpbase.mlpproperties(network, ref nin, ref nout, ref wcount);
alglib.ap.assert(s.nin==nin, "MLPTrainNetwork: number of inputs in trainer is not equal to number of inputs in network");
alglib.ap.assert(s.nout==nout, "MLPTrainNetwork: number of outputs in trainer is not equal to number of outputs in network");
alglib.ap.assert(nrestarts>=0, "MLPTrainNetwork: NRestarts<0.");
//
// Train
//
mlptrainnetworkx(s, nrestarts, -1, s.subset, -1, s.subset, 0, network, rep, true, trnpool);
}
/*************************************************************************
This function initializes temporaries needed for training session.
*************************************************************************/
private static void initmlptrnsessions(mlpbase.multilayerperceptron networktrained,
bool randomizenetwork,
mlptrainer trainer,
alglib.smp.shared_pool sessions)
{
int[] dummysubset = new int[0];
smlptrnsession t = new smlptrnsession();
smlptrnsession p = null;
if( alglib.smp.ae_shared_pool_is_initialized(sessions) )
{
//
// Pool was already initialized.
// Clear sessions stored in the pool.
//
alglib.smp.ae_shared_pool_first_recycled(sessions, ref p);
while( p!=null )
{
alglib.ap.assert(mlpbase.mlpsamearchitecture(p.network, networktrained), "InitMLPTrnSessions: internal consistency error");
p.bestrmserror = math.maxrealnumber;
alglib.smp.ae_shared_pool_next_recycled(sessions, ref p);
}
}
else
{
//
// Prepare session and seed pool
//
initmlptrnsession(networktrained, randomizenetwork, trainer, t);
alglib.smp.ae_shared_pool_set_seed(sessions, t);
}
}
/*************************************************************************
Single-threaded stub. HPC ALGLIB replaces it by multithreaded code.
*************************************************************************/
public static void _pexec_mlptrainnetwork(mlptrainer s,
mlpbase.multilayerperceptron network,
int nrestarts,
mlpreport rep)
{
mlptrainnetwork(s,network,nrestarts,rep);
}
/*************************************************************************
This function initializes temporaries needed for training session.
*************************************************************************/
private static void initmlpetrnsessions(mlpbase.multilayerperceptron individualnetwork,
mlptrainer trainer,
alglib.smp.shared_pool sessions)
{
mlpetrnsession t = new mlpetrnsession();
if( !alglib.smp.ae_shared_pool_is_initialized(sessions) )
{
initmlpetrnsession(individualnetwork, trainer, t);
alglib.smp.ae_shared_pool_set_seed(sessions, t);
}
}
/*************************************************************************
This function trains neural network ensemble passed to this function using
current dataset and early stopping training algorithm. Each early stopping
round performs NRestarts random restarts (thus, EnsembleSize*NRestarts
training rounds is performed in total).
INPUT PARAMETERS:
S - trainer object;
Ensemble - neural network ensemble. It must have same number of
inputs and outputs/classes as was specified during
creation of the trainer object.
NRestarts - number of restarts, >=0:
* NRestarts>0 means that specified number of random
restarts are performed during each ES round;
* NRestarts=0 is silently replaced by 1.
OUTPUT PARAMETERS:
Ensemble - trained ensemble;
Rep - it contains all type of errors.
NOTE: when no dataset was specified with MLPSetDataset/SetSparseDataset(),
or single-point dataset was passed, ensemble is filled by zero
values.
NOTE: this method uses sum-of-squares error function for training.
-- ALGLIB --
Copyright 22.08.2012 by Bochkanov Sergey
*************************************************************************/
public static void mlptrainensemblees(mlptrainer s,
mlpe.mlpensemble ensemble,
int nrestarts,
mlpreport rep)
{
int pcount = 0;
mlpreport tmprep = new mlpreport();
int nin = 0;
int nout = 0;
int wcount = 0;
int ntype = 0;
int ttype = 0;
int i = 0;
int k = 0;
int i_ = 0;
int i1_ = 0;
alglib.ap.assert(s.npoints>=0, "MLPTrainEnsembleES: parameter S is not initialized or is spoiled(S.NPoints<0)");
if( !mlpe.mlpeissoftmax(ensemble) )
{
ntype = 0;
}
else
{
ntype = 1;
}
if( s.rcpar )
{
ttype = 0;
}
else
{
ttype = 1;
}
alglib.ap.assert(ntype==ttype, "MLPTrainEnsembleES: internal error - type of input network is not similar to network type in trainer object");
nin = mlpbase.mlpgetinputscount(ensemble.network);
alglib.ap.assert(s.nin==nin, "MLPTrainEnsembleES: number of inputs in trainer is not equal to number of inputs in ensemble network");
nout = mlpbase.mlpgetoutputscount(ensemble.network);
alglib.ap.assert(s.nout==nout, "MLPTrainEnsembleES: number of outputs in trainer is not equal to number of outputs in ensemble network");
alglib.ap.assert(nrestarts>=0, "MLPTrainEnsembleES: NRestarts<0.");
wcount = mlpbase.mlpgetweightscount(ensemble.network);
//
// Initialize parameter Rep
//
rep.relclserror = 0;
rep.avgce = 0;
rep.rmserror = 0;
rep.avgerror = 0;
rep.avgrelerror = 0;
rep.ngrad = 0;
rep.nhess = 0;
rep.ncholesky = 0;
//
// Allocate
//
if( mlpbase.mlpissoftmax(ensemble.network) )
{
pcount = nin;
}
else
{
pcount = nin+nout;
}
apserv.ivectorsetlengthatleast(ref s.subset, s.npoints);
apserv.ivectorsetlengthatleast(ref s.valsubset, s.npoints);
apserv.rvectorsetlengthatleast(ref s.wbest, wcount);
apserv.rvectorsetlengthatleast(ref s.wfinal, wcount);
//
// Create LBFGS optimizer
//
minlbfgs.minlbfgscreate(wcount, Math.Min(wcount, s.lbfgsfactor), ensemble.network.weights, s.tstate);
minlbfgs.minlbfgssetcond(s.tstate, 0.0, 0.0, s.wstep, s.maxits);
minlbfgs.minlbfgssetxrep(s.tstate, true);
mlpbase.mlpcopy(ensemble.network, s.tnetwork);
//
// Train networks
//
if( (s.datatype==0 || s.datatype==1) && s.npoints>1 )
{
for(k=0; k<=ensemble.ensemblesize-1; k++)
{
//
// Split set
//
do
{
s.subsetsize = 0;
s.valsubsetsize = 0;
for(i=0; i<=s.npoints-1; i++)
{
if( (double)(math.randomreal())<(double)(0.66) )
{
//
// Assign sample to training set
//
s.subset[s.subsetsize] = i;
s.subsetsize = s.subsetsize+1;
}
else
{
//
// Assign sample to validation set
//
s.valsubset[s.valsubsetsize] = i;
s.valsubsetsize = s.valsubsetsize+1;
}
}
}
while( !(s.subsetsize!=0 && s.valsubsetsize!=0) );
//
// Train
//
mlptrainnetworkx(s, ensemble.network, s.tnetwork, s.tstate, nrestarts, s.subset, s.subsetsize, s.valsubset, s.valsubsetsize, s.wbest, s.wfinal, tmprep);
rep.ngrad = rep.ngrad+tmprep.ngrad;
//
// Save results
//
i1_ = (0) - (k*wcount);
for(i_=k*wcount; i_<=(k+1)*wcount-1;i_++)
{
ensemble.weights[i_] = ensemble.network.weights[i_+i1_];
}
i1_ = (0) - (k*pcount);
for(i_=k*pcount; i_<=(k+1)*pcount-1;i_++)
{
ensemble.columnmeans[i_] = ensemble.network.columnmeans[i_+i1_];
}
i1_ = (0) - (k*pcount);
for(i_=k*pcount; i_<=(k+1)*pcount-1;i_++)
{
ensemble.columnsigmas[i_] = ensemble.network.columnsigmas[i_+i1_];
}
}
}
else
{
for(i=0; i<=ensemble.ensemblesize*wcount-1; i++)
{
ensemble.network.weights[i] = 0.0;
ensemble.columnmeans[i] = 0.0;
ensemble.columnsigmas[i] = 1.0;
}
}
//
// Calculate errors.
//
if( s.datatype==0 )
{
mlpe.mlpeallerrors(ensemble, s.densexy, s.npoints, ref rep.relclserror, ref rep.avgce, ref rep.rmserror, ref rep.avgerror, ref rep.avgrelerror);
}
if( s.datatype==1 )
{
mlpe.mlpeallerrorssparse(ensemble, s.sparsexy, s.npoints, ref rep.relclserror, ref rep.avgce, ref rep.rmserror, ref rep.avgerror, ref rep.avgrelerror);
}
}