/*************************************************************************
This function initializes temporaries needed for training session.
*************************************************************************/
private static void initmlptrnsessions(mlpbase.multilayerperceptron networktrained,
bool randomizenetwork,
mlptrainer trainer,
alglib.smp.shared_pool sessions)
{
int[] dummysubset = new int[0];
smlptrnsession t = new smlptrnsession();
smlptrnsession p = null;
if( alglib.smp.ae_shared_pool_is_initialized(sessions) )
{
//
// Pool was already initialized.
// Clear sessions stored in the pool.
//
alglib.smp.ae_shared_pool_first_recycled(sessions, ref p);
while( p!=null )
{
alglib.ap.assert(mlpbase.mlpsamearchitecture(p.network, networktrained), "InitMLPTrnSessions: internal consistency error");
p.bestrmserror = math.maxrealnumber;
alglib.smp.ae_shared_pool_next_recycled(sessions, ref p);
}
}
else
{
//
// Prepare session and seed pool
//
initmlptrnsession(networktrained, randomizenetwork, trainer, t);
alglib.smp.ae_shared_pool_set_seed(sessions, t);
}
}
/*************************************************************************
This function initializes temporaries needed for training session.
-- ALGLIB --
Copyright 01.07.2013 by Bochkanov Sergey
*************************************************************************/
private static void initmlptrnsession(mlpbase.multilayerperceptron networktrained,
bool randomizenetwork,
mlptrainer trainer,
smlptrnsession session)
{
int nin = 0;
int nout = 0;
int wcount = 0;
int pcount = 0;
int[] dummysubset = new int[0];
//
// Prepare network:
// * copy input network to Session.Network
// * re-initialize preprocessor and weights if RandomizeNetwork=True
//
mlpbase.mlpcopy(networktrained, session.network);
if( randomizenetwork )
{
alglib.ap.assert(trainer.datatype==0 || trainer.datatype==1, "InitTemporaries: unexpected Trainer.DataType");
if( trainer.datatype==0 )
{
mlpbase.mlpinitpreprocessorsubset(session.network, trainer.densexy, trainer.npoints, dummysubset, -1);
}
if( trainer.datatype==1 )
{
mlpbase.mlpinitpreprocessorsparsesubset(session.network, trainer.sparsexy, trainer.npoints, dummysubset, -1);
}
mlpbase.mlprandomize(session.network);
session.randomizenetwork = true;
}
else
{
session.randomizenetwork = false;
}
//
// Determine network geometry and initialize optimizer
//
mlpbase.mlpproperties(session.network, ref nin, ref nout, ref wcount);
minlbfgs.minlbfgscreate(wcount, Math.Min(wcount, trainer.lbfgsfactor), session.network.weights, session.optimizer);
minlbfgs.minlbfgssetxrep(session.optimizer, true);
//
// Create buffers
//
session.wbuf0 = new double[wcount];
session.wbuf1 = new double[wcount];
//
// Initialize session result
//
mlpbase.mlpexporttunableparameters(session.network, ref session.bestparameters, ref pcount);
session.bestrmserror = math.maxrealnumber;
}
/*************************************************************************
This function performs step-by-step training of the neural network. Here
"step-by-step" means that training starts with MLPStartTrainingX call,
and then user subsequently calls MLPContinueTrainingX to perform one more
iteration of the training.
This function performs one more iteration of the training and returns
either True (training continues) or False (training stopped). In case True
was returned, Network weights are updated according to the current state
of the optimization progress. In case False was returned, no additional
updates is performed (previous update of the network weights moved us to
the final point, and no additional updates is needed).
EXAMPLE:
>
> [initialize network and trainer object]
>
> MLPStartTraining(Trainer, Network, True)
> while MLPContinueTraining(Trainer, Network) do
> [visualize training progress]
>
-- ALGLIB --
Copyright 13.08.2012 by Bochkanov Sergey
*************************************************************************/
private static bool mlpcontinuetrainingx(mlptrainer s,
int[] subset,
int subsetsize,
ref int ngradbatch,
smlptrnsession session)
{
bool result = new bool();
int nin = 0;
int nout = 0;
int wcount = 0;
int twcount = 0;
int ntype = 0;
int ttype = 0;
double decay = 0;
double v = 0;
int i = 0;
int j = 0;
int k = 0;
int trnsetsize = 0;
int epoch = 0;
int minibatchcount = 0;
int minibatchidx = 0;
int cursize = 0;
int idx0 = 0;
int idx1 = 0;
int i_ = 0;
//
// Reverse communication preparations
// I know it looks ugly, but it works the same way
// anywhere from C++ to Python.
//
// This code initializes locals by:
// * random values determined during code
// generation - on first subroutine call
// * values from previous call - on subsequent calls
//
if( session.rstate.stage>=0 )
{
nin = session.rstate.ia[0];
nout = session.rstate.ia[1];
wcount = session.rstate.ia[2];
twcount = session.rstate.ia[3];
ntype = session.rstate.ia[4];
ttype = session.rstate.ia[5];
i = session.rstate.ia[6];
j = session.rstate.ia[7];
k = session.rstate.ia[8];
trnsetsize = session.rstate.ia[9];
epoch = session.rstate.ia[10];
minibatchcount = session.rstate.ia[11];
minibatchidx = session.rstate.ia[12];
cursize = session.rstate.ia[13];
idx0 = session.rstate.ia[14];
idx1 = session.rstate.ia[15];
decay = session.rstate.ra[0];
v = session.rstate.ra[1];
}
else
{
nin = -983;
nout = -989;
wcount = -834;
twcount = 900;
ntype = -287;
ttype = 364;
i = 214;
j = -338;
k = -686;
trnsetsize = 912;
epoch = 585;
minibatchcount = 497;
minibatchidx = -271;
cursize = -581;
idx0 = 745;
idx1 = -533;
decay = -77;
v = 678;
}
if( session.rstate.stage==0 )
{
goto lbl_0;
}
//
// Routine body
//
//
// Check correctness of inputs
//
alglib.ap.assert(s.npoints>=0, "MLPContinueTrainingX: internal error - parameter S is not initialized or is spoiled(S.NPoints<0).");
if( s.rcpar )
{
ttype = 0;
}
else
{
ttype = 1;
}
if( !mlpbase.mlpissoftmax(session.network) )
{
ntype = 0;
}
else
{
ntype = 1;
}
alglib.ap.assert(ntype==ttype, "MLPContinueTrainingX: internal error - type of the resulting network is not similar to network type in trainer object.");
mlpbase.mlpproperties(session.network, ref nin, ref nout, ref wcount);
alglib.ap.assert(s.nin==nin, "MLPContinueTrainingX: internal error - number of inputs in trainer is not equal to number of inputs in the network.");
alglib.ap.assert(s.nout==nout, "MLPContinueTrainingX: internal error - number of outputs in trainer is not equal to number of outputs in the network.");
alglib.ap.assert(alglib.ap.len(subset)>=subsetsize, "MLPContinueTrainingX: internal error - parameter SubsetSize more than input subset size(Length(Subset)<SubsetSize).");
for(i=0; i<=subsetsize-1; i++)
{
alglib.ap.assert(subset[i]>=0 && subset[i]<=s.npoints-1, "MLPContinueTrainingX: internal error - parameter Subset contains incorrect index(Subset[I]<0 or Subset[I]>S.NPoints-1).");
}
//
// Quick exit on empty training set
//
if( s.npoints==0 || subsetsize==0 )
{
result = false;
return result;
}
//
// Minibatch training
//
if( session.algoused==1 )
{
alglib.ap.assert(false, "MINIBATCH TRAINING IS NOT IMPLEMENTED YET");
}
//
// Last option: full batch training
//
decay = s.decay;
lbl_1:
if( !minlbfgs.minlbfgsiteration(session.optimizer) )
{
goto lbl_2;
}
if( !session.optimizer.xupdated )
{
goto lbl_3;
}
for(i_=0; i_<=wcount-1;i_++)
{
session.network.weights[i_] = session.optimizer.x[i_];
}
session.rstate.stage = 0;
goto lbl_rcomm;
lbl_0:
lbl_3:
for(i_=0; i_<=wcount-1;i_++)
{
session.network.weights[i_] = session.optimizer.x[i_];
}
if( s.datatype==0 )
{
mlpbase.mlpgradbatchsubset(session.network, s.densexy, s.npoints, subset, subsetsize, ref session.optimizer.f, ref session.optimizer.g);
}
if( s.datatype==1 )
{
mlpbase.mlpgradbatchsparsesubset(session.network, s.sparsexy, s.npoints, subset, subsetsize, ref session.optimizer.f, ref session.optimizer.g);
}
//
// Increment number of operations performed on batch gradient
//
ngradbatch = ngradbatch+1;
v = 0.0;
for(i_=0; i_<=wcount-1;i_++)
{
v += session.network.weights[i_]*session.network.weights[i_];
}
session.optimizer.f = session.optimizer.f+0.5*decay*v;
for(i_=0; i_<=wcount-1;i_++)
{
session.optimizer.g[i_] = session.optimizer.g[i_] + decay*session.network.weights[i_];
}
goto lbl_1;
lbl_2:
minlbfgs.minlbfgsresultsbuf(session.optimizer, ref session.network.weights, session.optimizerrep);
result = false;
return result;
//
// Saving state
//
lbl_rcomm:
result = true;
session.rstate.ia[0] = nin;
session.rstate.ia[1] = nout;
session.rstate.ia[2] = wcount;
session.rstate.ia[3] = twcount;
session.rstate.ia[4] = ntype;
session.rstate.ia[5] = ttype;
session.rstate.ia[6] = i;
session.rstate.ia[7] = j;
session.rstate.ia[8] = k;
session.rstate.ia[9] = trnsetsize;
session.rstate.ia[10] = epoch;
session.rstate.ia[11] = minibatchcount;
session.rstate.ia[12] = minibatchidx;
session.rstate.ia[13] = cursize;
session.rstate.ia[14] = idx0;
session.rstate.ia[15] = idx1;
session.rstate.ra[0] = decay;
session.rstate.ra[1] = v;
return result;
}
/*************************************************************************
This function performs step-by-step training of the neural network. Here
"step-by-step" means that training starts with MLPStartTrainingX call,
and then user subsequently calls MLPContinueTrainingX to perform one more
iteration of the training.
After call to this function trainer object remembers network and is ready
to train it. However, no training is performed until first call to
MLPContinueTraining() function. Subsequent calls to MLPContinueTraining()
will advance traing progress one iteration further.
-- ALGLIB --
Copyright 13.08.2012 by Bochkanov Sergey
*************************************************************************/
private static void mlpstarttrainingx(mlptrainer s,
bool randomstart,
int algokind,
int[] subset,
int subsetsize,
smlptrnsession session)
{
int nin = 0;
int nout = 0;
int wcount = 0;
int ntype = 0;
int ttype = 0;
int i = 0;
//
// Check parameters
//
alglib.ap.assert(s.npoints>=0, "MLPStartTrainingX: internal error - parameter S is not initialized or is spoiled(S.NPoints<0)");
alglib.ap.assert(algokind==0 || algokind==-1, "MLPStartTrainingX: unexpected AlgoKind");
if( s.rcpar )
{
ttype = 0;
}
else
{
ttype = 1;
}
if( !mlpbase.mlpissoftmax(session.network) )
{
ntype = 0;
}
else
{
ntype = 1;
}
alglib.ap.assert(ntype==ttype, "MLPStartTrainingX: internal error - type of the resulting network is not similar to network type in trainer object");
mlpbase.mlpproperties(session.network, ref nin, ref nout, ref wcount);
alglib.ap.assert(s.nin==nin, "MLPStartTrainingX: number of inputs in trainer is not equal to number of inputs in the network.");
alglib.ap.assert(s.nout==nout, "MLPStartTrainingX: number of outputs in trainer is not equal to number of outputs in the network.");
alglib.ap.assert(alglib.ap.len(subset)>=subsetsize, "MLPStartTrainingX: internal error - parameter SubsetSize more than input subset size(Length(Subset)<SubsetSize)");
for(i=0; i<=subsetsize-1; i++)
{
alglib.ap.assert(subset[i]>=0 && subset[i]<=s.npoints-1, "MLPStartTrainingX: internal error - parameter Subset contains incorrect index(Subset[I]<0 or Subset[I]>S.NPoints-1)");
}
//
// Prepare session
//
minlbfgs.minlbfgssetcond(session.optimizer, 0.0, 0.0, s.wstep, s.maxits);
if( s.npoints>0 && subsetsize!=0 )
{
if( randomstart )
{
mlpbase.mlprandomize(session.network);
}
minlbfgs.minlbfgsrestartfrom(session.optimizer, session.network.weights);
}
else
{
for(i=0; i<=wcount-1; i++)
{
session.network.weights[i] = 0;
}
}
if( algokind==-1 )
{
session.algoused = s.algokind;
if( s.algokind==1 )
{
session.minibatchsize = s.minibatchsize;
}
}
else
{
session.algoused = 0;
}
hqrnd.hqrndrandomize(session.generator);
session.rstate.ia = new int[15+1];
session.rstate.ra = new double[1+1];
session.rstate.stage = -1;
}
public override void init()
{
densexy = new double[0,0];
sparsexy = new sparse.sparsematrix();
session = new smlptrnsession();
subset = new int[0];
valsubset = new int[0];
}
public override alglib.apobject make_copy()
{
smlptrnsession _result = new smlptrnsession();
_result.bestparameters = (double[])bestparameters.Clone();
_result.bestrmserror = bestrmserror;
_result.randomizenetwork = randomizenetwork;
_result.network = (mlpbase.multilayerperceptron)network.make_copy();
_result.optimizer = (minlbfgs.minlbfgsstate)optimizer.make_copy();
_result.optimizerrep = (minlbfgs.minlbfgsreport)optimizerrep.make_copy();
_result.wbuf0 = (double[])wbuf0.Clone();
_result.wbuf1 = (double[])wbuf1.Clone();
_result.allminibatches = (int[])allminibatches.Clone();
_result.currentminibatch = (int[])currentminibatch.Clone();
_result.rstate = (rcommstate)rstate.make_copy();
_result.algoused = algoused;
_result.minibatchsize = minibatchsize;
_result.generator = (hqrnd.hqrndstate)generator.make_copy();
return _result;
}
