/*************************************************************************
This function updates preconditioner for L-BFGS optimizer.
Parameters:
PrecType - preconditioner type:
* 0 for unpreconditioned iterations
* 1 for inexact LBFGS
* 2 for exact preconditioner update after each UpdateFreq its
UpdateFreq - update frequency
PrecCounter - iterations counter, must be zero on the first call,
automatically increased by this function. This
counter is used to implement "update-once-in-X-iterations"
scheme.
AULOptimizer - optimizer to tune
X - current point
Rho - penalty term
GammaK - current estimate of Hessian norm (used for
initialization of preconditioner). Can be zero, in
which case Hessian is assumed to be unit.
-- ALGLIB --
Copyright 06.06.2014 by Bochkanov Sergey
*************************************************************************/
private static void updatepreconditioner(int prectype,
int updatefreq,
ref int preccounter,
minlbfgs.minlbfgsstate auloptimizer,
double[] x,
double rho,
double gammak,
double[] bndl,
bool[] hasbndl,
double[] bndu,
bool[] hasbndu,
double[] nubc,
double[,] cleic,
double[] nulc,
double[] fi,
double[,] jac,
double[] nunlc,
ref double[] bufd,
ref double[] bufc,
ref double[,] bufw,
int n,
int nec,
int nic,
int ng,
int nh)
{
int i = 0;
double v = 0;
double p = 0;
double dp = 0;
double d2p = 0;
int i_ = 0;
alglib.ap.assert((double)(rho)>(double)(0), "MinNLC: integrity check failed");
apserv.rvectorsetlengthatleast(ref bufd, n);
apserv.rvectorsetlengthatleast(ref bufc, nec+nic+ng+nh);
apserv.rmatrixsetlengthatleast(ref bufw, nec+nic+ng+nh, n);
//
// Preconditioner before update from barrier/penalty functions
//
if( (double)(gammak)==(double)(0) )
{
gammak = 1;
}
for(i=0; i<=n-1; i++)
{
bufd[i] = gammak;
}
//
// Update diagonal Hessian using nonlinearity from boundary constraints:
// * penalty term from equality constraints
// * shift term from inequality constraints
//
// NOTE: penalty term for inequality constraints is ignored because it
// is large only in exceptional cases.
//
for(i=0; i<=n-1; i++)
{
if( (hasbndl[i] && hasbndu[i]) && (double)(bndl[i])==(double)(bndu[i]) )
{
minnlcequalitypenaltyfunction((x[i]-bndl[i])*rho, ref p, ref dp, ref d2p);
bufd[i] = bufd[i]+d2p*rho;
continue;
}
if( hasbndl[i] )
{
minnlcinequalityshiftfunction((x[i]-bndl[i])*rho+1, ref p, ref dp, ref d2p);
bufd[i] = bufd[i]+nubc[2*i+0]*d2p*rho;
}
if( hasbndu[i] )
{
minnlcinequalityshiftfunction((bndu[i]-x[i])*rho+1, ref p, ref dp, ref d2p);
bufd[i] = bufd[i]+nubc[2*i+1]*d2p*rho;
}
}
//
// Process linear constraints
//
for(i=0; i<=nec+nic-1; i++)
{
for(i_=0; i_<=n-1;i_++)
{
bufw[i,i_] = cleic[i,i_];
}
v = 0.0;
for(i_=0; i_<=n-1;i_++)
{
v += cleic[i,i_]*x[i_];
}
v = v-cleic[i,n];
if( i<nec )
{
//
// Equality constraint
//
minnlcequalitypenaltyfunction(v*rho, ref p, ref dp, ref d2p);
bufc[i] = d2p*rho;
}
else
{
//
// Inequality constraint
//
minnlcinequalityshiftfunction(-(v*rho)+1, ref p, ref dp, ref d2p);
bufc[i] = nulc[i]*d2p*rho;
}
}
//
// Process nonlinear constraints
//
for(i=0; i<=ng+nh-1; i++)
{
for(i_=0; i_<=n-1;i_++)
{
bufw[nec+nic+i,i_] = jac[1+i,i_];
}
v = fi[1+i];
if( i<ng )
{
//
// Equality constraint
//
minnlcequalitypenaltyfunction(v*rho, ref p, ref dp, ref d2p);
bufc[nec+nic+i] = d2p*rho;
}
else
{
//
// Inequality constraint
//
minnlcinequalityshiftfunction(-(v*rho)+1, ref p, ref dp, ref d2p);
bufc[nec+nic+i] = nunlc[i]*d2p*rho;
}
}
if( prectype==1 )
{
minlbfgs.minlbfgssetprecrankklbfgsfast(auloptimizer, bufd, bufc, bufw, nec+nic+ng+nh);
}
if( prectype==2 && preccounter%updatefreq==0 )
{
minlbfgs.minlbfgssetpreclowrankexact(auloptimizer, bufd, bufc, bufw, nec+nic+ng+nh);
}
apserv.inc(ref preccounter);
}
/*************************************************************************
Obsolete function, use MinLBFGSSetCholeskyPreconditioner() instead.
-- ALGLIB --
Copyright 13.10.2010 by Bochkanov Sergey
*************************************************************************/
public static void minlbfgssetcholeskypreconditioner(minlbfgs.minlbfgsstate state,
double[,] p,
bool isupper)
{
minlbfgs.minlbfgssetpreccholesky(state, p, isupper);
}
/*************************************************************************
This function clears preconditioner for L-BFGS optimizer (sets it do default
state);
Parameters:
AULOptimizer - optimizer to tune
-- ALGLIB --
Copyright 06.06.2014 by Bochkanov Sergey
*************************************************************************/
private static void clearpreconditioner(minlbfgs.minlbfgsstate auloptimizer)
{
minlbfgs.minlbfgssetprecdefault(auloptimizer);
}
public minlbfgsreport(minlbfgs.minlbfgsreport obj)
{
_innerobj = obj;
}
/*************************************************************************
Obsolete function, use MinLBFGSSetPrecDefault() instead.
-- ALGLIB --
Copyright 13.10.2010 by Bochkanov Sergey
*************************************************************************/
public static void minlbfgssetdefaultpreconditioner(minlbfgs.minlbfgsstate state)
{
minlbfgs.minlbfgssetprecdefault(state);
}
/*************************************************************************
Calculate test function IIP2
f(x) = sum( ((i*i+1)*x[i])^2, i=0..N-1)
It has high condition number which makes fast convergence unlikely without
good preconditioner.
*************************************************************************/
private static void calciip2(minlbfgs.minlbfgsstate state,
int n)
{
int i = 0;
if( state.needf | state.needfg )
{
state.f = 0;
}
for(i=0; i<=n-1; i++)
{
if( state.needf | state.needfg )
{
state.f = state.f+math.sqr(i*i+1)*math.sqr(state.x[i]);
}
if( state.needfg )
{
state.g[i] = math.sqr(i*i+1)*2*state.x[i];
}
}
}
public minlbfgsstate(minlbfgs.minlbfgsstate obj)
{
_innerobj = obj;
}
/*************************************************************************
Calculate test function #3
Simple variation of #1, much more nonlinear, with non-zero value at minimum.
It achieve two goals:
* makes unlikely premature convergence of algorithm .
* solves some issues with EpsF stopping condition which arise when
F(minimum) is zero
*************************************************************************/
private static void testfunc3(minlbfgs.minlbfgsstate state)
{
double s = 0;
s = 0.001;
if( (double)(state.x[0])<(double)(100) )
{
if( state.needf | state.needfg )
{
state.f = math.sqr(Math.Exp(state.x[0])-2)+math.sqr(math.sqr(state.x[1])+s)+math.sqr(state.x[2]-state.x[0]);
}
if( state.needfg )
{
state.g[0] = 2*(Math.Exp(state.x[0])-2)*Math.Exp(state.x[0])+2*(state.x[0]-state.x[2]);
state.g[1] = 2*(math.sqr(state.x[1])+s)*2*state.x[1];
state.g[2] = 2*(state.x[2]-state.x[0]);
}
}
else
{
if( state.needf | state.needfg )
{
state.f = Math.Sqrt(math.maxrealnumber);
}
if( state.needfg )
{
state.g[0] = Math.Sqrt(math.maxrealnumber);
state.g[1] = 0;
state.g[2] = 0;
}
}
}
private static void testfunc2(minlbfgs.minlbfgsstate state)
{
if( (double)(state.x[0])<(double)(100) )
{
state.f = math.sqr(Math.Exp(state.x[0])-2)+math.sqr(math.sqr(state.x[1]))+math.sqr(state.x[2]-state.x[0]);
state.g[0] = 2*(Math.Exp(state.x[0])-2)*Math.Exp(state.x[0])+2*(state.x[0]-state.x[2]);
state.g[1] = 4*state.x[1]*math.sqr(state.x[1]);
state.g[2] = 2*(state.x[2]-state.x[0]);
}
else
{
state.f = Math.Sqrt(math.maxrealnumber);
state.g[0] = Math.Sqrt(math.maxrealnumber);
state.g[1] = 0;
state.g[2] = 0;
}
}
/*************************************************************************
This function performs step-by-step training of the neural network. Here
"step-by-step" means that training starts with MLPStartTrainingX call,
and then user subsequently calls MLPContinueTrainingX to perform one more
iteration of the training.
This function performs one more iteration of the training and returns
either True (training continues) or False (training stopped). In case True
was returned, Network weights are updated according to the current state
of the optimization progress. In case False was returned, no additional
updates is performed (previous update of the network weights moved us to
the final point, and no additional updates is needed).
EXAMPLE:
>
> [initialize network and trainer object]
>
> MLPStartTraining(Trainer, Network, True)
> while MLPContinueTraining(Trainer, Network) do
> [visualize training progress]
>
INPUT PARAMETERS:
S - trainer object
Network - neural network which receives A COPY of the actual
network which is trained by the algorithm. After each
training roung state of the network being trained is
copied to this variable.
It must have same number of inputs and output/classes
as was specified during creation of the trainer object
and it must have exactly same architecture as the
second network (TNetwork).
TNetwork - neural network being trained.
State - LBFGS optimizer, already initialized, number of
dimensions must be equal to number of weights in the
networks.
Subset - some subset from training set(it stores row's numbers);
SubsetSize - size of subset(if SubsetSize<0 - used full dataset).
NGradBatch - number of calls MLPGradBatch function. Initial value
is zero;
OUTPUT PARAMETERS:
Network - weights of the neural network are rewritten by the
current approximation;
NGradBatch - number of calls MLPGradBatch function after training.
NOTE: this method uses sum-of-squares error function for training.
NOTE: it is expected that trainer object settings are NOT changed during
step-by-step training, i.e. no one changes stopping criteria or
training set during training. It is possible and there is no defense
against such actions, but algorithm behavior in such cases is
undefined and can be unpredictable.
NOTE: It is expected that Network is the same one which was passed to
MLPStartTraining() function. However, THIS function checks only
following:
* that number of network inputs is consistent with trainer object
settings
* that number of network outputs/classes is consistent with trainer
object settings
* that number of network weights is the same as number of weights in
the network passed to MLPStartTraining() function
Exception is thrown when these conditions are violated.
It is also expected that you do not change state of the network on
your own - the only party who has right to change network during its
training is a trainer object. Any attempt to interfere with trainer
may lead to unpredictable results.
-- ALGLIB --
Copyright 13.08.2012 by Bochkanov Sergey
*************************************************************************/
private static bool mlpcontinuetrainingx(mlptrainer s,
mlpbase.multilayerperceptron network,
mlpbase.multilayerperceptron tnetwork,
minlbfgs.minlbfgsstate state,
int[] subset,
int subsetsize,
ref int ngradbatch)
{
bool result = new bool();
int nin = 0;
int nout = 0;
int wcount = 0;
int twcount = 0;
int ntype = 0;
int ttype = 0;
double decay = 0;
double v = 0;
int i = 0;
int i_ = 0;
alglib.ap.assert(s.npoints>=0, "MLPContinueTrainingX: internal error - parameter S is not initialized or is spoiled(S.NPoints<0).");
if( s.rcpar )
{
ttype = 0;
}
else
{
ttype = 1;
}
if( !mlpbase.mlpissoftmax(network) )
{
ntype = 0;
}
else
{
ntype = 1;
}
alglib.ap.assert(ntype==ttype, "MLPContinueTrainingX: internal error - type of the resulting network is not similar to network type in trainer object.");
if( !mlpbase.mlpissoftmax(tnetwork) )
{
ntype = 0;
}
else
{
ntype = 1;
}
alglib.ap.assert(ntype==ttype, "MLPContinueTrainingX: internal error - type of the training network is not similar to network type in trainer object.");
mlpbase.mlpproperties(network, ref nin, ref nout, ref wcount);
alglib.ap.assert(s.nin==nin, "MLPContinueTrainingX: internal error - number of inputs in trainer is not equal to number of inputs in the network.");
alglib.ap.assert(s.nout==nout, "MLPContinueTrainingX: internal error - number of outputs in trainer is not equal to number of outputs in the network.");
mlpbase.mlpproperties(tnetwork, ref nin, ref nout, ref twcount);
alglib.ap.assert(s.nin==nin, "MLPContinueTrainingX: internal error - number of inputs in trainer is not equal to number of inputs in the training network.");
alglib.ap.assert(s.nout==nout, "MLPContinueTrainingX: internal error - number of outputs in trainer is not equal to number of outputs in the training network.");
alglib.ap.assert(twcount==wcount, "MLPContinueTrainingX: internal error - number of weights the resulting network is not equal to number of weights in the training network.");
alglib.ap.assert(alglib.ap.len(subset)>=subsetsize, "MLPContinueTrainingX: internal error - parameter SubsetSize more than input subset size(Length(Subset)<SubsetSize).");
for(i=0; i<=subsetsize-1; i++)
{
alglib.ap.assert(subset[i]>=0 && subset[i]<=s.npoints-1, "MLPContinueTrainingX: internal error - parameter Subset contains incorrect index(Subset[I]<0 or Subset[I]>S.NPoints-1).");
}
if( ((s.datatype==0 || s.datatype==1) && s.npoints>0) && subsetsize!=0 )
{
decay = s.decay;
while( minlbfgs.minlbfgsiteration(state) )
{
if( state.xupdated )
{
for(i_=0; i_<=wcount-1;i_++)
{
network.weights[i_] = tnetwork.weights[i_];
}
result = true;
return result;
}
for(i_=0; i_<=wcount-1;i_++)
{
tnetwork.weights[i_] = state.x[i_];
}
if( s.datatype==0 )
{
mlpbase.mlpgradbatchsubset(tnetwork, s.densexy, s.npoints, subset, subsetsize, ref state.f, ref state.g);
}
if( s.datatype==1 )
{
mlpbase.mlpgradbatchsparsesubset(tnetwork, s.sparsexy, s.npoints, subset, subsetsize, ref state.f, ref state.g);
}
//
// Increment number of operations performed on batch gradient
//
ngradbatch = ngradbatch+1;
v = 0.0;
for(i_=0; i_<=wcount-1;i_++)
{
v += tnetwork.weights[i_]*tnetwork.weights[i_];
}
state.f = state.f+0.5*decay*v;
for(i_=0; i_<=wcount-1;i_++)
{
state.g[i_] = state.g[i_] + decay*tnetwork.weights[i_];
}
}
for(i_=0; i_<=wcount-1;i_++)
{
network.weights[i_] = tnetwork.weights[i_];
}
}
result = false;
return result;
}
/*************************************************************************
This function performs step-by-step training of the neural network. Here
"step-by-step" means that training starts with MLPStartTrainingX call,
and then user subsequently calls MLPContinueTrainingX to perform one more
iteration of the training.
After call to this function trainer object remembers network and is ready
to train it. However, no training is performed until first call to
MLPContinueTraining() function. Subsequent calls to MLPContinueTraining()
will advance traing progress one iteration further.
EXAMPLE:
>
> ...initialize network and trainer object....
>
> MLPStartTraining(Trainer, Network, True)
> while MLPContinueTraining(Trainer, Network) do
> ...visualize training progress...
>
INPUT PARAMETERS:
S - trainer object;
Network - neural network which receives A COPY of the actual
network which is trained by the algorithm. After each
training roung state of the network being trained is
copied to this variable.
It must have same number of inputs and output/classes
as was specified during creation of the trainer object
and it must have exactly same architecture as the
second network (TNetwork).
TNetwork - neural network being trained.
State - LBFGS optimizer, already initialized, number of
dimensions must be equal to number of weights in the
networks.
RandomStart - randomize network before training or not:
* True means that network is randomized and its
initial state (one which was passed to the trainer
object) is lost;
* False means that training is started from the
current state of the network.
Subset - some subset from training set(it stores row's numbers);
SubsetSize - size of subset(if SubsetSize<0 - used full dataset).
OUTPUT PARAMETERS:
Network - neural network which is ready to training (weights are
initialized, preprocessor is initialized using current
training set)
NOTE: this method uses sum-of-squares error function for training.
NOTE: it is expected that trainer object settings are NOT changed during
step-by-step training, i.e. no one changes stopping criteria or
training set during training. It is possible and there is no defense
against such actions, but algorithm behavior in such cases is
undefined and can be unpredictable.
-- ALGLIB --
Copyright 13.08.2012 by Bochkanov Sergey
*************************************************************************/
private static void mlpstarttrainingx(mlptrainer s,
mlpbase.multilayerperceptron network,
mlpbase.multilayerperceptron tnetwork,
minlbfgs.minlbfgsstate state,
bool randomstart,
int[] subset,
int subsetsize)
{
int nin = 0;
int nout = 0;
int wcount = 0;
int twcount = 0;
int ntype = 0;
int ttype = 0;
int i = 0;
int i_ = 0;
alglib.ap.assert(s.npoints>=0, "MLPStartTrainingX: internal error - parameter S is not initialized or is spoiled(S.NPoints<0)");
if( s.rcpar )
{
ttype = 0;
}
else
{
ttype = 1;
}
if( !mlpbase.mlpissoftmax(network) )
{
ntype = 0;
}
else
{
ntype = 1;
}
alglib.ap.assert(ntype==ttype, "MLPStartTrainingX: internal error - type of the resulting network is not similar to network type in trainer object");
if( !mlpbase.mlpissoftmax(tnetwork) )
{
ntype = 0;
}
else
{
ntype = 1;
}
alglib.ap.assert(ntype==ttype, "MLPStartTrainingX: internal error - type of the training network is not similar to network type in trainer object");
mlpbase.mlpproperties(network, ref nin, ref nout, ref wcount);
alglib.ap.assert(s.nin==nin, "MLPStartTrainingX: number of inputs in trainer is not equal to number of inputs in the network.");
alglib.ap.assert(s.nout==nout, "MLPStartTrainingX: number of outputs in trainer is not equal to number of outputs in the network.");
mlpbase.mlpproperties(tnetwork, ref nin, ref nout, ref twcount);
alglib.ap.assert(s.nin==nin, "MLPStartTrainingX: number of inputs in trainer is not equal to number of inputs in the training network.");
alglib.ap.assert(s.nout==nout, "MLPStartTrainingX: number of outputs in trainer is not equal to number of outputs in the training network.");
alglib.ap.assert(twcount==wcount, "MLPStartTrainingX: number of weights the resulting network is not equal to number of weights in the training network.");
alglib.ap.assert(alglib.ap.len(subset)>=subsetsize, "MLPStartTrainingX: internal error - parameter SubsetSize more than input subset size(Length(Subset)<SubsetSize)");
for(i=0; i<=subsetsize-1; i++)
{
alglib.ap.assert(subset[i]>=0 && subset[i]<=s.npoints-1, "MLPStartTrainingX: internal error - parameter Subset contains incorrect index(Subset[I]<0 or Subset[I]>S.NPoints-1)");
}
if( ((s.datatype==0 || s.datatype==1) && s.npoints>0) && subsetsize!=0 )
{
//
// Prepare
//
if( s.datatype==0 )
{
mlpbase.mlpinitpreprocessorsubset(network, s.densexy, s.npoints, subset, subsetsize);
mlpbase.mlpinitpreprocessorsubset(tnetwork, s.densexy, s.npoints, subset, subsetsize);
}
if( s.datatype==1 )
{
mlpbase.mlpinitpreprocessorsparsesubset(network, s.sparsexy, s.npoints, subset, subsetsize);
mlpbase.mlpinitpreprocessorsparsesubset(tnetwork, s.sparsexy, s.npoints, subset, subsetsize);
}
//
// Process
//
if( randomstart )
{
mlpbase.mlprandomize(network);
}
minlbfgs.minlbfgsrestartfrom(state, network.weights);
}
else
{
for(i=0; i<=wcount-1; i++)
{
network.weights[i] = 0;
}
}
//
// Copy weights
//
for(i_=0; i_<=wcount-1;i_++)
{
tnetwork.weights[i_] = network.weights[i_];
}
}
/*************************************************************************
This function trains neural network passed to this function, using current
dataset (one which was passed to MLPSetDataset() or MLPSetSparseDataset())
and current training settings. Training from NRestarts random starting
positions is performed, best network is chosen.
Training is performed using current training algorithm.
INPUT PARAMETERS:
S - trainer object;
Network - neural network. It must have same number of inputs and
output/classes as was specified during creation of the
trainer object;
TNetwork - the training neural network.
User may look weights in parameter Network while
continue training process.
It has architecture like Network. You have to copy or
create new network with architecture like Network.
State - created LBFGS optimizer;
NRestarts - number of restarts, >=0:
* NRestarts>0 means that specified number of random
restarts are performed, best network is chosen after
training
* NRestarts=0 means that current state of the network
is used for training.
TrnSubset - some subset from training set(it stores row's numbers),
used as trainig set;
TrnSubsetSize- size of subset(if TrnSubsetSize<0 - used full dataset);
when TrnSubsetSize=0, network is filled by zero value,
and ValSubset parameter is IGNORED;
ValSubset - some subset from training set(it stores row's numbers),
used as validation set;
ValSubsetSize- size of subset(if ValSubsetSize<0 - used full dataset);
when ValSubsetSize<>0 this mean that is used early
stopping training algorithm;
BufWBest - buffer for storing interim resuls (BufWBest[0:WCOunt-1]
it has be allocated by user);
BufWFinal - buffer for storing interim resuls(BufWFinal[0:WCOunt-1]
it has be allocated by user).
OUTPUT PARAMETERS:
Network - trained network;
Rep - training report.
NOTE: when no dataset was specified with MLPSetDataset/SetSparseDataset(),
network is filled by zero values. Same behavior for functions
MLPStartTraining and MLPContinueTraining.
NOTE: this method uses sum-of-squares error function for training.
-- ALGLIB --
Copyright 13.08.2012 by Bochkanov Sergey
*************************************************************************/
private static void mlptrainnetworkx(mlptrainer s,
mlpbase.multilayerperceptron network,
mlpbase.multilayerperceptron tnetwork,
minlbfgs.minlbfgsstate state,
int nrestarts,
int[] trnsubset,
int trnsubsetsize,
int[] valsubset,
int valsubsetsize,
double[] bufwbest,
double[] bufwfinal,
mlpreport rep)
{
mlpbase.modelerrors modrep = new mlpbase.modelerrors();
double eval = 0;
double v = 0;
double ebestcur = 0;
double efinal = 0;
int ngradbatch = 0;
int nin = 0;
int nout = 0;
int wcount = 0;
int twcount = 0;
int itbest = 0;
int itcnt = 0;
int ntype = 0;
int ttype = 0;
bool rndstart = new bool();
int pass = 0;
int i = 0;
int i_ = 0;
alglib.ap.assert(s.npoints>=0, "MLPTrainNetworkX: internal error - parameter S is not initialized or is spoiled(S.NPoints<0)");
if( s.rcpar )
{
ttype = 0;
}
else
{
ttype = 1;
}
if( !mlpbase.mlpissoftmax(network) )
{
ntype = 0;
}
else
{
ntype = 1;
}
alglib.ap.assert(ntype==ttype, "MLPTrainNetworkX: internal error - type of the resulting network is not similar to network type in trainer object");
if( !mlpbase.mlpissoftmax(tnetwork) )
{
ntype = 0;
}
else
{
ntype = 1;
}
alglib.ap.assert(ntype==ttype, "MLPTrainNetworkX: internal error - type of the training network is not similar to network type in trainer object");
mlpbase.mlpproperties(network, ref nin, ref nout, ref wcount);
alglib.ap.assert(s.nin==nin, "MLPTrainNetworkX: internal error - number of inputs in trainer is not equal to number of inputs in the network.");
alglib.ap.assert(s.nout==nout, "MLPTrainNetworkX: internal error - number of outputs in trainer is not equal to number of outputs in the network.");
mlpbase.mlpproperties(tnetwork, ref nin, ref nout, ref twcount);
alglib.ap.assert(s.nin==nin, "MLPTrainNetworkX: internal error - number of inputs in trainer is not equal to number of inputs in the training network.");
alglib.ap.assert(s.nout==nout, "MLPTrainNetworkX: internal error - number of outputs in trainer is not equal to number of outputs in the training network.");
alglib.ap.assert(twcount==wcount, "MLPTrainNetworkX: internal error - number of weights the resulting network is not equal to number of weights in the training network.");
alglib.ap.assert(nrestarts>=0, "MLPTrainNetworkX: internal error - NRestarts<0.");
alglib.ap.assert(alglib.ap.len(trnsubset)>=trnsubsetsize, "MLPTrainNetworkX: internal error - parameter TrnSubsetSize more than input subset size(Length(TrnSubset)<TrnSubsetSize)");
for(i=0; i<=trnsubsetsize-1; i++)
{
alglib.ap.assert(trnsubset[i]>=0 && trnsubset[i]<=s.npoints-1, "MLPTrainNetworkX: internal error - parameter TrnSubset contains incorrect index(TrnSubset[I]<0 or TrnSubset[I]>S.NPoints-1)");
}
alglib.ap.assert(alglib.ap.len(valsubset)>=valsubsetsize, "MLPTrainNetworkX: internal error - parameter ValSubsetSize more than input subset size(Length(ValSubset)<ValSubsetSize)");
for(i=0; i<=valsubsetsize-1; i++)
{
alglib.ap.assert(valsubset[i]>=0 && valsubset[i]<=s.npoints-1, "MLPTrainNetworkX: internal error - parameter ValSubset contains incorrect index(ValSubset[I]<0 or ValSubset[I]>S.NPoints-1)");
}
//
// Initialize parameter Rep
//
rep.relclserror = 0;
rep.avgce = 0;
rep.rmserror = 0;
rep.avgerror = 0;
rep.avgrelerror = 0;
rep.ngrad = 0;
rep.nhess = 0;
rep.ncholesky = 0;
if( ((s.datatype==0 || s.datatype==1) && s.npoints>0) && trnsubsetsize!=0 )
{
//
// Prepare
//
efinal = math.maxrealnumber;
if( nrestarts!=0 )
{
rndstart = true;
}
else
{
rndstart = false;
nrestarts = 1;
}
ngradbatch = 0;
eval = 0;
ebestcur = 0;
for(pass=1; pass<=nrestarts; pass++)
{
mlpstarttrainingx(s, network, tnetwork, state, rndstart, trnsubset, trnsubsetsize);
itbest = 0;
itcnt = 0;
if( s.datatype==0 )
{
ebestcur = mlpbase.mlperrorsubset(network, s.densexy, s.npoints, valsubset, valsubsetsize);
}
if( s.datatype==1 )
{
ebestcur = mlpbase.mlperrorsparsesubset(network, s.sparsexy, s.npoints, valsubset, valsubsetsize);
}
for(i_=0; i_<=wcount-1;i_++)
{
bufwbest[i_] = network.weights[i_];
}
while( mlpcontinuetrainingx(s, network, tnetwork, state, trnsubset, trnsubsetsize, ref ngradbatch) )
{
if( s.datatype==0 )
{
eval = mlpbase.mlperrorsubset(network, s.densexy, s.npoints, valsubset, valsubsetsize);
}
if( s.datatype==1 )
{
eval = mlpbase.mlperrorsparsesubset(network, s.sparsexy, s.npoints, valsubset, valsubsetsize);
}
if( (double)(eval)<=(double)(ebestcur) )
{
for(i_=0; i_<=wcount-1;i_++)
{
bufwbest[i_] = network.weights[i_];
}
ebestcur = eval;
itbest = itcnt;
}
if( itcnt>30 && (double)(itcnt)>(double)(1.5*itbest) )
{
break;
}
itcnt = itcnt+1;
}
for(i_=0; i_<=wcount-1;i_++)
{
network.weights[i_] = bufwbest[i_];
}
//
// Compare with final(the best) answer.
//
v = 0.0;
for(i_=0; i_<=wcount-1;i_++)
{
v += bufwbest[i_]*bufwbest[i_];
}
if( s.datatype==0 )
{
ebestcur = mlpbase.mlperrorsubset(network, s.densexy, s.npoints, trnsubset, trnsubsetsize)+0.5*s.decay*v;
}
if( s.datatype==1 )
{
ebestcur = mlpbase.mlperrorsparsesubset(network, s.sparsexy, s.npoints, trnsubset, trnsubsetsize)+0.5*s.decay*v;
}
if( (double)(ebestcur)<(double)(efinal) )
{
for(i_=0; i_<=wcount-1;i_++)
{
bufwfinal[i_] = bufwbest[i_];
}
efinal = ebestcur;
}
}
//
// Final network
//
for(i_=0; i_<=wcount-1;i_++)
{
network.weights[i_] = bufwfinal[i_];
}
rep.ngrad = ngradbatch;
}
else
{
for(i=0; i<=wcount-1; i++)
{
network.weights[i] = 0;
}
}
//
// Calculate errors.
//
if( s.datatype==0 )
{
mlpbase.mlpallerrorssubset(network, s.densexy, s.npoints, trnsubset, trnsubsetsize, modrep);
}
if( s.datatype==1 )
{
mlpbase.mlpallerrorssparsesubset(network, s.sparsexy, s.npoints, trnsubset, trnsubsetsize, modrep);
}
rep.relclserror = modrep.relclserror;
rep.avgce = modrep.avgce;
rep.rmserror = modrep.rmserror;
rep.avgerror = modrep.avgerror;
rep.avgrelerror = modrep.avgrelerror;
}
