/// <summary>
/// Update the weights in the neural network.
/// </summary>
public void UpdateWeights()
{
var w = (double[])_weights.Clone();
for (int i = 0; i < w.Length; i++)
{
w[i] += _deltas[i];
}
NetworkCODEC.ArrayToNetwork(w, _network);
}
/// <summary>
/// Called just before a training iteration.
/// </summary>
public void PreIteration()
{
BasicNetwork network = this.train.Network;
if (network != null)
{
this.lastError = this.train.Error;
this.lastNetwork = NetworkCODEC.NetworkToArray(network);
this.train.Error = this.lastError;
}
}
/// <summary>
/// Decode the genomes into a neural network.
/// </summary>
///
public override sealed void Decode()
{
var net = new double[_networkChromosome.Genes.Count];
for (int i = 0; i < net.Length; i++)
{
var gene = (DoubleGene)_networkChromosome.Genes[i];
net[i] = gene.Value;
}
NetworkCODEC.ArrayToNetwork(net, (BasicNetwork)Organism);
}
public void TestRandomizeNeuronOutput()
{
double[] d = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 };
BasicNetwork network = EncogUtility.SimpleFeedForward(2, 3, 0, 1, false);
NetworkCODEC.ArrayToNetwork(d, network);
PruneSelective prune = new PruneSelective(network);
prune.RandomizeNeuron(100, 100, 2, 0);
Assert.AreEqual("100,100,100,100,0,0,0,0,0,0,0,0,0", network.DumpWeights());
}
/// <summary>
/// Perform one iteration.
/// </summary>
public override void Iteration()
{
LUDecomposition decomposition;
PreIteration();
_hessian.Clear();
_weights = NetworkCODEC.NetworkToArray(_network);
_hessian.Compute();
double currentError = _hessian.SSE;
SaveDiagonal();
double startingError = currentError;
bool done = false;
while (!done)
{
ApplyLambda();
decomposition = new LUDecomposition(_hessian.HessianMatrix);
if (decomposition.IsNonsingular)
{
_deltas = decomposition.Solve(_hessian.Gradients);
UpdateWeights();
currentError = CalculateError();
if (currentError < startingError)
{
_lambda /= LevenbergMarquardtTraining.ScaleLambda;
done = true;
}
}
if (!done)
{
_lambda *= LevenbergMarquardtTraining.ScaleLambda;
if (_lambda > LevenbergMarquardtTraining.LambdaMax)
{
_lambda = LevenbergMarquardtTraining.LambdaMax;
done = true;
}
}
}
Error = currentError;
PostIteration();
}
/// <summary>
/// Update the weights.
/// </summary>
///
/// <returns>The sum squared of the weights.</returns>
public double UpdateWeights()
{
double result = 0;
var w = (double[])_weights.Clone();
for (int i = 0; i < w.Length; i++)
{
w[i] += _deltas[i];
result += w[i] * w[i];
}
NetworkCODEC.ArrayToNetwork(w, _network);
return(result / 2.0d);
}
/// <summary>
/// Randomize the weights and thresholds. This function does most of the
/// work of the class. Each call to this class will randomize the data
/// according to the current temperature. The higher the temperature the
/// more randomness.
/// </summary>
public void Randomize()
{
double[] array = NetworkCODEC
.NetworkToArray(network);
for (int i = 0; i < array.Length; i++)
{
double add = NeuralSimulatedAnnealing.CUT - ThreadSafeRandom.NextDouble();
add /= this.anneal.StartTemperature;
add *= this.anneal.Temperature;
array[i] = array[i] + add;
}
NetworkCODEC.ArrayToNetwork(array, network);
}
/// <summary>
/// Randomize the weights and bias values. This function does most of the
/// work of the class. Each call to this class will randomize the data
/// according to the current temperature. The higher the temperature the more
/// randomness.
/// </summary>
///
public void Randomize()
{
double[] array = NetworkCODEC
.NetworkToArray(_network);
for (int i = 0; i < array.Length; i++)
{
double add = Cut - ThreadSafeRandom.NextDouble();
add /= _anneal.StartTemperature;
add *= _anneal.Temperature;
array[i] = array[i] + add;
}
NetworkCODEC.ArrayToNetwork(array,
_network);
}
/// <summary>
/// Update the weights.
/// </summary>
/// <returns>The sum squared of the weights.</returns>
public double UpdateWeights()
{
double result = 0;
double[] w = (double[])this.weights.Clone();
for (int i = 0; i < w.Length; i++)
{
w[i] += this.deltas[i];
result += w[i] * w[i];
}
NetworkCODEC.ArrayToNetwork(w, this.network);
return(result / 2.0);
}
//
// You can use the following additional attributes as you write your tests:
//
// Use ClassInitialize to run code before running the first test in the class
// [ClassInitialize()]
// public static void MyClassInitialize(TestContext testContext) { }
//
// Use ClassCleanup to run code after all tests in a class have run
// [ClassCleanup()]
// public static void MyClassCleanup() { }
//
// Use TestInitialize to run code before running each test
// [TestInitialize()]
// public void MyTestInitialize() { }
//
// Use TestCleanup to run code after each test has run
// [TestCleanup()]
// public void MyTestCleanup() { }
//
#endregion
private BasicNetwork ObtainNetwork()
{
BasicNetwork network = EncogUtility.SimpleFeedForward(2, 3, 0, 4, false);
double[] weights = { 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25 };
NetworkCODEC.ArrayToNetwork(weights, network);
Assert.AreEqual(1.0, network.GetWeight(1, 0, 0), 0.01);
Assert.AreEqual(2.0, network.GetWeight(1, 1, 0), 0.01);
Assert.AreEqual(3.0, network.GetWeight(1, 2, 0), 0.01);
Assert.AreEqual(4.0, network.GetWeight(1, 3, 0), 0.01);
Assert.AreEqual(5.0, network.GetWeight(1, 0, 1), 0.01);
Assert.AreEqual(6.0, network.GetWeight(1, 1, 1), 0.01);
Assert.AreEqual(7.0, network.GetWeight(1, 2, 1), 0.01);
Assert.AreEqual(8.0, network.GetWeight(1, 3, 1), 0.01);
Assert.AreEqual(9.0, network.GetWeight(1, 0, 2), 0.01);
Assert.AreEqual(10.0, network.GetWeight(1, 1, 2), 0.01);
Assert.AreEqual(11.0, network.GetWeight(1, 2, 2), 0.01);
Assert.AreEqual(12.0, network.GetWeight(1, 3, 2), 0.01);
Assert.AreEqual(13.0, network.GetWeight(1, 0, 3), 0.01);
Assert.AreEqual(14.0, network.GetWeight(1, 1, 3), 0.01);
Assert.AreEqual(15.0, network.GetWeight(1, 2, 3), 0.01);
Assert.AreEqual(16.0, network.GetWeight(1, 3, 3), 0.01);
Assert.AreEqual(17.0, network.GetWeight(0, 0, 0), 0.01);
Assert.AreEqual(18.0, network.GetWeight(0, 1, 0), 0.01);
Assert.AreEqual(19.0, network.GetWeight(0, 2, 0), 0.01);
Assert.AreEqual(20.0, network.GetWeight(0, 0, 1), 0.01);
Assert.AreEqual(21.0, network.GetWeight(0, 1, 1), 0.01);
Assert.AreEqual(22.0, network.GetWeight(0, 2, 1), 0.01);
Assert.AreEqual(20.0, network.GetWeight(0, 0, 1), 0.01);
Assert.AreEqual(21.0, network.GetWeight(0, 1, 1), 0.01);
Assert.AreEqual(22.0, network.GetWeight(0, 2, 1), 0.01);
Assert.AreEqual(23.0, network.GetWeight(0, 0, 2), 0.01);
Assert.AreEqual(24.0, network.GetWeight(0, 1, 2), 0.01);
Assert.AreEqual(25.0, network.GetWeight(0, 2, 2), 0.01);
return(network);
}
/// <summary>
/// Constructor.
/// </summary>
/// <param name="network">an initialised Encog network.
/// The networks in the swarm will be created with
/// the same topology as this network.</param>
/// <param name="randomizer">any type of Encog network weight initialisation
/// object.</param>
/// <param name="calculateScore">any type of Encog network scoring/fitness object.</param>
/// <param name="populationSize">the swarm size.</param>
public NeuralPSO(BasicNetwork network,
IRandomizer randomizer, ICalculateScore calculateScore,
int populationSize)
: base(TrainingImplementationType.Iterative)
{
// initialisation of the member variables
m_populationSize = populationSize;
m_randomizer = randomizer;
m_calculateScore = calculateScore;
m_bestNetwork = network;
m_networks = new BasicNetwork[m_populationSize];
m_velocities = null;
m_bestVectors = new double[m_populationSize][];
m_bestErrors = new double[m_populationSize];
m_bestVectorIndex = -1;
// get a vector from the network.
m_bestVector = NetworkCODEC.NetworkToArray(m_bestNetwork);
m_va = new VectorAlgebra();
}
/// <summary>
/// Called just after a training iteration.
/// </summary>
public void PostIteration()
{
if (this.ready)
{
if (this.train.Error > this.lastError)
{
#if logging
if (this.logger.IsDebugEnabled)
{
this.logger
.Debug("Greedy strategy dropped last iteration.");
}
#endif
this.train.Error = this.lastError;
NetworkCODEC.ArrayToNetwork(this.lastNetwork, this.train
.Network);
}
}
else
{
this.ready = true;
}
}
/// <inheritdoc/>
public override void Iteration()
{
if (_converged)
{
return;
}
int n = _start.Length;
for (int i = 0; i < n; i++)
{
_p[i + n * n] = _start[i];
}
_y[n] = Fn(_start);
for (int j = 0; j < n; j++)
{
double x = _start[j];
_start[j] = _start[j] + _step[j] * _del;
for (int i = 0; i < n; i++)
{
_p[i + j * n] = _start[i];
}
_y[j] = Fn(_start);
_start[j] = x;
}
/*
* The simplex construction is complete.
*
* Find highest and lowest Y values. YNEWLO = Y(IHI) indicates the
* vertex of the simplex to be replaced.
*/
_ylo = _y[0];
_ilo = 0;
for (int i = 1; i < _nn; i++)
{
if (_y[i] < _ylo)
{
_ylo = _y[i];
_ilo = i;
}
}
/*
* Inner loop.
*/
for (;;)
{
/*
* if (kcount <= icount) { break; }
*/
_ynewlo = _y[0];
_ihi = 0;
for (int i = 1; i < _nn; i++)
{
if (_ynewlo < _y[i])
{
_ynewlo = _y[i];
_ihi = i;
}
}
/*
* Calculate PBAR, the centroid of the simplex vertices excepting
* the vertex with Y value YNEWLO.
*/
for (int i = 0; i < n; i++)
{
_z = 0.0;
for (int j = 0; j < _nn; j++)
{
_z = _z + _p[i + j * n];
}
_z = _z - _p[i + _ihi * n];
_pbar[i] = _z / n;
}
/*
* Reflection through the centroid.
*/
for (int i = 0; i < n; i++)
{
_pstar[i] = _pbar[i] + rcoeff
* (_pbar[i] - _p[i + _ihi * n]);
}
_ystar = Fn(_pstar);
/*
* Successful reflection, so extension.
*/
if (_ystar < _ylo)
{
for (int i = 0; i < n; i++)
{
_p2Star[i] = _pbar[i] + ecoeff
* (_pstar[i] - _pbar[i]);
}
_y2Star = Fn(_p2Star);
/*
* Check extension.
*/
if (_ystar < _y2Star)
{
for (int i = 0; i < n; i++)
{
_p[i + _ihi * n] = _pstar[i];
}
_y[_ihi] = _ystar;
}
/*
* Retain extension or contraction.
*/
else
{
for (int i = 0; i < n; i++)
{
_p[i + _ihi * n] = _p2Star[i];
}
_y[_ihi] = _y2Star;
}
}
/*
* No extension.
*/
else
{
_l = 0;
for (int i = 0; i < _nn; i++)
{
if (_ystar < _y[i])
{
_l = _l + 1;
}
}
if (1 < _l)
{
for (int i = 0; i < n; i++)
{
_p[i + _ihi * n] = _pstar[i];
}
_y[_ihi] = _ystar;
}
/*
* Contraction on the Y(IHI) side of the centroid.
*/
else if (_l == 0)
{
for (int i = 0; i < n; i++)
{
_p2Star[i] = _pbar[i] + ccoeff
* (_p[i + _ihi * n] - _pbar[i]);
}
_y2Star = Fn(_p2Star);
/*
* Contract the whole simplex.
*/
if (_y[_ihi] < _y2Star)
{
for (int j = 0; j < _nn; j++)
{
for (int i = 0; i < n; i++)
{
_p[i + j * n] = (_p[i + j * n] + _p[i
+ _ilo * n]) * 0.5;
_trainedWeights[i] = _p[i + j * n];
}
_y[j] = Fn(_trainedWeights);
}
_ylo = _y[0];
_ilo = 0;
for (int i = 1; i < _nn; i++)
{
if (_y[i] < _ylo)
{
_ylo = _y[i];
_ilo = i;
}
}
continue;
}
/*
* Retain contraction.
*/
for (int i = 0; i < n; i++)
{
_p[i + _ihi * n] = _p2Star[i];
}
_y[_ihi] = _y2Star;
}
/*
* Contraction on the reflection side of the centroid.
*/
else if (_l == 1)
{
for (int i = 0; i < n; i++)
{
_p2Star[i] = _pbar[i] + ccoeff
* (_pstar[i] - _pbar[i]);
}
_y2Star = Fn(_p2Star);
/*
* Retain reflection?
*/
if (_y2Star <= _ystar)
{
for (int i = 0; i < n; i++)
{
_p[i + _ihi * n] = _p2Star[i];
}
_y[_ihi] = _y2Star;
}
else
{
for (int i = 0; i < n; i++)
{
_p[i + _ihi * n] = _pstar[i];
}
_y[_ihi] = _ystar;
}
}
}
/*
* Check if YLO improved.
*/
if (_y[_ihi] < _ylo)
{
_ylo = _y[_ihi];
_ilo = _ihi;
}
_jcount = _jcount - 1;
if (0 < _jcount)
{
continue;
}
/*
* Check to see if minimum reached.
*/
// if (icount <= kcount)
{
_jcount = _konvge;
_z = 0.0;
for (int i = 0; i < _nn; i++)
{
_z = _z + _y[i];
}
double x = _z / _nn;
_z = 0.0;
for (int i = 0; i < _nn; i++)
{
var inner = _y[i] - x;
_z = _z + inner * inner;
}
if (_z <= _rq)
{
break;
}
}
}
/*
* Factorial tests to check that YNEWLO is a local minimum.
*/
for (int i = 0; i < n; i++)
{
_trainedWeights[i] = _p[i + _ilo * n];
}
_ynewlo = _y[_ilo];
bool fault = false;
for (int i = 0; i < n; i++)
{
_del = _step[i] * eps;
_trainedWeights[i] += _del;
_z = Fn(_trainedWeights);
if (_z < _ynewlo)
{
fault = true;
break;
}
_trainedWeights[i] = _trainedWeights[i] - _del
- _del;
_z = Fn(_trainedWeights);
if (_z < _ynewlo)
{
fault = true;
break;
}
_trainedWeights[i] += _del;
}
if (!fault)
{
_converged = true;
}
else
{
/*
* Restart the procedure.
*/
for (int i = 0; i < n; i++)
{
_start[i] = _trainedWeights[i];
}
_del = eps;
}
Error = _ynewlo;
NetworkCODEC.ArrayToNetwork(_trainedWeights, _network);
}
/// <summary>
/// Calculate the error for the neural network with a given set of weights.
/// </summary>
/// <param name="weights">The weights to use.</param>
/// <returns>The current error.</returns>
public double Fn(double[] weights)
{
NetworkCODEC.ArrayToNetwork(weights, _network);
return(_network.CalculateError(Training));
}
/// <summary>
/// Convert an array of doubles to the current best network.
/// </summary>
///
/// <param name="array">An array.</param>
public void PutArray(double[] array)
{
NetworkCODEC.ArrayToNetwork(array,
_network);
}
/// <summary>
/// Sets the state of the networks in the swarm
/// </summary>
/// <param name="particleIndex">index of the network in the swarm</param>
/// <param name="state">an array of weights and biases</param>
protected void SetNetworkState(int particleIndex, double[] state)
{
NetworkCODEC.ArrayToNetwork(state, m_networks[particleIndex]);
}
/// <summary>
/// Returns the state of a network in the swarm
/// </summary>
/// <param name="particleIndex">index of the network in the swarm</param>
/// <returns>an array of weights and biases for the given network</returns>
protected double[] GetNetworkState(int particleIndex)
{
return(NetworkCODEC.NetworkToArray(m_networks[particleIndex]));
}
/// <summary>
/// Determine if this neural network is equal to another. Equal neural
/// networks have the same weight matrix and bias values, within a specified
/// precision.
/// </summary>
///
/// <param name="other">The other neural network.</param>
/// <param name="precision">The number of decimal places to compare to.</param>
/// <returns>True if the two neural networks are equal.</returns>
public bool Equals(BasicNetwork other, int precision)
{
return(NetworkCODEC.Equals(this, other, precision));
}
/// <summary>
/// Perform one iteration.
/// </summary>
///
public override void Iteration()
{
LUDecomposition decomposition = null;
PreIteration();
_weights = NetworkCODEC.NetworkToArray(_network);
IComputeJacobian j = new JacobianChainRule(_network,
_indexableTraining);
double sumOfSquaredErrors = j.Calculate(_weights);
double sumOfSquaredWeights = CalculateSumOfSquaredWeights();
// this.setError(j.getError());
CalculateHessian(j.Jacobian, j.RowErrors);
// Define the objective function
// bayesian regularization objective function
double objective = _beta * sumOfSquaredErrors + _alpha
* sumOfSquaredWeights;
double current = objective + 1.0d;
// Start the main Levenberg-Macquardt method
_lambda /= ScaleLambda;
// We'll try to find a direction with less error
// (or where the objective function is smaller)
while ((current >= objective) &&
(_lambda < LambdaMax))
{
_lambda *= ScaleLambda;
// Update diagonal (Levenberg-Marquardt formula)
for (int i = 0; i < _parametersLength; i++)
{
_hessian[i][i] = _diagonal[i]
+ (_lambda + _alpha);
}
// Decompose to solve the linear system
decomposition = new LUDecomposition(_hessianMatrix);
// Check if the Jacobian has become non-invertible
if (!decomposition.IsNonsingular)
{
continue;
}
// Solve using LU (or SVD) decomposition
_deltas = decomposition.Solve(_gradient);
// Update weights using the calculated deltas
sumOfSquaredWeights = UpdateWeights();
// Calculate the new error
sumOfSquaredErrors = 0.0d;
for (int i = 0; i < _trainingLength; i++)
{
_indexableTraining.GetRecord(i, _pair);
IMLData actual = _network
.Compute(_pair.Input);
double e = _pair.Ideal[0]
- actual[0];
sumOfSquaredErrors += e * e;
}
sumOfSquaredErrors /= 2.0d;
// Update the objective function
current = _beta * sumOfSquaredErrors + _alpha
* sumOfSquaredWeights;
// If the object function is bigger than before, the method
// is tried again using a greater dumping factor.
}
// If this iteration caused a error drop, then next iteration
// will use a smaller damping factor.
_lambda /= ScaleLambda;
if (_useBayesianRegularization && (decomposition != null))
{
// Compute the trace for the inverse Hessian
double trace = Trace(decomposition.Inverse());
// Poland update's formula:
_gamma = _parametersLength - (_alpha * trace);
_alpha = _parametersLength
/ (2.0d * sumOfSquaredWeights + trace);
_beta = Math.Abs((_trainingLength - _gamma)
/ (2.0d * sumOfSquaredErrors));
}
Error = sumOfSquaredErrors;
PostIteration();
}
/// <summary>
/// Get the network as an array of doubles.
/// </summary>
/// <returns>The network as an array of doubles.</returns>
public double[] GetArray()
{
return(NetworkCODEC
.NetworkToArray(network));
}
/// <summary>
/// Perform one iteration.
/// </summary>
public override void Iteration()
{
LUDecomposition decomposition = null;
double trace = 0;
PreIteration();
this.weights = NetworkCODEC.NetworkToArray(this.network);
IComputeJacobian j = new JacobianChainRule(this.network,
this.indexableTraining);
double sumOfSquaredErrors = j.Calculate(this.weights);
double sumOfSquaredWeights = CalculateSumOfSquaredWeights();
// this.setError(j.getError());
CalculateHessian(j.Jacobian, j.RowErrors);
// Define the objective function
// bayesian regularization objective function
double objective = this.beta * sumOfSquaredErrors + this.alpha
* sumOfSquaredWeights;
double current = objective + 1.0;
// Start the main Levenberg-Macquardt method
this.lambda /= LevenbergMarquardtTraining.SCALE_LAMBDA;
// We'll try to find a direction with less error
// (or where the objective function is smaller)
while ((current >= objective) &&
(this.lambda < LevenbergMarquardtTraining.LAMBDA_MAX))
{
this.lambda *= LevenbergMarquardtTraining.SCALE_LAMBDA;
// Update diagonal (Levenberg-Marquardt formula)
for (int i = 0; i < this.parametersLength; i++)
{
this.hessian[i][i] = this.diagonal[i]
+ (this.lambda + this.alpha);
}
// Decompose to solve the linear system
decomposition = new LUDecomposition(
this.hessianMatrix);
// Check if the Jacobian has become non-invertible
if (!decomposition.IsNonsingular)
{
continue;
}
// Solve using LU (or SVD) decomposition
this.deltas = decomposition.Solve(this.gradient);
// Update weights using the calculated deltas
sumOfSquaredWeights = UpdateWeights();
// Calculate the new error
sumOfSquaredErrors = 0.0;
for (int i = 0; i < this.trainingLength; i++)
{
this.indexableTraining.GetRecord(i, this.pair);
INeuralData actual = this.network.Compute(this.pair
.Input);
double e = this.pair.Ideal[0]
- actual[0];
sumOfSquaredErrors += e * e;
}
sumOfSquaredErrors /= 2.0;
// Update the objective function
current = this.beta * sumOfSquaredErrors + this.alpha
* sumOfSquaredWeights;
// If the object function is bigger than before, the method
// is tried again using a greater dumping factor.
}
// If this iteration caused a error drop, then next iteration
// will use a smaller damping factor.
this.lambda /= LevenbergMarquardtTraining.SCALE_LAMBDA;
if (useBayesianRegularization && decomposition != null)
{
// Compute the trace for the inverse Hessian
trace = Trace(decomposition.Inverse());
// Poland update's formula:
gamma = this.parametersLength - (alpha * trace);
alpha = this.parametersLength / (2.0 * sumOfSquaredWeights + trace);
beta = Math.Abs((this.trainingLength - gamma) / (2.0 * sumOfSquaredErrors));
}
this.Error = sumOfSquaredErrors;
PostIteration();
}
