/// <summary>
/// Create an instance of competitive training.
/// </summary>
/// <param name="network">The network to train.</param>
/// <param name="learningRate">The learning rate, how much to apply per iteration.</param>
/// <param name="training">The training set (unsupervised).</param>
/// <param name="neighborhood">The neighborhood function to use.</param>
public CompetitiveTraining(BasicNetwork network,
double learningRate, INeuralDataSet training,
INeighborhoodFunction neighborhood)
{
this.neighborhood = neighborhood;
Training = training;
this.LearningRate = learningRate;
this.network = network;
this.inputLayer = network.GetLayer(BasicNetwork.TAG_INPUT);
this.outputLayer = network.GetLayer(BasicNetwork.TAG_OUTPUT);
this.synapses = network.Structure.GetPreviousSynapses(
this.outputLayer);
this.inputNeuronCount = this.inputLayer.NeuronCount;
this.outputNeuronCount = this.outputLayer.NeuronCount;
this.ForceWinner = false;
Error = 0;
// setup the correction matrix
foreach (ISynapse synapse in this.synapses)
{
Matrix matrix = new Matrix(synapse.WeightMatrix.Rows,
synapse.WeightMatrix.Cols);
this.correctionMatrix[synapse] = matrix;
}
// create the BMU class
this.bmuUtil = new BestMatchingUnit(this);
}
/// <summary>
/// Validate that the neural network would work with the specified training set.
/// </summary>
/// <param name="network">The neural network that is to be evaluated.</param>
/// <param name="training">The training set that we should evaluate </param>
public static void ValidateNetworkForTraining(BasicNetwork network, INeuralDataSet training)
{
ILayer inputLayer = network.GetLayer(BasicNetwork.TAG_INPUT);
ILayer outputLayer = network.GetLayer(BasicNetwork.TAG_OUTPUT);
if (inputLayer == null)
{
throw new NeuralNetworkError("This operation requires that the neural network have an input layer.");
}
if (outputLayer == null)
{
throw new NeuralNetworkError("This operation requires that the neural network have an output layer.");
}
if (inputLayer.NeuronCount != training.InputSize)
{
throw new NeuralNetworkError("The input layer size of "
+ inputLayer.NeuronCount
+ " must match the training input size of "
+ training.InputSize + ".");
}
if (training.IdealSize > 0 &&
outputLayer.NeuronCount != training.IdealSize)
{
throw new NeuralNetworkError("The output layer size of "
+ inputLayer.NeuronCount
+ " must match the training input size of "
+ training.IdealSize + ".");
}
}
/// <summary>
/// Construct the LMA object.
/// </summary>
/// <param name="network">The network to train. Must have a single output neuron.</param>
/// <param name="training">The training data to use. Must be indexable.</param>
public SVDTraining(BasicNetwork network, INeuralDataSet training)
{
ILayer outputLayer = network.GetLayer(BasicNetwork.TAG_OUTPUT);
if (outputLayer == null)
{
throw new TrainingError("SVD requires an output layer.");
}
if (outputLayer.NeuronCount != 1)
{
throw new TrainingError("SVD requires an output layer with a single neuron.");
}
if (network.GetLayer(RadialBasisPattern.RBF_LAYER) == null)
{
throw new TrainingError("SVD is only tested to work on radial basis function networks.");
}
rbfLayer = (RadialBasisFunctionLayer)network.GetLayer(RadialBasisPattern.RBF_LAYER);
this.Training = training;
this.network = network;
this.trainingLength = (int)this.Training.InputSize;
BasicNeuralData input = new BasicNeuralData(this.Training.InputSize);
BasicNeuralData ideal = new BasicNeuralData(this.Training.IdealSize);
this.pair = new BasicNeuralDataPair(input, ideal);
}
/// <summary>
/// Setup the network logic, read parameters from the network.
/// </summary>
/// <param name="network">The network that this logic class belongs to.</param>
public void Init(BasicNetwork network)
{
this.network = network;
this.f1Layer = network.GetLayer(BAMPattern.TAG_F1);
this.f2Layer = network.GetLayer(BAMPattern.TAG_F2);
this.synapseF1ToF2 = network.Structure.FindSynapse(this.f1Layer, this.f2Layer, true);
this.synapseF2ToF1 = network.Structure.FindSynapse(this.f2Layer, this.f1Layer, true);
}
/// <summary>
/// Determine if the specified neural network can be flat. If it can a null
/// is returned, otherwise, an error is returned to show why the network
/// cannot be flattened.
/// </summary>
/// <param name="eml">The network to check.</param>
/// <returns>Null, if the net can not be flattened, an error message
/// otherwise.</returns>
public override String IsValid(IEngineMachineLearning eml)
{
if (!(eml is BasicNetwork))
{
return("Only a BasicNetwork can be converted to a flat network.");
}
BasicNetwork network = (BasicNetwork)eml;
ILayer inputLayer = network.GetLayer(BasicNetwork.TAG_INPUT);
ILayer outputLayer = network.GetLayer(BasicNetwork.TAG_OUTPUT);
if (inputLayer == null)
{
return("To convert to a flat network, there must be an input layer.");
}
if (outputLayer == null)
{
return("To convert to a flat network, there must be an output layer.");
}
if (!(network.Logic is FeedforwardLogic) ||
(network.Logic is ThermalLogic))
{
return("To convert to flat, must be using FeedforwardLogic or SimpleRecurrentLogic.");
}
foreach (ILayer layer in network.Structure.Layers)
{
if (layer.Next.Count > 2)
{
return("To convert to flat a network must have at most two outbound synapses.");
}
if (layer.GetType() != typeof(ContextLayer) &&
layer.GetType() != typeof(BasicLayer) &&
layer.GetType() != typeof(RadialBasisFunctionLayer))
{
return("To convert to flat a network must have only BasicLayer and ContextLayer layers.");
}
}
foreach (ISynapse synapse in network.Structure.Synapses)
{
if (synapse is NEATSynapse)
{
return("A NEAT synapse cannot be flattened.");
}
}
return(null);
}
/// <summary>
/// Format the network as a human readable string that lists the
/// hidden layers.
/// </summary>
/// <param name="network">The network to format.</param>
/// <returns>A human readable string.</returns>
public static String NetworkToString(BasicNetwork network)
{
StringBuilder result = new StringBuilder();
int num = 1;
ILayer layer = network.GetLayer(BasicNetwork.TAG_INPUT);
// display only hidden layers
while (layer.Next.Count > 0)
{
layer = layer.Next[0].ToLayer;
if (result.Length > 0)
{
result.Append(",");
}
result.Append("H");
result.Append(num++);
result.Append("=");
result.Append(layer.NeuronCount);
}
return result.ToString();
}
/// <summary>
/// Construct the LMA object.
/// </summary>
/// <param name="network">The network to train. Must have a single output neuron.</param>
/// <param name="training">The training data to use. Must be indexable.</param>
public SVDTraining(BasicNetwork network, INeuralDataSet training)
{
ILayer outputLayer = network.GetLayer(BasicNetwork.TAG_OUTPUT);
if (outputLayer == null)
{
throw new TrainingError("SVD requires an output layer.");
}
if (outputLayer.NeuronCount != 1)
{
throw new TrainingError("SVD requires an output layer with a single neuron.");
}
if (network.GetLayer(RadialBasisPattern.RBF_LAYER) == null)
throw new TrainingError("SVD is only tested to work on radial basis function networks.");
rbfLayer = (RadialBasisFunctionLayer)network.GetLayer(RadialBasisPattern.RBF_LAYER);
this.Training = training;
this.network = network;
this.trainingLength = (int)this.Training.InputSize;
BasicNeuralData input = new BasicNeuralData(this.Training.InputSize);
BasicNeuralData ideal = new BasicNeuralData(this.Training.IdealSize);
this.pair = new BasicNeuralDataPair(input, ideal);
}
/// <summary>
/// Construct the depth calculation object.
/// </summary>
/// <param name="network">The network that we are calculating for.</param>
public CalculateDepth(BasicNetwork network)
{
this.network = network;
this.outputLayer = network.GetLayer(BasicNetwork.TAG_OUTPUT);
if( this.outputLayer!=null )
Calculate(0, this.outputLayer);
}
/// <summary>
/// Construct the object and find the parts of the network.
/// </summary>
/// <param name="network">The network to train.</param>
public FindCPN(BasicNetwork network)
{
if (network.Structure.Layers.Count != 3)
{
String str = "A CPN network must have exactly 3 layers";
#if logging
if (logger.IsErrorEnabled)
{
logger.Error(str);
}
#endif
throw new TrainingError(str);
}
this.inputLayer = network.GetLayer(BasicNetwork.TAG_INPUT);
this.outstarLayer = network.GetLayer(CPNPattern.TAG_OUTSTAR);
this.instarLayer = network.GetLayer(CPNPattern.TAG_INSTAR);
if (this.outstarLayer == null)
{
String str = "Can't find an OUTSTAR layer, this is required.";
#if logging
if (logger.IsErrorEnabled)
{
logger.Error(str);
}
#endif
throw new TrainingError(str);
}
if (this.instarLayer == null)
{
String str = "Can't find an OUTSTAR layer, this is required.";
#if logging
if (logger.IsErrorEnabled)
{
logger.Error(str);
}
#endif
throw new TrainingError(str);
}
this.instarSynapse = this.inputLayer.Next[0];
this.outstarSynapse = this.instarLayer.Next[0];
}
/// <summary>
/// Construct the depth calculation object.
/// </summary>
/// <param name="network">The network that we are calculating for.</param>
public CalculateDepth(BasicNetwork network)
{
this.network = network;
this.outputLayer = network.GetLayer(BasicNetwork.TAG_OUTPUT);
if (this.outputLayer != null)
{
Calculate(0, this.outputLayer);
}
}
/// <summary>
/// The Nguyen-Widrow initialization algorithm is the following :
///
/// 1. Initialize all weight of hidden layers with (ranged) random values
/// 2. For each hidden layer
/// 2.1 calculate beta value, 0.7 * Nth(#neurons of input layer) root of
/// #neurons of current layer
/// 2.2 for each synapse
/// 2.1.1 for each weight
/// 2.1.2 Adjust weight by dividing by norm of weight for neuron and
/// multiplying by beta value
/// </summary>
/// <param name="network">The network to randomize.</param>
public override void Randomize(BasicNetwork network)
{
base.Randomize(network);
int neuronCount = 0;
foreach (ILayer layer in network.Structure.Layers)
{
neuronCount += layer.NeuronCount;
}
ILayer inputLayer = network.GetLayer(BasicNetwork.TAG_INPUT);
ILayer outputLayer = network.GetLayer(BasicNetwork.TAG_OUTPUT);
if (inputLayer == null)
{
throw new EncogError("Must have an input layer for Nguyen-Widrow.");
}
if (outputLayer == null)
{
throw new EncogError("Must have an output layer for Nguyen-Widrow.");
}
int hiddenNeurons = neuronCount - inputLayer.NeuronCount
- outputLayer.NeuronCount;
if (hiddenNeurons < 1)
{
throw new EncogError("Must have hidden neurons for Nguyen-Widrow.");
}
double beta = 0.7 * Math.Pow(hiddenNeurons, 1.0 / inputLayer
.NeuronCount);
foreach (ISynapse synapse in network.Structure.Synapses)
{
Randomize(beta, synapse);
}
network.Structure.FlatUpdate = FlatUpdateNeeded.Flatten;
network.Structure.FlattenWeights();
}
/// <summary>
/// Construct a NEAT training class.
/// </summary>
/// <param name="score">How to score the networks.</param>
/// <param name="network">The network to base this on.</param>
/// <param name="population">The population to use.</param>
public NEATTraining(ICalculateScore score, BasicNetwork network,
IPopulation population)
{
ILayer inputLayer = network.GetLayer(BasicNetwork.TAG_INPUT);
ILayer outputLayer = network.GetLayer(BasicNetwork.TAG_OUTPUT);
this.CalculateScore = new GeneticScoreAdapter(score);
this.Comparator = new GenomeComparator(CalculateScore);
this.inputCount = inputLayer.NeuronCount;
this.outputCount = outputLayer.NeuronCount;
this.Population = population;
foreach (IGenome obj in population.Genomes)
{
NEATGenome neat = (NEATGenome)obj;
neat.GA = this;
}
Init();
}
/// <summary>
/// Perform a training iteration.
/// </summary>
public override void Iteration()
{
ErrorCalculation errorCalculation = new ErrorCalculation();
ILayer inputLayer = network.GetLayer(BasicNetwork.TAG_INPUT);
ILayer outputLayer = network.GetLayer(BasicNetwork.TAG_OUTPUT);
foreach (INeuralDataPair pair in this.training)
{
// calculate the error
INeuralData output = this.network.Compute(pair.Input);
for (int currentAdaline = 0; currentAdaline < output.Count; currentAdaline++)
{
double diff = pair.Ideal[currentAdaline]
- output[currentAdaline];
// weights
for (int i = 0; i < inputLayer
.NeuronCount; i++)
{
double input = pair.Input[i];
synapse.WeightMatrix.Add(i, currentAdaline,
learningRate * diff * input);
}
// bias
double t = outputLayer.BiasWeights[
currentAdaline];
t += learningRate * diff;
outputLayer.BiasWeights[currentAdaline] = t;
}
errorCalculation.UpdateError(output.Data, pair.Ideal.Data);
}
// set the global error
this.Error = errorCalculation.Calculate();
}
/// <summary>
/// Construct he ADALINE trainer.
/// </summary>
/// <param name="network">The network to train.</param>
/// <param name="training">The training set.</param>
/// <param name="learningRate">The learning rate.</param>
public TrainAdaline(BasicNetwork network, INeuralDataSet training,
double learningRate)
{
if (network.Structure.Layers.Count > 2)
throw new NeuralNetworkError(
"An ADALINE network only has two layers.");
this.network = network;
ILayer input = network.GetLayer(BasicNetwork.TAG_INPUT);
this.synapse = input.Next[0];
this.training = training;
this.learningRate = learningRate;
}
/// <summary>
/// Construct he ADALINE trainer.
/// </summary>
/// <param name="network">The network to train.</param>
/// <param name="training">The training set.</param>
/// <param name="learningRate">The learning rate.</param>
public TrainAdaline(BasicNetwork network, INeuralDataSet training,
double learningRate)
{
if (network.Structure.Layers.Count > 2)
{
throw new NeuralNetworkError(
"An ADALINE network only has two layers.");
}
this.network = network;
ILayer input = network.GetLayer(BasicNetwork.TAG_INPUT);
this.synapse = input.Next[0];
this.training = training;
this.learningRate = learningRate;
}
/// <summary>
/// Construct the LMA object.
/// </summary>
/// <param name="network">The network to train. Must have a single output neuron.</param>
/// <param name="training">The training data to use. Must be indexable.</param>
public LevenbergMarquardtTraining(BasicNetwork network,
INeuralDataSet training)
{
if (!(training is IIndexable))
{
throw new TrainingError(
"Levenberg Marquardt requires an indexable training set.");
}
ILayer outputLayer = network.GetLayer(BasicNetwork.TAG_OUTPUT);
if (outputLayer == null)
{
throw new TrainingError(
"Levenberg Marquardt requires an output layer.");
}
if (outputLayer.NeuronCount != 1)
{
throw new TrainingError(
"Levenberg Marquardt requires an output layer with a single neuron.");
}
this.Training = training;
this.indexableTraining = (IIndexable)Training;
this.network = network;
this.trainingLength = (int)this.indexableTraining.Count;
this.parametersLength = this.network.Structure.CalculateSize();
this.hessianMatrix = new Matrix(this.parametersLength,
this.parametersLength);
this.hessian = this.hessianMatrix.Data;
this.alpha = 0.0;
this.beta = 1.0;
this.lambda = 0.1;
this.deltas = new double[this.parametersLength];
this.gradient = new double[this.parametersLength];
this.diagonal = new double[this.parametersLength];
BasicNeuralData input = new BasicNeuralData(
this.indexableTraining.InputSize);
BasicNeuralData ideal = new BasicNeuralData(
this.indexableTraining.IdealSize);
this.pair = new BasicNeuralDataPair(input, ideal);
}
/// <summary>
/// Format the network as a human readable string that lists the
/// hidden layers.
/// </summary>
/// <param name="network">The network to format.</param>
/// <returns>A human readable string.</returns>
public static String NetworkToString(BasicNetwork network)
{
StringBuilder result = new StringBuilder();
int num = 1;
ILayer layer = network.GetLayer(BasicNetwork.TAG_INPUT);
// display only hidden layers
while (layer.Next.Count > 0)
{
layer = layer.Next[0].ToLayer;
if (result.Length > 0)
{
result.Append(",");
}
result.Append("H");
result.Append(num++);
result.Append("=");
result.Append(layer.NeuronCount);
}
return(result.ToString());
}
public void Execute(IExampleInterface app)
{
//Specify the number of dimensions and the number of neurons per dimension
int dimensions = 2;
int numNeuronsPerDimension = 7;
//Set the standard RBF neuron width.
//Literature seems to suggest this is a good default value.
double volumeNeuronWidth = 2.0 / numNeuronsPerDimension;
//RBF can struggle when it comes to flats at the edge of the sample space.
//We have added the ability to include wider neurons on the sample space boundary which greatly
//improves fitting to flats
bool includeEdgeRBFs = true;
#region Setup
//General setup is the same as before
RadialBasisPattern pattern = new RadialBasisPattern();
pattern.InputNeurons = dimensions;
pattern.OutputNeurons = 1;
//Total number of neurons required.
//Total number of Edges is calculated possibly for future use but not used any further here
int numNeurons = (int)Math.Pow(numNeuronsPerDimension, dimensions);
int numEdges = (int)(dimensions * Math.Pow(2, dimensions - 1));
pattern.AddHiddenLayer(numNeurons);
BasicNetwork network = pattern.Generate();
RadialBasisFunctionLayer rbfLayer = (RadialBasisFunctionLayer)network.GetLayer(RadialBasisPattern.RBF_LAYER);
network.Reset();
//Position the multidimensional RBF neurons, with equal spacing, within the provided sample space from 0 to 1.
rbfLayer.SetRBFCentersAndWidthsEqualSpacing(0, 1, RBFEnum.Gaussian, dimensions, volumeNeuronWidth, includeEdgeRBFs);
#endregion
//Create some training data that can not easily be represented by gaussians
//There are other training examples for both 1D and 2D
//Degenerate training data only provides outputs as 1 or 0 (averaging over all outputs for a given set of inputs would produce something approaching the smooth training data).
//Smooth training data provides true values for the provided input dimensions.
Create2DSmoothTainingDataGit();
//Create the training set and train.
INeuralDataSet trainingSet = new BasicNeuralDataSet(INPUT, IDEAL);
ITrain train = new SVDTraining(network, trainingSet);
//SVD is a single step solve
int epoch = 1;
do
{
train.Iteration();
Console.WriteLine("Epoch #" + epoch + " Error:" + train.Error);
epoch++;
} while ((epoch < 1) && (train.Error > 0.001));
// test the neural network
Console.WriteLine("Neural Network Results:");
//Create a testing array which may be to a higher resoltion than the original training data
Set2DTestingArrays(100);
trainingSet = new BasicNeuralDataSet(INPUT, IDEAL);
//Write out the results data
using (var sw = new System.IO.StreamWriter("results.csv", false))
{
foreach (INeuralDataPair pair in trainingSet)
{
INeuralData output = network.Compute(pair.Input);
//1D//sw.WriteLine(InverseScale(pair.Input[0]) + ", " + Chop(InverseScale(output[0])));// + ", " + pair.Ideal[0]);
sw.WriteLine(InverseScale(pair.Input[0]) + ", " + InverseScale(pair.Input[1]) + ", " + Chop(InverseScale(output[0])));// + ", " + pair.Ideal[0]);// + ",ideal=" + pair.Ideal[0]);
//3D//sw.WriteLine(InverseScale(pair.Input[0]) + ", " + InverseScale(pair.Input[1]) + ", " + InverseScale(pair.Input[2]) + ", " + Chop(InverseScale(output[0])));// + ", " + pair.Ideal[0]);// + ",ideal=" + pair.Ideal[0]);
//Console.WriteLine(pair.Input[0] + ", actual=" + output[0] + ",ideal=" + pair.Ideal[0]);
}
}
Console.WriteLine("\nFit output saved to results.csv");
Console.WriteLine("\nComplete - Please press the 'any' key to close.");
Console.ReadKey();
}
/// <summary>
/// Construct a NEAT training class.
/// </summary>
/// <param name="score">How to score the networks.</param>
/// <param name="network">The network to base this on.</param>
/// <param name="population">The population to use.</param>
public NEATTraining(ICalculateScore score, BasicNetwork network,
IPopulation population)
{
ILayer inputLayer = network.GetLayer(BasicNetwork.TAG_INPUT);
ILayer outputLayer = network.GetLayer(BasicNetwork.TAG_OUTPUT);
this.CalculateScore = new GeneticScoreAdapter(score);
this.Comparator = new GenomeComparator(CalculateScore);
this.inputCount = inputLayer.NeuronCount;
this.outputCount = outputLayer.NeuronCount;
this.Population = population;
foreach (IGenome obj in population.Genomes)
{
NEATGenome neat = (NEATGenome)obj;
neat.GA = this;
}
Init();
}
/// <summary>
/// Create an instance of competitive training.
/// </summary>
/// <param name="network">The network to train.</param>
/// <param name="learningRate">The learning rate, how much to apply per iteration.</param>
/// <param name="training">The training set (unsupervised).</param>
/// <param name="neighborhood">The neighborhood function to use.</param>
public CompetitiveTraining(BasicNetwork network,
double learningRate, INeuralDataSet training,
INeighborhoodFunction neighborhood)
{
this.neighborhood = neighborhood;
Training = training;
this.LearningRate = learningRate;
this.network = network;
this.inputLayer = network.GetLayer(BasicNetwork.TAG_INPUT);
this.outputLayer = network.GetLayer(BasicNetwork.TAG_OUTPUT);
this.synapses = network.Structure.GetPreviousSynapses(
this.outputLayer);
this.inputNeuronCount = this.inputLayer.NeuronCount;
this.outputNeuronCount = this.outputLayer.NeuronCount;
this.ForceWinner = false;
Error = 0;
// setup the correction matrix
foreach (ISynapse synapse in this.synapses)
{
Matrix matrix = new Matrix(synapse.WeightMatrix.Rows,
synapse.WeightMatrix.Cols);
this.correctionMatrix[synapse] = matrix;
}
// create the BMU class
this.bmuUtil = new BestMatchingUnit(this);
}
/// <summary>
/// Construct the object and find the parts of the network.
/// </summary>
/// <param name="network">The network to train.</param>
public FindCPN(BasicNetwork network)
{
if (network.Structure.Layers.Count != 3)
{
String str = "A CPN network must have exactly 3 layers";
#if logging
if (logger.IsErrorEnabled)
{
logger.Error(str);
}
#endif
throw new TrainingError(str);
}
this.inputLayer = network.GetLayer(BasicNetwork.TAG_INPUT);
this.outstarLayer = network.GetLayer(CPNPattern.TAG_OUTSTAR);
this.instarLayer = network.GetLayer(CPNPattern.TAG_INSTAR);
if (this.outstarLayer == null)
{
String str = "Can't find an OUTSTAR layer, this is required.";
#if logging
if (logger.IsErrorEnabled)
{
logger.Error(str);
}
#endif
throw new TrainingError(str);
}
if (this.instarLayer == null)
{
String str = "Can't find an OUTSTAR layer, this is required.";
#if logging
if (logger.IsErrorEnabled)
{
logger.Error(str);
}
#endif
throw new TrainingError(str);
}
this.instarSynapse = this.inputLayer.Next[0];
this.outstarSynapse = this.instarLayer.Next[0];
}
/// <summary>
/// Setup the network logic, read parameters from the network.
/// </summary>
/// <param name="network">The network that this logic class belongs to.</param>
public void Init(BasicNetwork network)
{
this.network = network;
this.f1Layer = network.GetLayer(BAMPattern.TAG_F1);
this.f2Layer = network.GetLayer(BAMPattern.TAG_F2);
this.synapseF1ToF2 = network.Structure.FindSynapse(this.f1Layer, this.f2Layer, true);
this.synapseF2ToF1 = network.Structure.FindSynapse(this.f2Layer, this.f1Layer, true);
}
