/// <summary> /// Create an instance of competitive training. /// </summary> /// <param name="network">The network to train.</param> /// <param name="learningRate">The learning rate, how much to apply per iteration.</param> /// <param name="training">The training set (unsupervised).</param> /// <param name="neighborhood">The neighborhood function to use.</param> public CompetitiveTraining(BasicNetwork network, double learningRate, INeuralDataSet training, INeighborhoodFunction neighborhood) { this.neighborhood = neighborhood; Training = training; this.LearningRate = learningRate; this.network = network; this.inputLayer = network.GetLayer(BasicNetwork.TAG_INPUT); this.outputLayer = network.GetLayer(BasicNetwork.TAG_OUTPUT); this.synapses = network.Structure.GetPreviousSynapses( this.outputLayer); this.inputNeuronCount = this.inputLayer.NeuronCount; this.outputNeuronCount = this.outputLayer.NeuronCount; this.ForceWinner = false; Error = 0; // setup the correction matrix foreach (ISynapse synapse in this.synapses) { Matrix matrix = new Matrix(synapse.WeightMatrix.Rows, synapse.WeightMatrix.Cols); this.correctionMatrix[synapse] = matrix; } // create the BMU class this.bmuUtil = new BestMatchingUnit(this); }
/// <summary> /// Validate that the neural network would work with the specified training set. /// </summary> /// <param name="network">The neural network that is to be evaluated.</param> /// <param name="training">The training set that we should evaluate </param> public static void ValidateNetworkForTraining(BasicNetwork network, INeuralDataSet training) { ILayer inputLayer = network.GetLayer(BasicNetwork.TAG_INPUT); ILayer outputLayer = network.GetLayer(BasicNetwork.TAG_OUTPUT); if (inputLayer == null) { throw new NeuralNetworkError("This operation requires that the neural network have an input layer."); } if (outputLayer == null) { throw new NeuralNetworkError("This operation requires that the neural network have an output layer."); } if (inputLayer.NeuronCount != training.InputSize) { throw new NeuralNetworkError("The input layer size of " + inputLayer.NeuronCount + " must match the training input size of " + training.InputSize + "."); } if (training.IdealSize > 0 && outputLayer.NeuronCount != training.IdealSize) { throw new NeuralNetworkError("The output layer size of " + inputLayer.NeuronCount + " must match the training input size of " + training.IdealSize + "."); } }
/// <summary> /// Construct the LMA object. /// </summary> /// <param name="network">The network to train. Must have a single output neuron.</param> /// <param name="training">The training data to use. Must be indexable.</param> public SVDTraining(BasicNetwork network, INeuralDataSet training) { ILayer outputLayer = network.GetLayer(BasicNetwork.TAG_OUTPUT); if (outputLayer == null) { throw new TrainingError("SVD requires an output layer."); } if (outputLayer.NeuronCount != 1) { throw new TrainingError("SVD requires an output layer with a single neuron."); } if (network.GetLayer(RadialBasisPattern.RBF_LAYER) == null) { throw new TrainingError("SVD is only tested to work on radial basis function networks."); } rbfLayer = (RadialBasisFunctionLayer)network.GetLayer(RadialBasisPattern.RBF_LAYER); this.Training = training; this.network = network; this.trainingLength = (int)this.Training.InputSize; BasicNeuralData input = new BasicNeuralData(this.Training.InputSize); BasicNeuralData ideal = new BasicNeuralData(this.Training.IdealSize); this.pair = new BasicNeuralDataPair(input, ideal); }
/// <summary> /// Setup the network logic, read parameters from the network. /// </summary> /// <param name="network">The network that this logic class belongs to.</param> public void Init(BasicNetwork network) { this.network = network; this.f1Layer = network.GetLayer(BAMPattern.TAG_F1); this.f2Layer = network.GetLayer(BAMPattern.TAG_F2); this.synapseF1ToF2 = network.Structure.FindSynapse(this.f1Layer, this.f2Layer, true); this.synapseF2ToF1 = network.Structure.FindSynapse(this.f2Layer, this.f1Layer, true); }
/// <summary> /// Determine if the specified neural network can be flat. If it can a null /// is returned, otherwise, an error is returned to show why the network /// cannot be flattened. /// </summary> /// <param name="eml">The network to check.</param> /// <returns>Null, if the net can not be flattened, an error message /// otherwise.</returns> public override String IsValid(IEngineMachineLearning eml) { if (!(eml is BasicNetwork)) { return("Only a BasicNetwork can be converted to a flat network."); } BasicNetwork network = (BasicNetwork)eml; ILayer inputLayer = network.GetLayer(BasicNetwork.TAG_INPUT); ILayer outputLayer = network.GetLayer(BasicNetwork.TAG_OUTPUT); if (inputLayer == null) { return("To convert to a flat network, there must be an input layer."); } if (outputLayer == null) { return("To convert to a flat network, there must be an output layer."); } if (!(network.Logic is FeedforwardLogic) || (network.Logic is ThermalLogic)) { return("To convert to flat, must be using FeedforwardLogic or SimpleRecurrentLogic."); } foreach (ILayer layer in network.Structure.Layers) { if (layer.Next.Count > 2) { return("To convert to flat a network must have at most two outbound synapses."); } if (layer.GetType() != typeof(ContextLayer) && layer.GetType() != typeof(BasicLayer) && layer.GetType() != typeof(RadialBasisFunctionLayer)) { return("To convert to flat a network must have only BasicLayer and ContextLayer layers."); } } foreach (ISynapse synapse in network.Structure.Synapses) { if (synapse is NEATSynapse) { return("A NEAT synapse cannot be flattened."); } } return(null); }
/// <summary> /// Format the network as a human readable string that lists the /// hidden layers. /// </summary> /// <param name="network">The network to format.</param> /// <returns>A human readable string.</returns> public static String NetworkToString(BasicNetwork network) { StringBuilder result = new StringBuilder(); int num = 1; ILayer layer = network.GetLayer(BasicNetwork.TAG_INPUT); // display only hidden layers while (layer.Next.Count > 0) { layer = layer.Next[0].ToLayer; if (result.Length > 0) { result.Append(","); } result.Append("H"); result.Append(num++); result.Append("="); result.Append(layer.NeuronCount); } return result.ToString(); }
/// <summary> /// Construct the LMA object. /// </summary> /// <param name="network">The network to train. Must have a single output neuron.</param> /// <param name="training">The training data to use. Must be indexable.</param> public SVDTraining(BasicNetwork network, INeuralDataSet training) { ILayer outputLayer = network.GetLayer(BasicNetwork.TAG_OUTPUT); if (outputLayer == null) { throw new TrainingError("SVD requires an output layer."); } if (outputLayer.NeuronCount != 1) { throw new TrainingError("SVD requires an output layer with a single neuron."); } if (network.GetLayer(RadialBasisPattern.RBF_LAYER) == null) throw new TrainingError("SVD is only tested to work on radial basis function networks."); rbfLayer = (RadialBasisFunctionLayer)network.GetLayer(RadialBasisPattern.RBF_LAYER); this.Training = training; this.network = network; this.trainingLength = (int)this.Training.InputSize; BasicNeuralData input = new BasicNeuralData(this.Training.InputSize); BasicNeuralData ideal = new BasicNeuralData(this.Training.IdealSize); this.pair = new BasicNeuralDataPair(input, ideal); }
/// <summary> /// Construct the depth calculation object. /// </summary> /// <param name="network">The network that we are calculating for.</param> public CalculateDepth(BasicNetwork network) { this.network = network; this.outputLayer = network.GetLayer(BasicNetwork.TAG_OUTPUT); if( this.outputLayer!=null ) Calculate(0, this.outputLayer); }
/// <summary> /// Construct the object and find the parts of the network. /// </summary> /// <param name="network">The network to train.</param> public FindCPN(BasicNetwork network) { if (network.Structure.Layers.Count != 3) { String str = "A CPN network must have exactly 3 layers"; #if logging if (logger.IsErrorEnabled) { logger.Error(str); } #endif throw new TrainingError(str); } this.inputLayer = network.GetLayer(BasicNetwork.TAG_INPUT); this.outstarLayer = network.GetLayer(CPNPattern.TAG_OUTSTAR); this.instarLayer = network.GetLayer(CPNPattern.TAG_INSTAR); if (this.outstarLayer == null) { String str = "Can't find an OUTSTAR layer, this is required."; #if logging if (logger.IsErrorEnabled) { logger.Error(str); } #endif throw new TrainingError(str); } if (this.instarLayer == null) { String str = "Can't find an OUTSTAR layer, this is required."; #if logging if (logger.IsErrorEnabled) { logger.Error(str); } #endif throw new TrainingError(str); } this.instarSynapse = this.inputLayer.Next[0]; this.outstarSynapse = this.instarLayer.Next[0]; }
/// <summary> /// Construct the depth calculation object. /// </summary> /// <param name="network">The network that we are calculating for.</param> public CalculateDepth(BasicNetwork network) { this.network = network; this.outputLayer = network.GetLayer(BasicNetwork.TAG_OUTPUT); if (this.outputLayer != null) { Calculate(0, this.outputLayer); } }
/// <summary> /// The Nguyen-Widrow initialization algorithm is the following : /// /// 1. Initialize all weight of hidden layers with (ranged) random values /// 2. For each hidden layer /// 2.1 calculate beta value, 0.7 * Nth(#neurons of input layer) root of /// #neurons of current layer /// 2.2 for each synapse /// 2.1.1 for each weight /// 2.1.2 Adjust weight by dividing by norm of weight for neuron and /// multiplying by beta value /// </summary> /// <param name="network">The network to randomize.</param> public override void Randomize(BasicNetwork network) { base.Randomize(network); int neuronCount = 0; foreach (ILayer layer in network.Structure.Layers) { neuronCount += layer.NeuronCount; } ILayer inputLayer = network.GetLayer(BasicNetwork.TAG_INPUT); ILayer outputLayer = network.GetLayer(BasicNetwork.TAG_OUTPUT); if (inputLayer == null) { throw new EncogError("Must have an input layer for Nguyen-Widrow."); } if (outputLayer == null) { throw new EncogError("Must have an output layer for Nguyen-Widrow."); } int hiddenNeurons = neuronCount - inputLayer.NeuronCount - outputLayer.NeuronCount; if (hiddenNeurons < 1) { throw new EncogError("Must have hidden neurons for Nguyen-Widrow."); } double beta = 0.7 * Math.Pow(hiddenNeurons, 1.0 / inputLayer .NeuronCount); foreach (ISynapse synapse in network.Structure.Synapses) { Randomize(beta, synapse); } network.Structure.FlatUpdate = FlatUpdateNeeded.Flatten; network.Structure.FlattenWeights(); }
/// <summary> /// Construct a NEAT training class. /// </summary> /// <param name="score">How to score the networks.</param> /// <param name="network">The network to base this on.</param> /// <param name="population">The population to use.</param> public NEATTraining(ICalculateScore score, BasicNetwork network, IPopulation population) { ILayer inputLayer = network.GetLayer(BasicNetwork.TAG_INPUT); ILayer outputLayer = network.GetLayer(BasicNetwork.TAG_OUTPUT); this.CalculateScore = new GeneticScoreAdapter(score); this.Comparator = new GenomeComparator(CalculateScore); this.inputCount = inputLayer.NeuronCount; this.outputCount = outputLayer.NeuronCount; this.Population = population; foreach (IGenome obj in population.Genomes) { NEATGenome neat = (NEATGenome)obj; neat.GA = this; } Init(); }
/// <summary> /// Perform a training iteration. /// </summary> public override void Iteration() { ErrorCalculation errorCalculation = new ErrorCalculation(); ILayer inputLayer = network.GetLayer(BasicNetwork.TAG_INPUT); ILayer outputLayer = network.GetLayer(BasicNetwork.TAG_OUTPUT); foreach (INeuralDataPair pair in this.training) { // calculate the error INeuralData output = this.network.Compute(pair.Input); for (int currentAdaline = 0; currentAdaline < output.Count; currentAdaline++) { double diff = pair.Ideal[currentAdaline] - output[currentAdaline]; // weights for (int i = 0; i < inputLayer .NeuronCount; i++) { double input = pair.Input[i]; synapse.WeightMatrix.Add(i, currentAdaline, learningRate * diff * input); } // bias double t = outputLayer.BiasWeights[ currentAdaline]; t += learningRate * diff; outputLayer.BiasWeights[currentAdaline] = t; } errorCalculation.UpdateError(output.Data, pair.Ideal.Data); } // set the global error this.Error = errorCalculation.Calculate(); }
/// <summary> /// Construct he ADALINE trainer. /// </summary> /// <param name="network">The network to train.</param> /// <param name="training">The training set.</param> /// <param name="learningRate">The learning rate.</param> public TrainAdaline(BasicNetwork network, INeuralDataSet training, double learningRate) { if (network.Structure.Layers.Count > 2) throw new NeuralNetworkError( "An ADALINE network only has two layers."); this.network = network; ILayer input = network.GetLayer(BasicNetwork.TAG_INPUT); this.synapse = input.Next[0]; this.training = training; this.learningRate = learningRate; }
/// <summary> /// Construct he ADALINE trainer. /// </summary> /// <param name="network">The network to train.</param> /// <param name="training">The training set.</param> /// <param name="learningRate">The learning rate.</param> public TrainAdaline(BasicNetwork network, INeuralDataSet training, double learningRate) { if (network.Structure.Layers.Count > 2) { throw new NeuralNetworkError( "An ADALINE network only has two layers."); } this.network = network; ILayer input = network.GetLayer(BasicNetwork.TAG_INPUT); this.synapse = input.Next[0]; this.training = training; this.learningRate = learningRate; }
/// <summary> /// Construct the LMA object. /// </summary> /// <param name="network">The network to train. Must have a single output neuron.</param> /// <param name="training">The training data to use. Must be indexable.</param> public LevenbergMarquardtTraining(BasicNetwork network, INeuralDataSet training) { if (!(training is IIndexable)) { throw new TrainingError( "Levenberg Marquardt requires an indexable training set."); } ILayer outputLayer = network.GetLayer(BasicNetwork.TAG_OUTPUT); if (outputLayer == null) { throw new TrainingError( "Levenberg Marquardt requires an output layer."); } if (outputLayer.NeuronCount != 1) { throw new TrainingError( "Levenberg Marquardt requires an output layer with a single neuron."); } this.Training = training; this.indexableTraining = (IIndexable)Training; this.network = network; this.trainingLength = (int)this.indexableTraining.Count; this.parametersLength = this.network.Structure.CalculateSize(); this.hessianMatrix = new Matrix(this.parametersLength, this.parametersLength); this.hessian = this.hessianMatrix.Data; this.alpha = 0.0; this.beta = 1.0; this.lambda = 0.1; this.deltas = new double[this.parametersLength]; this.gradient = new double[this.parametersLength]; this.diagonal = new double[this.parametersLength]; BasicNeuralData input = new BasicNeuralData( this.indexableTraining.InputSize); BasicNeuralData ideal = new BasicNeuralData( this.indexableTraining.IdealSize); this.pair = new BasicNeuralDataPair(input, ideal); }
/// <summary> /// Format the network as a human readable string that lists the /// hidden layers. /// </summary> /// <param name="network">The network to format.</param> /// <returns>A human readable string.</returns> public static String NetworkToString(BasicNetwork network) { StringBuilder result = new StringBuilder(); int num = 1; ILayer layer = network.GetLayer(BasicNetwork.TAG_INPUT); // display only hidden layers while (layer.Next.Count > 0) { layer = layer.Next[0].ToLayer; if (result.Length > 0) { result.Append(","); } result.Append("H"); result.Append(num++); result.Append("="); result.Append(layer.NeuronCount); } return(result.ToString()); }
public void Execute(IExampleInterface app) { //Specify the number of dimensions and the number of neurons per dimension int dimensions = 2; int numNeuronsPerDimension = 7; //Set the standard RBF neuron width. //Literature seems to suggest this is a good default value. double volumeNeuronWidth = 2.0 / numNeuronsPerDimension; //RBF can struggle when it comes to flats at the edge of the sample space. //We have added the ability to include wider neurons on the sample space boundary which greatly //improves fitting to flats bool includeEdgeRBFs = true; #region Setup //General setup is the same as before RadialBasisPattern pattern = new RadialBasisPattern(); pattern.InputNeurons = dimensions; pattern.OutputNeurons = 1; //Total number of neurons required. //Total number of Edges is calculated possibly for future use but not used any further here int numNeurons = (int)Math.Pow(numNeuronsPerDimension, dimensions); int numEdges = (int)(dimensions * Math.Pow(2, dimensions - 1)); pattern.AddHiddenLayer(numNeurons); BasicNetwork network = pattern.Generate(); RadialBasisFunctionLayer rbfLayer = (RadialBasisFunctionLayer)network.GetLayer(RadialBasisPattern.RBF_LAYER); network.Reset(); //Position the multidimensional RBF neurons, with equal spacing, within the provided sample space from 0 to 1. rbfLayer.SetRBFCentersAndWidthsEqualSpacing(0, 1, RBFEnum.Gaussian, dimensions, volumeNeuronWidth, includeEdgeRBFs); #endregion //Create some training data that can not easily be represented by gaussians //There are other training examples for both 1D and 2D //Degenerate training data only provides outputs as 1 or 0 (averaging over all outputs for a given set of inputs would produce something approaching the smooth training data). //Smooth training data provides true values for the provided input dimensions. Create2DSmoothTainingDataGit(); //Create the training set and train. INeuralDataSet trainingSet = new BasicNeuralDataSet(INPUT, IDEAL); ITrain train = new SVDTraining(network, trainingSet); //SVD is a single step solve int epoch = 1; do { train.Iteration(); Console.WriteLine("Epoch #" + epoch + " Error:" + train.Error); epoch++; } while ((epoch < 1) && (train.Error > 0.001)); // test the neural network Console.WriteLine("Neural Network Results:"); //Create a testing array which may be to a higher resoltion than the original training data Set2DTestingArrays(100); trainingSet = new BasicNeuralDataSet(INPUT, IDEAL); //Write out the results data using (var sw = new System.IO.StreamWriter("results.csv", false)) { foreach (INeuralDataPair pair in trainingSet) { INeuralData output = network.Compute(pair.Input); //1D//sw.WriteLine(InverseScale(pair.Input[0]) + ", " + Chop(InverseScale(output[0])));// + ", " + pair.Ideal[0]); sw.WriteLine(InverseScale(pair.Input[0]) + ", " + InverseScale(pair.Input[1]) + ", " + Chop(InverseScale(output[0])));// + ", " + pair.Ideal[0]);// + ",ideal=" + pair.Ideal[0]); //3D//sw.WriteLine(InverseScale(pair.Input[0]) + ", " + InverseScale(pair.Input[1]) + ", " + InverseScale(pair.Input[2]) + ", " + Chop(InverseScale(output[0])));// + ", " + pair.Ideal[0]);// + ",ideal=" + pair.Ideal[0]); //Console.WriteLine(pair.Input[0] + ", actual=" + output[0] + ",ideal=" + pair.Ideal[0]); } } Console.WriteLine("\nFit output saved to results.csv"); Console.WriteLine("\nComplete - Please press the 'any' key to close."); Console.ReadKey(); }