/// <summary> /// Construct a trainer for flat networks to use the Manhattan update rule. /// </summary> /// /// <param name="network">The network to train.</param> /// <param name="training">The training data to use.</param> /// <param name="learningRate">The learning rate to use.</param> public TrainFlatNetworkManhattan(FlatNetwork network, IEngineDataSet training, double learningRate) : base(network, training) { this.learningRate = learningRate; this.zeroTolerance = RPROPConst.DEFAULT_ZERO_TOLERANCE; }
/// <summary> /// Construct a gradient worker. /// </summary> /// /// <param name="network">The network to train.</param> /// <param name="owner">The owner that is doing the training.</param> /// <param name="training">The training data.</param> /// <param name="low">The low index to use in the training data.</param> /// <param name="high">The high index to use in the training data.</param> public GradientWorkerCPU(FlatNetwork network, TrainFlatNetworkProp owner, IEngineIndexableSet training, int low, int high) { this.errorCalculation = new ErrorCalculation(); this.network = network; this.training = training; this.low = low; this.high = high; this.owner = owner; this.stopwatch = new Stopwatch(); this.layerDelta = new double[network.LayerOutput.Length]; this.gradients = new double[network.Weights.Length]; this.actual = new double[network.OutputCount]; this.weights = network.Weights; this.layerIndex = network.LayerIndex; this.layerCounts = network.LayerCounts; this.weightIndex = network.WeightIndex; this.layerOutput = network.LayerOutput; this.layerFeedCounts = network.LayerFeedCounts; this.pair = BasicEngineData.CreatePair(network.InputCount, network.OutputCount); }
/// <summary> /// Create an array of activations based on a flat network. /// </summary> /// <param name="flat">The flat network.</param> /// <returns></returns> public int[] CreateActivations(FlatNetwork flat) { var result = new int[flat.ActivationFunctions.Length]; for (int i = 0; i < flat.ActivationFunctions.Length; i++) { IActivationFunction af = flat.ActivationFunctions[i]; if (af is ActivationLinear) { result[i] = 0; } else if (af is ActivationTANH) { result[i] = 1; } if (af is ActivationSigmoid) { result[i] = 2; } if (af is ActivationElliottSymmetric) { result[i] = 3; } if (af is ActivationElliott) { result[i] = 4; } } return(result); }
/// <summary> /// Construct the training object. /// </summary> /// /// <param name="network">The network to train.</param> /// <param name="training">The training data to use.</param> public TrainFlatNetworkSCG(FlatNetwork network, IEngineDataSet training) : base(network, training) { this.success = true; this.success = true; this.delta = 0; this.lambda2 = 0; this.lambda = TrainFlatNetworkSCG.FIRST_LAMBDA; this.oldError = 0; this.magP = 0; this.restart = false; this.weights = EngineArray.ArrayCopy(network.Weights); int numWeights = this.weights.Length; // this.gradients = new double[numWeights]; this.oldWeights = new double[numWeights]; this.oldGradient = new double[numWeights]; this.p = new double[numWeights]; this.r = new double[numWeights]; this.shouldInit = true; }
/// <summary> /// Creat new index values for the network. /// </summary> /// private void ReindexNetwork() { FlatNetwork flat = _network.Structure.Flat; int neuronCount = 0; int weightCount = 0; for (int i = 0; i < flat.LayerCounts.Length; i++) { if (i > 0) { int from = flat.LayerFeedCounts[i - 1]; int to = flat.LayerCounts[i]; weightCount += from * to; } flat.LayerIndex[i] = neuronCount; flat.WeightIndex[i] = weightCount; neuronCount += flat.LayerCounts[i]; } flat.LayerOutput = new double[neuronCount]; flat.LayerSums = new double[neuronCount]; flat.ClearContext(); flat.InputCount = flat.LayerFeedCounts[flat.LayerCounts.Length - 1]; flat.OutputCount = flat.LayerFeedCounts[0]; }
/// <summary> /// Build the synapse and layer structure. This method should be called after /// you are done adding layers to a network, or change the network's logic /// property. /// </summary> /// public void FinalizeStructure() { if (_layers.Count < 2) { throw new NeuralNetworkError( "There must be at least two layers before the structure is finalized."); } var flatLayers = new FlatLayer[_layers.Count]; for (int i = 0; i < _layers.Count; i++) { var layer = (BasicLayer)_layers[i]; if (layer.Activation == null) { layer.Activation = new ActivationLinear(); } flatLayers[i] = layer; } _flat = new FlatNetwork(flatLayers); FinalizeLimit(); _layers.Clear(); EnforceLimit(); }
/// <summary> /// Tran a network using RPROP. /// </summary> /// /// <param name="flat">The network to train.</param> /// <param name="trainingSet">The training data to use.</param> public TrainFlatNetworkResilient(FlatNetwork flat, IMLDataSet trainingSet) : this( flat, trainingSet, RPROPConst.DefaultZeroTolerance, RPROPConst.DefaultInitialUpdate, RPROPConst.DefaultMaxStep) { }
/// <summary> /// Construct a gradient worker. /// </summary> /// /// <param name="theNetwork">The network to train.</param> /// <param name="theOwner">The owner that is doing the training.</param> /// <param name="theTraining">The training data.</param> /// <param name="theLow">The low index to use in the training data.</param> /// <param name="theHigh">The high index to use in the training data.</param> /// <param name="theFlatSpots">Holds an array of flat spot constants.</param> public GradientWorker(FlatNetwork theNetwork, Propagation theOwner, IMLDataSet theTraining, int theLow, int theHigh, double[] theFlatSpots, IErrorFunction ef) { _errorCalculation = new ErrorCalculation(); _network = theNetwork; _training = theTraining; _low = theLow; _high = theHigh; _owner = theOwner; _flatSpot = theFlatSpots; _layerDelta = new double[_network.LayerOutput.Length]; _gradients = new double[_network.Weights.Length]; _actual = new double[_network.OutputCount]; _weights = _network.Weights; _layerIndex = _network.LayerIndex; _layerCounts = _network.LayerCounts; _weightIndex = _network.WeightIndex; _layerOutput = _network.LayerOutput; _layerSums = _network.LayerSums; _layerFeedCounts = _network.LayerFeedCounts; _ef = ef; }
/// <summary> /// Create an array of doubles to hold the specified flat network. /// </summary> /// <param name="flat">The flat network to use as a model.</param> /// <returns>The new array.</returns> public double[] CreateParams(FlatNetwork flat) { var result = new double[flat.ActivationFunctions.Length]; EngineArray.Fill(result, 1); return(result); }
/// <summary> /// Construct a QPROP trainer for flat networks. /// </summary> /// <param name="network">The network to train.</param> /// <param name="training">The training data.</param> /// <param name="theLearningRate">The learning rate. 2 is a good suggestion as /// a learning rate to start with. If it fails to converge, /// then drop it. Just like backprop, except QPROP can /// take higher learning rates.</param> public TrainFlatNetworkQPROP(FlatNetwork network, IMLDataSet training, double theLearningRate) : base(network, training) { LearningRate = theLearningRate; LastDelta = new double[Network.Weights.Length]; Decay = 0.0001d; OutputEpsilon = 0.35; }
/// <summary> /// Construct a backprop trainer for flat networks. /// </summary> /// /// <param name="network">The network to train.</param> /// <param name="training">The training data.</param> /// <param name="theLearningRate">The learning rate.</param> /// <param name="theMomentum">The momentum.</param> public TrainFlatNetworkBackPropagation(FlatNetwork network, IMLDataSet training, double theLearningRate, double theMomentum) : base(network, training) { _momentum = theMomentum; _learningRate = theLearningRate; _lastDelta = new double[network.Weights.Length]; }
/// <summary> /// Assign random values to the network. The range will be the min/max of /// existing neurons. /// </summary> /// /// <param name="targetLayer">The target layer.</param> /// <param name="neuron">The target neuron.</param> public void RandomizeNeuron(int targetLayer, int neuron) { FlatNetwork flat = _network.Structure.Flat; double low = EngineArray.Min(flat.Weights); double high = EngineArray.Max(flat.Weights); RandomizeNeuron(targetLayer, neuron, true, low, high, false, 0.0d); }
/// <summary> /// Construct a backprop trainer for flat networks. /// </summary> /// /// <param name="network">The network to train.</param> /// <param name="training">The training data.</param> /// <param name="learningRate">The learning rate.</param> /// <param name="momentum">The momentum.</param> public TrainFlatNetworkBackPropagation(FlatNetwork network, IEngineDataSet training, double learningRate, double momentum) : base(network, training) { this.momentum = momentum; this.learningRate = learningRate; this.lastDelta = new double[network.Weights.Length]; }
private void CheckWithModel(FlatNetwork model, FlatNetwork pruned) { Assert.AreEqual(model.Weights.Length, pruned.Weights.Length); Assert.AreEqual(model.ContextTargetOffset, pruned.ContextTargetOffset); Assert.AreEqual(model.ContextTargetSize, pruned.ContextTargetSize); Assert.AreEqual(model.LayerCounts, pruned.LayerCounts); Assert.AreEqual(model.LayerFeedCounts, pruned.LayerFeedCounts); Assert.AreEqual(model.LayerIndex, pruned.LayerIndex); Assert.AreEqual(model.LayerOutput.Length, pruned.LayerOutput.Length); Assert.AreEqual(model.WeightIndex, pruned.WeightIndex); }
/// <inheritdoc/> public virtual void Init(BasicNetwork theNetwork, IMLDataSet theTraining) { int weightCount = theNetwork.Structure.Flat.Weights.Length; _flat = theNetwork.Flat; _training = theTraining; _network = theNetwork; _gradients = new double[weightCount]; _hessianMatrix = new Matrix(weightCount, weightCount); _hessian = _hessianMatrix.Data; }
/// <inheritdoc/> public virtual void Init(BasicNetwork theNetwork, IMLDataSet theTraining) { int weightCount = theNetwork.Structure.Flat.Weights.Length; flat = theNetwork.Flat; training = theTraining; network = theNetwork; gradients = new double[weightCount]; hessianMatrix = new Matrix(weightCount, weightCount); hessian = hessianMatrix.Data; derivative = new double[weightCount]; }
/// <summary> /// Construct a cross validation trainer. /// </summary> /// /// <param name="train">The training</param> /// <param name="k">The number of folds.</param> public CrossValidationKFold(IMLTrain train, int k) : base(train.Method, (FoldedDataSet)train.Training) { _train = train; Folded.Fold(k); _flatNetwork = ((BasicNetwork)train.Method).Structure.Flat; _networks = new NetworkFold[k]; for (int i = 0; i < _networks.Length; i++) { _networks[i] = new NetworkFold(_flatNetwork); } }
/// <summary> /// Construct a resilient trainer for flat networks. /// </summary> /// /// <param name="network">The network to train.</param> /// <param name="training">The training data to use.</param> /// <param name="zeroTolerance">How close a number should be to zero to be counted as zero.</param> /// <param name="initialUpdate">The initial update value.</param> /// <param name="maxStep">The maximum step value.</param> public TrainFlatNetworkResilient(FlatNetwork network, IEngineDataSet training, double zeroTolerance, double initialUpdate, double maxStep) : base(network, training) { this.updateValues = new double[network.Weights.Length]; this.zeroTolerance = zeroTolerance; this.maxStep = maxStep; for (int i = 0; i < this.updateValues.Length; i++) { this.updateValues[i] = initialUpdate; } }
/// <summary> /// Construct a cross validation trainer. /// </summary> /// <param name="train">The training.</param> /// <param name="k">The number of folds.</param> public CrossValidationKFold(ITrain train, int k) : base(train.Network, (FoldedDataSet)train.Training) { this.train = train; Folded.Fold(k); this.flatNetwork = train.Network.Structure.Flat; this.networks = new NetworkFold[k]; for (int i = 0; i < networks.Length; i++) { this.networks[i] = new NetworkFold(flatNetwork); } }
/// <summary> /// Construct a resilient trainer for flat networks. /// </summary> /// /// <param name="network">The network to train.</param> /// <param name="training">The training data to use.</param> /// <param name="zeroTolerance">How close a number should be to zero to be counted as zero.</param> /// <param name="initialUpdate">The initial update value.</param> /// <param name="maxStep">The maximum step value.</param> public TrainFlatNetworkResilient(FlatNetwork network, IMLDataSet training, double zeroTolerance, double initialUpdate, double maxStep) : base(network, training) { _updateValues = new double[network.Weights.Length]; _zeroTolerance = zeroTolerance; _maxStep = maxStep; _lastWeightChanged = new double[Network.Weights.Length]; _lastDelta = new double[Network.Weights.Length]; for (int i = 0; i < _updateValues.Length; i++) { _updateValues[i] = initialUpdate; } }
/// <summary> /// Construct a propagation object. /// </summary> /// /// <param name="network">The network.</param> /// <param name="training">The training set.</param> protected Propagation(IContainsFlat network, IMLDataSet training) : base(TrainingImplementationType.Iterative) { _network = network; _flat = network.Flat; _training = training; Gradients = new double[_flat.Weights.Length]; _lastGradient = new double[_flat.Weights.Length]; _indexable = training; _numThreads = 0; _reportedException = null; FixFlatSpot = true; ErrorFunction = new LinearErrorFunction(); }
/// <summary> /// Train a flat network multithreaded. /// </summary> /// /// <param name="network">The network to train.</param> /// <param name="training">The training data to use.</param> protected TrainFlatNetworkProp(FlatNetwork network, IMLDataSet training) { _training = training; _network = network; Gradients = new double[_network.Weights.Length]; _lastGradient = new double[_network.Weights.Length]; _indexable = training; _numThreads = 0; _reportedException = null; FixFlatSpot = true; ErrorFunction = new LinearErrorFunction(); }
/// <inheritdoc/> public void Save(Stream os, Object obj) { var xout = new EncogWriteHelper(os); var net = (BasicNetwork)obj; FlatNetwork flat = net.Structure.Flat; xout.AddSection("BASIC"); xout.AddSubSection("PARAMS"); xout.AddProperties(net.Properties); xout.AddSubSection("NETWORK"); xout.WriteProperty(BasicNetwork.TagBeginTraining, flat.BeginTraining); xout.WriteProperty(BasicNetwork.TagConnectionLimit, flat.ConnectionLimit); xout.WriteProperty(BasicNetwork.TagContextTargetOffset, flat.ContextTargetOffset); xout.WriteProperty(BasicNetwork.TagContextTargetSize, flat.ContextTargetSize); xout.WriteProperty(BasicNetwork.TagEndTraining, flat.EndTraining); xout.WriteProperty(BasicNetwork.TagHasContext, flat.HasContext); xout.WriteProperty(PersistConst.InputCount, flat.InputCount); xout.WriteProperty(BasicNetwork.TagLayerCounts, flat.LayerCounts); xout.WriteProperty(BasicNetwork.TagLayerFeedCounts, flat.LayerFeedCounts); xout.WriteProperty(BasicNetwork.TagLayerContextCount, flat.LayerContextCount); xout.WriteProperty(BasicNetwork.TagLayerIndex, flat.LayerIndex); xout.WriteProperty(PersistConst.Output, flat.LayerOutput); xout.WriteProperty(PersistConst.OutputCount, flat.OutputCount); xout.WriteProperty(BasicNetwork.TagWeightIndex, flat.WeightIndex); xout.WriteProperty(PersistConst.Weights, flat.Weights); xout.WriteProperty(BasicNetwork.TagBiasActivation, flat.BiasActivation); xout.AddSubSection("ACTIVATION"); foreach (IActivationFunction af in flat.ActivationFunctions) { xout.AddColumn(af.GetType().Name); for (int i = 0; i < af.Params.Length; i++) { xout.AddColumn(af.Params[i]); } xout.WriteLine(); } xout.Flush(); }
/// <summary> /// Construct a kernel to train the network. /// </summary> /// /// <param name="device">The OpenCL device to use.</param> /// <param name="flat">The network to train.</param> /// <param name="training">The training data.</param> /// <param name="tempDataSize">How much temp data.</param> public KernelNetworkTrain(EncogCLDevice device, FlatNetwork flat, IEngineIndexableSet training, int tempDataSize) : base(device, "Encog.Engine.Resources.KernelNetTrain.txt", "NetworkTrain") { this.training = training; this.trainingLength = (int)this.training.Count; this.device = device; this.flat = flat; this.weightInArray = new float[flat.Weights.Length]; this.weightOutArray = new float[flat.Weights.Length]; this.tempDataArray = new float[tempDataSize]; this.gradients = new float[flat.Weights.Length]; this.layerDeltaSize = 0; for (int i = 0; i < flat.LayerCounts.Length; i++) { this.layerDeltaSize += flat.LayerCounts[i]; } int inputSize = flat.InputCount; int idealSize = flat.OutputCount; this.inputArray = new float[inputSize * this.trainingLength]; this.idealArray = new float[idealSize * this.trainingLength]; this.paramArray = new int[10]; IEngineData pair = BasicEngineData.CreatePair( flat.InputCount, flat.OutputCount); int inputIndex = 0; int idealIndex = 0; for (int i = 0; i < this.trainingLength; i++) { training.GetRecord(i, pair); for (int col = 0; col < flat.InputCount; col++) { this.inputArray[inputIndex++] = (float)pair.InputArray[col]; } for (int col = 0; col < flat.OutputCount; col++) { this.idealArray[idealIndex++] = (float)pair.IdealArray[col]; } } }
/// <summary> /// Train a flat network multithreaded. /// </summary> /// /// <param name="network">The network to train.</param> /// <param name="training">The training data to use.</param> public TrainFlatNetworkProp(FlatNetwork network, IEngineDataSet training) { if (!(training is IEngineIndexableSet)) { throw new EncogEngineError( "Training data must be Indexable for this training type."); } this.training = training; this.network = network; this.gradients = new double[this.network.Weights.Length]; this.lastGradient = new double[this.network.Weights.Length]; this.indexable = (IEngineIndexableSet)training; this.numThreads = 0; this.reportedException = null; }
/// <summary> /// Train a flat network multithreaded. /// </summary> /// /// <param name="network">The network to train.</param> /// <param name="training">The training data to use.</param> /// <param name="profile">The OpenCL training profile.</param> public TrainFlatNetworkOpenCL(FlatNetwork network, IEngineDataSet training, OpenCLTrainingProfile profile) { (new ValidateForOpenCL()).Validate(network); if (!(training is IEngineIndexableSet)) { throw new EncogEngineError( "Training data must be Indexable for this training type."); } if (EncogEngine.Instance.CL == null) { throw new EncogEngineError( "You must enable OpenCL before using this training type."); } this.profile = profile; this.network = network; this.training = (IEngineIndexableSet)training; }
public StochasticGradientDescent(IContainsFlat network, IMLDataSet training, IGenerateRandom theRandom) : base(TrainingImplementationType.Iterative) { Training = training; UpdateRule = new AdamUpdate(); if (!(training is BatchDataSet)) { BatchSize = 25; } _method = network; _flat = network.Flat; _layerDelta = new double[_flat.LayerOutput.Length]; _gradients = new double[_flat.Weights.Length]; _errorCalculation = new ErrorCalculation(); _rnd = theRandom; LearningRate = 0.001; Momentum = 0.9; }
/// <summary> /// Compile the kernel. /// </summary> /// /// <param name="options">The options.</param> /// <param name="network">The network to compile for.</param> /// <param name="profile">The OpenCL training profile to use.</param> public void Compile(IDictionary <String, String> options, OpenCLTrainingProfile profile, FlatNetwork network) { IActivationFunction activation = network.ActivationFunctions[0]; StringBuilder source = new StringBuilder(); source.Append("#define ACTIVATION(x,slope)"); source.Append(activation.GetOpenCLExpression(false)); source.Append("\r\n"); source.Append("#define DERIVATIVE(x,slope)"); source.Append(activation.GetOpenCLExpression(true)); source.Append("\r\n"); source.Append(ResourceLoader.LoadString(SourceName)); CLSource = source.ToString(); Compile(options); profile.CalculateKernelParams(this, training); // setup Init(profile); }
public static FlatNetwork ToFlatNetwork(this MatrixNetwork @this) { var net = new FlatNetwork(); foreach (var layer in @this.Layers) { var flatLayer = new FlatLayer(); for (var i = 0; i < layer.Weights.RowCount; i++) { var el = new { w = layer.Weights.Row(i), b = layer.Biases[i] }; var flatNeuron = new FlatNeuron(); flatNeuron.Weights.AddRange(el.w); flatNeuron.Bias = el.b; flatLayer.Neurons.Add(flatNeuron); } net.Layers.Add(flatLayer); } return(net); }
/// <summary> /// Construct the chain rule worker. /// </summary> /// <param name="theNetwork">The network to calculate a Hessian for.</param> /// <param name="theTraining">The training data.</param> /// <param name="theLow">The low range.</param> /// <param name="theHigh">The high range.</param> public ChainRuleWorker(FlatNetwork theNetwork, IMLDataSet theTraining, int theLow, int theHigh) { int weightCount = theNetwork.Weights.Length; _training = theTraining; _flat = theNetwork; _layerDelta = new double[_flat.LayerOutput.Length]; _actual = new double[_flat.OutputCount]; _derivative = new double[weightCount]; _totDeriv = new double[weightCount]; _gradients = new double[weightCount]; _weights = _flat.Weights; _layerIndex = _flat.LayerIndex; _layerCounts = _flat.LayerCounts; _weightIndex = _flat.WeightIndex; _layerOutput = _flat.LayerOutput; _layerSums = _flat.LayerSums; _layerFeedCounts = _flat.LayerFeedCounts; _low = theLow; _high = theHigh; }
/// <summary> /// Construct the training object. /// </summary> /// /// <param name="network">The network to train.</param> /// <param name="training">The training data to use.</param> public TrainFlatNetworkSCG(FlatNetwork network, IMLDataSet training) : base(network, training) { _success = true; _success = true; _delta = 0; _lambda2 = 0; _lambda = FirstLambda; _oldError = 0; _magP = 0; _restart = false; _weights = EngineArray.ArrayCopy(network.Weights); int numWeights = _weights.Length; _oldWeights = new double[numWeights]; _oldGradient = new double[numWeights]; _p = new double[numWeights]; _r = new double[numWeights]; _mustInit = true; }
/// <summary> /// Construct a trainer for flat networks to use the Manhattan update rule. /// </summary> /// /// <param name="network">The network to train.</param> /// <param name="training">The training data to use.</param> /// <param name="theLearningRate">The learning rate to use.</param> public TrainFlatNetworkManhattan(FlatNetwork network, IMLDataSet training, double theLearningRate) : base(network, training) { _learningRate = theLearningRate; _zeroTolerance = RPROPConst.DefaultZeroTolerance; }
/// <summary> /// Construct a gradient worker. /// </summary> /// /// <param name="theNetwork">The network to train.</param> /// <param name="theOwner">The owner that is doing the training.</param> /// <param name="theTraining">The training data.</param> /// <param name="theLow">The low index to use in the training data.</param> /// <param name="theHigh">The high index to use in the training data.</param> /// <param name="theFlatSpots">Holds an array of flat spot constants.</param> public GradientWorker(FlatNetwork theNetwork, TrainFlatNetworkProp theOwner, IMLDataSet theTraining, int theLow, int theHigh, double[] theFlatSpots, IErrorFunction ef) { _errorCalculation = new ErrorCalculation(); _network = theNetwork; _training = theTraining; _low = theLow; _high = theHigh; _owner = theOwner; _flatSpot = theFlatSpots; _layerDelta = new double[_network.LayerOutput.Length]; _gradients = new double[_network.Weights.Length]; _actual = new double[_network.OutputCount]; _weights = _network.Weights; _layerIndex = _network.LayerIndex; _layerCounts = _network.LayerCounts; _weightIndex = _network.WeightIndex; _layerOutput = _network.LayerOutput; _layerSums = _network.LayerSums; _layerFeedCounts = _network.LayerFeedCounts; _ef = ef; _pair = BasicMLDataPair.CreatePair(_network.InputCount, _network.OutputCount); }