Esempio n. 1
0
 /// <summary>
 /// Construct a trainer for flat networks to use the Manhattan update rule.
 /// </summary>
 ///
 /// <param name="network">The network to train.</param>
 /// <param name="training">The training data to use.</param>
 /// <param name="learningRate">The learning rate to use.</param>
 public TrainFlatNetworkManhattan(FlatNetwork network,
                                  IEngineDataSet training, double learningRate)
     : base(network, training)
 {
     this.learningRate  = learningRate;
     this.zeroTolerance = RPROPConst.DEFAULT_ZERO_TOLERANCE;
 }
        /// <summary>
        /// Construct a gradient worker.
        /// </summary>
        ///
        /// <param name="network">The network to train.</param>
        /// <param name="owner">The owner that is doing the training.</param>
        /// <param name="training">The training data.</param>
        /// <param name="low">The low index to use in the training data.</param>
        /// <param name="high">The high index to use in the training data.</param>
        public GradientWorkerCPU(FlatNetwork network,
                                 TrainFlatNetworkProp owner,
                                 IEngineIndexableSet training, int low, int high)
        {
            this.errorCalculation = new ErrorCalculation();
            this.network          = network;
            this.training         = training;
            this.low   = low;
            this.high  = high;
            this.owner = owner;

            this.stopwatch = new Stopwatch();

            this.layerDelta = new double[network.LayerOutput.Length];
            this.gradients  = new double[network.Weights.Length];
            this.actual     = new double[network.OutputCount];

            this.weights         = network.Weights;
            this.layerIndex      = network.LayerIndex;
            this.layerCounts     = network.LayerCounts;
            this.weightIndex     = network.WeightIndex;
            this.layerOutput     = network.LayerOutput;
            this.layerFeedCounts = network.LayerFeedCounts;

            this.pair = BasicEngineData.CreatePair(network.InputCount,
                                                   network.OutputCount);
        }
Esempio n. 3
0
        /// <summary>
        ///     Create an array of activations based on a flat network.
        /// </summary>
        /// <param name="flat">The flat network.</param>
        /// <returns></returns>
        public int[] CreateActivations(FlatNetwork flat)
        {
            var result = new int[flat.ActivationFunctions.Length];

            for (int i = 0; i < flat.ActivationFunctions.Length; i++)
            {
                IActivationFunction af = flat.ActivationFunctions[i];

                if (af is ActivationLinear)
                {
                    result[i] = 0;
                }
                else if (af is ActivationTANH)
                {
                    result[i] = 1;
                }
                if (af is ActivationSigmoid)
                {
                    result[i] = 2;
                }
                if (af is ActivationElliottSymmetric)
                {
                    result[i] = 3;
                }
                if (af is ActivationElliott)
                {
                    result[i] = 4;
                }
            }

            return(result);
        }
        /// <summary>
        /// Construct the training object.
        /// </summary>
        ///
        /// <param name="network">The network to train.</param>
        /// <param name="training">The training data to use.</param>
        public TrainFlatNetworkSCG(FlatNetwork network,
                                   IEngineDataSet training)
            : base(network, training)
        {
            this.success = true;

            this.success  = true;
            this.delta    = 0;
            this.lambda2  = 0;
            this.lambda   = TrainFlatNetworkSCG.FIRST_LAMBDA;
            this.oldError = 0;
            this.magP     = 0;
            this.restart  = false;

            this.weights = EngineArray.ArrayCopy(network.Weights);
            int numWeights = this.weights.Length;

            // this.gradients = new double[numWeights];
            this.oldWeights  = new double[numWeights];
            this.oldGradient = new double[numWeights];

            this.p          = new double[numWeights];
            this.r          = new double[numWeights];
            this.shouldInit = true;
        }
Esempio n. 5
0
        /// <summary>
        /// Creat new index values for the network.
        /// </summary>
        ///
        private void ReindexNetwork()
        {
            FlatNetwork flat = _network.Structure.Flat;

            int neuronCount = 0;
            int weightCount = 0;

            for (int i = 0; i < flat.LayerCounts.Length; i++)
            {
                if (i > 0)
                {
                    int from = flat.LayerFeedCounts[i - 1];
                    int to   = flat.LayerCounts[i];
                    weightCount += from * to;
                }
                flat.LayerIndex[i]  = neuronCount;
                flat.WeightIndex[i] = weightCount;
                neuronCount        += flat.LayerCounts[i];
            }

            flat.LayerOutput = new double[neuronCount];
            flat.LayerSums   = new double[neuronCount];
            flat.ClearContext();

            flat.InputCount  = flat.LayerFeedCounts[flat.LayerCounts.Length - 1];
            flat.OutputCount = flat.LayerFeedCounts[0];
        }
Esempio n. 6
0
        /// <summary>
        /// Build the synapse and layer structure. This method should be called after
        /// you are done adding layers to a network, or change the network's logic
        /// property.
        /// </summary>
        ///
        public void FinalizeStructure()
        {
            if (_layers.Count < 2)
            {
                throw new NeuralNetworkError(
                          "There must be at least two layers before the structure is finalized.");
            }

            var flatLayers = new FlatLayer[_layers.Count];

            for (int i = 0; i < _layers.Count; i++)
            {
                var layer = (BasicLayer)_layers[i];
                if (layer.Activation == null)
                {
                    layer.Activation = new ActivationLinear();
                }

                flatLayers[i] = layer;
            }

            _flat = new FlatNetwork(flatLayers);

            FinalizeLimit();
            _layers.Clear();
            EnforceLimit();
        }
 /// <summary>
 /// Tran a network using RPROP.
 /// </summary>
 ///
 /// <param name="flat">The network to train.</param>
 /// <param name="trainingSet">The training data to use.</param>
 public TrainFlatNetworkResilient(FlatNetwork flat,
                                  IMLDataSet trainingSet)
     : this(
         flat, trainingSet, RPROPConst.DefaultZeroTolerance, RPROPConst.DefaultInitialUpdate,
         RPROPConst.DefaultMaxStep)
 {
 }
Esempio n. 8
0
        /// <summary>
        /// Construct a gradient worker.
        /// </summary>
        ///
        /// <param name="theNetwork">The network to train.</param>
        /// <param name="theOwner">The owner that is doing the training.</param>
        /// <param name="theTraining">The training data.</param>
        /// <param name="theLow">The low index to use in the training data.</param>
        /// <param name="theHigh">The high index to use in the training data.</param>
        /// <param name="theFlatSpots">Holds an array of flat spot constants.</param>
        public GradientWorker(FlatNetwork theNetwork,
                              Propagation theOwner, IMLDataSet theTraining,
                              int theLow, int theHigh, double[] theFlatSpots, IErrorFunction ef)
        {
            _errorCalculation = new ErrorCalculation();
            _network          = theNetwork;
            _training         = theTraining;
            _low      = theLow;
            _high     = theHigh;
            _owner    = theOwner;
            _flatSpot = theFlatSpots;

            _layerDelta = new double[_network.LayerOutput.Length];
            _gradients  = new double[_network.Weights.Length];
            _actual     = new double[_network.OutputCount];

            _weights         = _network.Weights;
            _layerIndex      = _network.LayerIndex;
            _layerCounts     = _network.LayerCounts;
            _weightIndex     = _network.WeightIndex;
            _layerOutput     = _network.LayerOutput;
            _layerSums       = _network.LayerSums;
            _layerFeedCounts = _network.LayerFeedCounts;
            _ef = ef;
        }
Esempio n. 9
0
        /// <summary>
        ///     Create an array of doubles to hold the specified flat network.
        /// </summary>
        /// <param name="flat">The flat network to use as a model.</param>
        /// <returns>The new array.</returns>
        public double[] CreateParams(FlatNetwork flat)
        {
            var result = new double[flat.ActivationFunctions.Length];

            EngineArray.Fill(result, 1);
            return(result);
        }
 /// <summary>
 /// Construct a QPROP trainer for flat networks.
 /// </summary>
 /// <param name="network">The network to train.</param>
 /// <param name="training">The training data.</param>
 /// <param name="theLearningRate">The learning rate.  2 is a good suggestion as 
 ///          a learning rate to start with.  If it fails to converge, 
 ///          then drop it.  Just like backprop, except QPROP can 
 ///            take higher learning rates.</param>
 public TrainFlatNetworkQPROP(FlatNetwork network,
                              IMLDataSet training, double theLearningRate) : base(network, training)
 {
     LearningRate = theLearningRate;
     LastDelta = new double[Network.Weights.Length];
     Decay = 0.0001d;
     OutputEpsilon = 0.35;
 }
 /// <summary>
 /// Construct a QPROP trainer for flat networks.
 /// </summary>
 /// <param name="network">The network to train.</param>
 /// <param name="training">The training data.</param>
 /// <param name="theLearningRate">The learning rate.  2 is a good suggestion as
 ///          a learning rate to start with.  If it fails to converge,
 ///          then drop it.  Just like backprop, except QPROP can
 ///            take higher learning rates.</param>
 public TrainFlatNetworkQPROP(FlatNetwork network,
                              IMLDataSet training, double theLearningRate) : base(network, training)
 {
     LearningRate  = theLearningRate;
     LastDelta     = new double[Network.Weights.Length];
     Decay         = 0.0001d;
     OutputEpsilon = 0.35;
 }
 /// <summary>
 /// Construct a backprop trainer for flat networks.
 /// </summary>
 ///
 /// <param name="network">The network to train.</param>
 /// <param name="training">The training data.</param>
 /// <param name="theLearningRate">The learning rate.</param>
 /// <param name="theMomentum">The momentum.</param>
 public TrainFlatNetworkBackPropagation(FlatNetwork network,
                                        IMLDataSet training, double theLearningRate,
                                        double theMomentum) : base(network, training)
 {
     _momentum = theMomentum;
     _learningRate = theLearningRate;
     _lastDelta = new double[network.Weights.Length];
 }
Esempio n. 13
0
        /// <summary>
        /// Assign random values to the network. The range will be the min/max of
        /// existing neurons.
        /// </summary>
        ///
        /// <param name="targetLayer">The target layer.</param>
        /// <param name="neuron">The target neuron.</param>
        public void RandomizeNeuron(int targetLayer, int neuron)
        {
            FlatNetwork flat = _network.Structure.Flat;
            double      low  = EngineArray.Min(flat.Weights);
            double      high = EngineArray.Max(flat.Weights);

            RandomizeNeuron(targetLayer, neuron, true, low, high, false, 0.0d);
        }
 /// <summary>
 /// Construct a backprop trainer for flat networks.
 /// </summary>
 ///
 /// <param name="network">The network to train.</param>
 /// <param name="training">The training data.</param>
 /// <param name="theLearningRate">The learning rate.</param>
 /// <param name="theMomentum">The momentum.</param>
 public TrainFlatNetworkBackPropagation(FlatNetwork network,
                                        IMLDataSet training, double theLearningRate,
                                        double theMomentum) : base(network, training)
 {
     _momentum     = theMomentum;
     _learningRate = theLearningRate;
     _lastDelta    = new double[network.Weights.Length];
 }
 /// <summary>
 /// Construct a backprop trainer for flat networks.
 /// </summary>
 ///
 /// <param name="network">The network to train.</param>
 /// <param name="training">The training data.</param>
 /// <param name="learningRate">The learning rate.</param>
 /// <param name="momentum">The momentum.</param>
 public TrainFlatNetworkBackPropagation(FlatNetwork network,
                                        IEngineDataSet training, double learningRate,
                                        double momentum)
     : base(network, training)
 {
     this.momentum     = momentum;
     this.learningRate = learningRate;
     this.lastDelta    = new double[network.Weights.Length];
 }
 private void CheckWithModel(FlatNetwork model, FlatNetwork pruned)
 {
     Assert.AreEqual(model.Weights.Length, pruned.Weights.Length);
     Assert.AreEqual(model.ContextTargetOffset, pruned.ContextTargetOffset);
     Assert.AreEqual(model.ContextTargetSize, pruned.ContextTargetSize);
     Assert.AreEqual(model.LayerCounts, pruned.LayerCounts);
     Assert.AreEqual(model.LayerFeedCounts, pruned.LayerFeedCounts);
     Assert.AreEqual(model.LayerIndex, pruned.LayerIndex);
     Assert.AreEqual(model.LayerOutput.Length, pruned.LayerOutput.Length);
     Assert.AreEqual(model.WeightIndex, pruned.WeightIndex);
 }
Esempio n. 17
0
        /// <inheritdoc/>
        public virtual void Init(BasicNetwork theNetwork, IMLDataSet theTraining)
        {
            int weightCount = theNetwork.Structure.Flat.Weights.Length;

            _flat          = theNetwork.Flat;
            _training      = theTraining;
            _network       = theNetwork;
            _gradients     = new double[weightCount];
            _hessianMatrix = new Matrix(weightCount, weightCount);
            _hessian       = _hessianMatrix.Data;
        }
Esempio n. 18
0
        /// <inheritdoc/>
        public virtual void Init(BasicNetwork theNetwork, IMLDataSet theTraining)
        {
            int weightCount = theNetwork.Structure.Flat.Weights.Length;

            flat          = theNetwork.Flat;
            training      = theTraining;
            network       = theNetwork;
            gradients     = new double[weightCount];
            hessianMatrix = new Matrix(weightCount, weightCount);
            hessian       = hessianMatrix.Data;
            derivative    = new double[weightCount];
        }
Esempio n. 19
0
        /// <summary>
        /// Construct a cross validation trainer.
        /// </summary>
        ///
        /// <param name="train">The training</param>
        /// <param name="k">The number of folds.</param>
        public CrossValidationKFold(IMLTrain train, int k) : base(train.Method, (FoldedDataSet)train.Training)
        {
            _train = train;
            Folded.Fold(k);

            _flatNetwork = ((BasicNetwork)train.Method).Structure.Flat;

            _networks = new NetworkFold[k];
            for (int i = 0; i < _networks.Length; i++)
            {
                _networks[i] = new NetworkFold(_flatNetwork);
            }
        }
Esempio n. 20
0
        /// <summary>
        /// Construct a resilient trainer for flat networks.
        /// </summary>
        ///
        /// <param name="network">The network to train.</param>
        /// <param name="training">The training data to use.</param>
        /// <param name="zeroTolerance">How close a number should be to zero to be counted as zero.</param>
        /// <param name="initialUpdate">The initial update value.</param>
        /// <param name="maxStep">The maximum step value.</param>
        public TrainFlatNetworkResilient(FlatNetwork network,
                                         IEngineDataSet training, double zeroTolerance,
                                         double initialUpdate, double maxStep)
            : base(network, training)
        {
            this.updateValues  = new double[network.Weights.Length];
            this.zeroTolerance = zeroTolerance;
            this.maxStep       = maxStep;

            for (int i = 0; i < this.updateValues.Length; i++)
            {
                this.updateValues[i] = initialUpdate;
            }
        }
        /// <summary>
        /// Construct a cross validation trainer.
        /// </summary>
        /// <param name="train">The training.</param>
        /// <param name="k">The number of folds.</param>
        public CrossValidationKFold(ITrain train, int k)
            : base(train.Network, (FoldedDataSet)train.Training)
        {
            this.train = train;
            Folded.Fold(k);

            this.flatNetwork = train.Network.Structure.Flat;

            this.networks = new NetworkFold[k];
            for (int i = 0; i < networks.Length; i++)
            {
                this.networks[i] = new NetworkFold(flatNetwork);
            }
        }
        /// <summary>
        /// Construct a resilient trainer for flat networks.
        /// </summary>
        ///
        /// <param name="network">The network to train.</param>
        /// <param name="training">The training data to use.</param>
        /// <param name="zeroTolerance">How close a number should be to zero to be counted as zero.</param>
        /// <param name="initialUpdate">The initial update value.</param>
        /// <param name="maxStep">The maximum step value.</param>
        public TrainFlatNetworkResilient(FlatNetwork network,
                                         IMLDataSet training, double zeroTolerance,
                                         double initialUpdate, double maxStep) : base(network, training)
        {
            _updateValues = new double[network.Weights.Length];
            _zeroTolerance = zeroTolerance;
            _maxStep = maxStep;
            _lastWeightChanged = new double[Network.Weights.Length];
            _lastDelta = new double[Network.Weights.Length];

            for (int i = 0; i < _updateValues.Length; i++)
            {
                _updateValues[i] = initialUpdate;
            }
        }
Esempio n. 23
0
        /// <summary>
        /// Construct a propagation object.
        /// </summary>
        ///
        /// <param name="network">The network.</param>
        /// <param name="training">The training set.</param>
        protected Propagation(IContainsFlat network, IMLDataSet training) : base(TrainingImplementationType.Iterative)
        {
            _network  = network;
            _flat     = network.Flat;
            _training = training;

            Gradients     = new double[_flat.Weights.Length];
            _lastGradient = new double[_flat.Weights.Length];

            _indexable         = training;
            _numThreads        = 0;
            _reportedException = null;
            FixFlatSpot        = true;
            ErrorFunction      = new LinearErrorFunction();
        }
        /// <summary>
        /// Construct a resilient trainer for flat networks.
        /// </summary>
        ///
        /// <param name="network">The network to train.</param>
        /// <param name="training">The training data to use.</param>
        /// <param name="zeroTolerance">How close a number should be to zero to be counted as zero.</param>
        /// <param name="initialUpdate">The initial update value.</param>
        /// <param name="maxStep">The maximum step value.</param>
        public TrainFlatNetworkResilient(FlatNetwork network,
                                         IMLDataSet training, double zeroTolerance,
                                         double initialUpdate, double maxStep) : base(network, training)
        {
            _updateValues      = new double[network.Weights.Length];
            _zeroTolerance     = zeroTolerance;
            _maxStep           = maxStep;
            _lastWeightChanged = new double[Network.Weights.Length];
            _lastDelta         = new double[Network.Weights.Length];

            for (int i = 0; i < _updateValues.Length; i++)
            {
                _updateValues[i] = initialUpdate;
            }
        }
        /// <summary>
        /// Train a flat network multithreaded.
        /// </summary>
        ///
        /// <param name="network">The network to train.</param>
        /// <param name="training">The training data to use.</param>
        protected TrainFlatNetworkProp(FlatNetwork network,
                                       IMLDataSet training)
        {
            _training = training;
            _network  = network;

            Gradients     = new double[_network.Weights.Length];
            _lastGradient = new double[_network.Weights.Length];

            _indexable         = training;
            _numThreads        = 0;
            _reportedException = null;
            FixFlatSpot        = true;
            ErrorFunction      = new LinearErrorFunction();
        }
        /// <inheritdoc/>
        public void Save(Stream os, Object obj)
        {
            var         xout = new EncogWriteHelper(os);
            var         net  = (BasicNetwork)obj;
            FlatNetwork flat = net.Structure.Flat;

            xout.AddSection("BASIC");
            xout.AddSubSection("PARAMS");
            xout.AddProperties(net.Properties);
            xout.AddSubSection("NETWORK");

            xout.WriteProperty(BasicNetwork.TagBeginTraining,
                               flat.BeginTraining);
            xout.WriteProperty(BasicNetwork.TagConnectionLimit,
                               flat.ConnectionLimit);
            xout.WriteProperty(BasicNetwork.TagContextTargetOffset,
                               flat.ContextTargetOffset);
            xout.WriteProperty(BasicNetwork.TagContextTargetSize,
                               flat.ContextTargetSize);
            xout.WriteProperty(BasicNetwork.TagEndTraining, flat.EndTraining);
            xout.WriteProperty(BasicNetwork.TagHasContext, flat.HasContext);
            xout.WriteProperty(PersistConst.InputCount, flat.InputCount);
            xout.WriteProperty(BasicNetwork.TagLayerCounts, flat.LayerCounts);
            xout.WriteProperty(BasicNetwork.TagLayerFeedCounts,
                               flat.LayerFeedCounts);
            xout.WriteProperty(BasicNetwork.TagLayerContextCount,
                               flat.LayerContextCount);
            xout.WriteProperty(BasicNetwork.TagLayerIndex, flat.LayerIndex);
            xout.WriteProperty(PersistConst.Output, flat.LayerOutput);
            xout.WriteProperty(PersistConst.OutputCount, flat.OutputCount);
            xout.WriteProperty(BasicNetwork.TagWeightIndex, flat.WeightIndex);
            xout.WriteProperty(PersistConst.Weights, flat.Weights);
            xout.WriteProperty(BasicNetwork.TagBiasActivation,
                               flat.BiasActivation);
            xout.AddSubSection("ACTIVATION");

            foreach (IActivationFunction af  in  flat.ActivationFunctions)
            {
                xout.AddColumn(af.GetType().Name);
                for (int i = 0; i < af.Params.Length; i++)
                {
                    xout.AddColumn(af.Params[i]);
                }
                xout.WriteLine();
            }

            xout.Flush();
        }
Esempio n. 27
0
        /// <summary>
        /// Construct a kernel to train the network.
        /// </summary>
        ///
        /// <param name="device">The OpenCL device to use.</param>
        /// <param name="flat">The network to train.</param>
        /// <param name="training">The training data.</param>
        /// <param name="tempDataSize">How much temp data.</param>
        public KernelNetworkTrain(EncogCLDevice device,
                                  FlatNetwork flat, IEngineIndexableSet training,
                                  int tempDataSize)
            : base(device, "Encog.Engine.Resources.KernelNetTrain.txt", "NetworkTrain")
        {
            this.training       = training;
            this.trainingLength = (int)this.training.Count;
            this.device         = device;
            this.flat           = flat;
            this.weightInArray  = new float[flat.Weights.Length];
            this.weightOutArray = new float[flat.Weights.Length];
            this.tempDataArray  = new float[tempDataSize];
            this.gradients      = new float[flat.Weights.Length];

            this.layerDeltaSize = 0;
            for (int i = 0; i < flat.LayerCounts.Length; i++)
            {
                this.layerDeltaSize += flat.LayerCounts[i];
            }

            int inputSize = flat.InputCount;
            int idealSize = flat.OutputCount;

            this.inputArray = new float[inputSize * this.trainingLength];
            this.idealArray = new float[idealSize * this.trainingLength];
            this.paramArray = new int[10];

            IEngineData pair = BasicEngineData.CreatePair(
                flat.InputCount, flat.OutputCount);

            int inputIndex = 0;
            int idealIndex = 0;

            for (int i = 0; i < this.trainingLength; i++)
            {
                training.GetRecord(i, pair);
                for (int col = 0; col < flat.InputCount; col++)
                {
                    this.inputArray[inputIndex++] = (float)pair.InputArray[col];
                }

                for (int col = 0; col < flat.OutputCount; col++)
                {
                    this.idealArray[idealIndex++] = (float)pair.IdealArray[col];
                }
            }
        }
Esempio n. 28
0
        /// <summary>
        /// Train a flat network multithreaded.
        /// </summary>
        ///
        /// <param name="network">The network to train.</param>
        /// <param name="training">The training data to use.</param>
        public TrainFlatNetworkProp(FlatNetwork network,
                                    IEngineDataSet training)
        {
            if (!(training is IEngineIndexableSet))
            {
                throw new EncogEngineError(
                          "Training data must be Indexable for this training type.");
            }

            this.training = training;
            this.network  = network;

            this.gradients    = new double[this.network.Weights.Length];
            this.lastGradient = new double[this.network.Weights.Length];

            this.indexable         = (IEngineIndexableSet)training;
            this.numThreads        = 0;
            this.reportedException = null;
        }
        /// <summary>
        /// Train a flat network multithreaded.
        /// </summary>
        ///
        /// <param name="network">The network to train.</param>
        /// <param name="training">The training data to use.</param>
        /// <param name="profile">The OpenCL training profile.</param>
        public TrainFlatNetworkOpenCL(FlatNetwork network,
                                      IEngineDataSet training, OpenCLTrainingProfile profile)
        {
            (new ValidateForOpenCL()).Validate(network);

            if (!(training is IEngineIndexableSet))
            {
                throw new EncogEngineError(
                          "Training data must be Indexable for this training type.");
            }

            if (EncogEngine.Instance.CL == null)
            {
                throw new EncogEngineError(
                          "You must enable OpenCL before using this training type.");
            }

            this.profile  = profile;
            this.network  = network;
            this.training = (IEngineIndexableSet)training;
        }
        public StochasticGradientDescent(IContainsFlat network,
                                         IMLDataSet training, IGenerateRandom theRandom) :
            base(TrainingImplementationType.Iterative)
        {
            Training   = training;
            UpdateRule = new AdamUpdate();

            if (!(training is BatchDataSet))
            {
                BatchSize = 25;
            }

            _method           = network;
            _flat             = network.Flat;
            _layerDelta       = new double[_flat.LayerOutput.Length];
            _gradients        = new double[_flat.Weights.Length];
            _errorCalculation = new ErrorCalculation();
            _rnd         = theRandom;
            LearningRate = 0.001;
            Momentum     = 0.9;
        }
Esempio n. 31
0
        /// <summary>
        /// Compile the kernel.
        /// </summary>
        ///
        /// <param name="options">The options.</param>
        /// <param name="network">The network to compile for.</param>
        /// <param name="profile">The OpenCL training profile to use.</param>
        public void Compile(IDictionary <String, String> options,
                            OpenCLTrainingProfile profile, FlatNetwork network)
        {
            IActivationFunction activation = network.ActivationFunctions[0];
            StringBuilder       source     = new StringBuilder();

            source.Append("#define ACTIVATION(x,slope)");
            source.Append(activation.GetOpenCLExpression(false));
            source.Append("\r\n");

            source.Append("#define DERIVATIVE(x,slope)");
            source.Append(activation.GetOpenCLExpression(true));
            source.Append("\r\n");

            source.Append(ResourceLoader.LoadString(SourceName));
            CLSource = source.ToString();

            Compile(options);
            profile.CalculateKernelParams(this, training);
            // setup
            Init(profile);
        }
Esempio n. 32
0
        public static FlatNetwork ToFlatNetwork(this MatrixNetwork @this)
        {
            var net = new FlatNetwork();

            foreach (var layer in @this.Layers)
            {
                var flatLayer = new FlatLayer();
                for (var i = 0; i < layer.Weights.RowCount; i++)
                {
                    var el = new { w = layer.Weights.Row(i), b = layer.Biases[i] };

                    var flatNeuron = new FlatNeuron();
                    flatNeuron.Weights.AddRange(el.w);
                    flatNeuron.Bias = el.b;

                    flatLayer.Neurons.Add(flatNeuron);
                }

                net.Layers.Add(flatLayer);
            }
            return(net);
        }
        /// <summary>
        /// Construct the chain rule worker.
        /// </summary>
        /// <param name="theNetwork">The network to calculate a Hessian for.</param>
        /// <param name="theTraining">The training data.</param>
        /// <param name="theLow">The low range.</param>
        /// <param name="theHigh">The high range.</param>
        public ChainRuleWorker(FlatNetwork theNetwork, IMLDataSet theTraining, int theLow, int theHigh)
        {
            int weightCount = theNetwork.Weights.Length;

            _training = theTraining;
            _flat     = theNetwork;

            _layerDelta = new double[_flat.LayerOutput.Length];
            _actual     = new double[_flat.OutputCount];
            _derivative = new double[weightCount];
            _totDeriv   = new double[weightCount];
            _gradients  = new double[weightCount];

            _weights         = _flat.Weights;
            _layerIndex      = _flat.LayerIndex;
            _layerCounts     = _flat.LayerCounts;
            _weightIndex     = _flat.WeightIndex;
            _layerOutput     = _flat.LayerOutput;
            _layerSums       = _flat.LayerSums;
            _layerFeedCounts = _flat.LayerFeedCounts;
            _low             = theLow;
            _high            = theHigh;
        }
        /// <summary>
        /// Construct the training object.
        /// </summary>
        ///
        /// <param name="network">The network to train.</param>
        /// <param name="training">The training data to use.</param>
        public TrainFlatNetworkSCG(FlatNetwork network,
                                   IMLDataSet training) : base(network, training)
        {
            _success = true;

            _success = true;
            _delta = 0;
            _lambda2 = 0;
            _lambda = FirstLambda;
            _oldError = 0;
            _magP = 0;
            _restart = false;

            _weights = EngineArray.ArrayCopy(network.Weights);
            int numWeights = _weights.Length;

            _oldWeights = new double[numWeights];
            _oldGradient = new double[numWeights];

            _p = new double[numWeights];
            _r = new double[numWeights];

            _mustInit = true;
        }
 /// <summary>
 /// Tran a network using RPROP.
 /// </summary>
 ///
 /// <param name="flat">The network to train.</param>
 /// <param name="trainingSet">The training data to use.</param>
 public TrainFlatNetworkResilient(FlatNetwork flat,
                                  IMLDataSet trainingSet)
     : this(
         flat, trainingSet, RPROPConst.DefaultZeroTolerance, RPROPConst.DefaultInitialUpdate,
         RPROPConst.DefaultMaxStep)
 {
 }
 /// <summary>
 /// Construct a trainer for flat networks to use the Manhattan update rule.
 /// </summary>
 ///
 /// <param name="network">The network to train.</param>
 /// <param name="training">The training data to use.</param>
 /// <param name="theLearningRate">The learning rate to use.</param>
 public TrainFlatNetworkManhattan(FlatNetwork network,
                                  IMLDataSet training, double theLearningRate) : base(network, training)
 {
     _learningRate = theLearningRate;
     _zeroTolerance = RPROPConst.DefaultZeroTolerance;
 }
Esempio n. 37
0
        /// <summary>
        /// Construct a gradient worker.
        /// </summary>
        ///
        /// <param name="theNetwork">The network to train.</param>
        /// <param name="theOwner">The owner that is doing the training.</param>
        /// <param name="theTraining">The training data.</param>
        /// <param name="theLow">The low index to use in the training data.</param>
        /// <param name="theHigh">The high index to use in the training data.</param>
        /// <param name="theFlatSpots">Holds an array of flat spot constants.</param>
        public GradientWorker(FlatNetwork theNetwork,
                                 TrainFlatNetworkProp theOwner, IMLDataSet theTraining,
                                 int theLow, int theHigh, double[] theFlatSpots, IErrorFunction ef)
        {
            _errorCalculation = new ErrorCalculation();
            _network = theNetwork;
            _training = theTraining;
            _low = theLow;
            _high = theHigh;
            _owner = theOwner;
            _flatSpot = theFlatSpots;

            _layerDelta = new double[_network.LayerOutput.Length];
            _gradients = new double[_network.Weights.Length];
            _actual = new double[_network.OutputCount];

            _weights = _network.Weights;
            _layerIndex = _network.LayerIndex;
            _layerCounts = _network.LayerCounts;
            _weightIndex = _network.WeightIndex;
            _layerOutput = _network.LayerOutput;
            _layerSums = _network.LayerSums;
            _layerFeedCounts = _network.LayerFeedCounts;
            _ef = ef;

            _pair = BasicMLDataPair.CreatePair(_network.InputCount,
                                              _network.OutputCount);
        }
        /// <summary>
        /// Train a flat network multithreaded.
        /// </summary>
        ///
        /// <param name="network">The network to train.</param>
        /// <param name="training">The training data to use.</param>
        protected TrainFlatNetworkProp(FlatNetwork network,
                                    IMLDataSet training)
        {
            _training = training;
            _network = network;

            Gradients = new double[_network.Weights.Length];
            _lastGradient = new double[_network.Weights.Length];

            _indexable = training;
            _numThreads = 0;
            _reportedException = null;
            FixFlatSpot = true;
            ErrorFunction = new LinearErrorFunction();
        }