Network Training Properties class.
Beispiel #1
0
 /// <summary>
 /// Resets the neurons in the entire graph (see <see cref="Neuron.Reset(NetworkTrainingProperties)"/>).
 /// </summary>
 /// <param name="properties">Network training properties object.</param>
 public void ResetStates(NetworkTrainingProperties properties)
 {
     foreach (var node in this.GetVertices().OfType <Neuron>())
     {
         node.Reset(properties);
     }
 }
Beispiel #2
0
        /// <summary>Calculates and returns the error derivative (<see cref="Delta"/>) of this node.</summary>
        /// <param name="t">Error derivative from the previous layer (n + 1).</param>
        /// <param name="properties">Network training properties.</param>
        /// <returns>A double.</returns>
        public virtual double Error(double t, NetworkTrainingProperties properties)
        {
            _DeltaL = Delta;

            if (Out.Count == 0)
            {
                Delta = delta = -(t - Output);
            }

            else
            {
                if (In.Count > 0 && Out.Count > 0)
                {
                    double hp = this.ActivationFunction.Derivative(this.Input);
                    delta = Out.Sum(e => e.Weight * t) * hp;
                }

                Delta = Out.Sum(s => s.Target.delta * this.Output);
            }

            if (this.In.Count > 0)
            {
                for (int edge = 0; edge < this.In.Count; edge++)
                {
                    this.In[edge].Source.Error(this.Delta, properties);
                }
            }

            return(Delta);
        }
Beispiel #3
0
        public virtual ISequenceModel Generate(Matrix X, Matrix Y)
        {
            this.Preprocess(X);
            // because I said so...
            if (MaxIterations == -1)
            {
                MaxIterations = 500;
            }

            var network = Network.New().Create(X.Cols, Y.Cols, Activation, OutputFunction, epsilon: Epsilon);

            INetworkTrainer trainer = new GradientDescentTrainer();

            var model = new NeuralNetworkModel
            {
                Descriptor        = Descriptor,
                NormalizeFeatures = base.NormalizeFeatures,
                FeatureNormalizer = base.FeatureNormalizer,
                FeatureProperties = base.FeatureProperties,
                Network           = network
            };

            OnModelChanged(this, ModelEventArgs.Make(model, "Initialized"));

            NetworkTrainingProperties properties = NetworkTrainingProperties.Create(network, X.Rows, X.Cols, this.LearningRate, this.Lambda, this.MaxIterations);

            Vector loss = Vector.Zeros(this.MaxIterations);

            for (int i = 0; i < MaxIterations; i++)
            {
                properties.Iteration = i;

                network.ResetStates(properties);

                for (int x = 0; x < X.Rows; x++)
                {
                    network.Forward(X[x, VectorType.Row]);
                    //OnModelChanged(this, ModelEventArgs.Make(model, "Forward"));
                    network.Back(Y[x, VectorType.Row], properties, trainer);

                    loss[i] += network.Cost;
                }

                var output = String.Format("Run ({0}/{1}): {2}", i, MaxIterations, network.Cost);
                OnModelChanged(this, ModelEventArgs.Make(model, output));

                if (this.LossMinimized(loss, i))
                {
                    break;
                }
            }

            return(model);
        }
Beispiel #4
0
        /// <summary>Backpropagates the errors through the network given the supplied label.</summary>
        /// <param name="y">Label to process.</param>
        /// <param name="properties">Network training properties for use in learning.</param>
        public void Back(double y, NetworkTrainingProperties properties)
        {
            this.Cost = Score.ComputeRMSE(Vector.Create(this.Out.Length, () => y), this.Out.Select(s => s.Output).ToVector());

            // propagate error gradients
            for (int i = 0; i < Out.Length; i++)
            {
                Out[i].Error(y, properties);
            }

            // reset weights
            for (int i = 0; i < Out.Length; i++)
            {
                Out[i].Update(properties);
            }
        }
Beispiel #5
0
        /// <summary>Propagates a weight update event upstream through the network using the supplied learning rate.</summary>
        /// <param name="properties">Network training properties.</param>
        public virtual void Update(NetworkTrainingProperties properties)
        {
            for (var edge = 0; edge < In.Count; edge++)
            {
                Delta = 1.0 / properties.Examples * Delta;

                if (edge > 0)
                {
                    Delta = Delta + properties.Lambda / properties.Examples * In[edge].Weight;
                }

                if (!Constrained)
                {
                    In[edge].Weight = In[edge].Weight - properties.LearningRate * Delta;
                }
                In[edge].Source.Update(properties);
            }
        }
Beispiel #6
0
        /// <summary>
        /// Propagates a weight update event upstream through the network.
        /// </summary>
        /// <param name="properties">Network training properties.</param>
        /// <param name="networkTrainer">Network training method.</param>
        public virtual void Update(NetworkTrainingProperties properties, INetworkTrainer networkTrainer)
        {
            for (int edge = 0; edge < this.In.Count; edge++)
            {
                Delta = (1.0 / properties.Examples) * Delta;

                if (edge > 0)
                {
                    Delta = Delta + ((properties.Lambda / properties.Examples) * this.In[edge].Weight);
                }

                if (!this.Constrained)
                {
                    this.In[edge].Weight = networkTrainer.Update(this.In[edge].ParentId, this.In[edge].ChildId,
                                                                 nameof(Edge.Weight), this.In[edge].Weight, Delta, properties);
                }
                this.In[edge].Source.Update(properties, networkTrainer);
            }
        }
Beispiel #7
0
        /// <summary>Backpropagates the errors through the network given the supplied sequence label.</summary>
        /// <param name="y">Output vector to process.</param>
        /// <param name="properties">Network training properties for use in learning.</param>
        /// <param name="update">Indicates whether to update the weights after computing the errors.</param>
        public void Back(Vector y, NetworkTrainingProperties properties, bool update = true)
        {
            Cost = Score.ComputeRMSE(y, Out.Select(s => s.Output).ToVector());

            // CK
            // propagate error gradients
            for (var i = 0; i < Out.Length; i++)
            {
                Out[i].Error(y[i], properties);
            }

            if (update)
            {
                for (var i = 0; i < Out.Length; i++)
                {
                    Out[i].Update(properties);
                }
            }
        }
Beispiel #8
0
        /// <summary>Backpropagates the errors through the network given the supplied sequence label.</summary>
        /// <param name="y">Output vector to process.</param>
        /// <param name="properties">Network training properties for use in learning.</param>
        /// <param name="networkTrainer">Network training method.</param>
        /// <param name="update">Indicates whether to update the weights after computing the errors.</param>
        public void Back(Vector y, NetworkTrainingProperties properties, INetworkTrainer networkTrainer, bool update = true)
        {
            this.Cost = this.LossFunction.Compute(this.Output(), y);

            // CK
            // propagate error gradients
            for (int i = 0; i < Out.Length; i++)
            {
                Out[i].Error(y[i], properties);
            }

            if (update)
            {
                // reset weights
                for (int i = 0; i < Out.Length; i++)
                {
                    Out[i].Update(properties, networkTrainer);
                }
            }
        }
        public virtual ISequenceModel Generate(Matrix X, Matrix Y)
        {
            Preprocess(X);
            // because I said so...
            if (MaxIterations == -1)
            {
                MaxIterations = 500;
            }

            var network = Network.New().Create(X.Cols, Y.Cols, Activation, OutputFunction, epsilon: Epsilon);

            var model = new NeuralNetworkModel
            {
                Descriptor        = Descriptor,
                NormalizeFeatures = NormalizeFeatures,
                FeatureNormalizer = FeatureNormalizer,
                FeatureProperties = FeatureProperties,
                Network           = network
            };

            OnModelChanged(this, ModelEventArgs.Make(model, "Initialized"));

            var properties = NetworkTrainingProperties.Create(network, X.Rows, X.Cols, LearningRate, Lambda, MaxIterations);

            for (var i = 0; i < MaxIterations; i++)
            {
                properties.Iteration = i;

                for (var x = 0; x < X.Rows; x++)
                {
                    network.Forward(X[x, VectorType.Row]);
                    //OnModelChanged(this, ModelEventArgs.Make(model, "Forward"));
                    network.Back(Y[x, VectorType.Row], properties);
                }

                var output = string.Format("Run ({0}/{1}): {2}", i, MaxIterations, network.Cost);
                OnModelChanged(this, ModelEventArgs.Make(model, output));
            }

            return(model);
        }
Beispiel #10
0
        /// <summary>Propagates a weight update event upstream through the network using the supplied learning rate.</summary>
        /// <param name="properties">Network training properties.</param>
        public virtual void Update(NetworkTrainingProperties properties)
        {
            for (int edge = 0; edge < this.In.Count; edge++)
            {
                Delta = (1.0 / properties.Examples) * Delta;

                if (edge > 0)
                {
                    Delta = Delta + ((properties.Lambda / properties.Examples) * this.In[edge].Weight);
                }

                if (!this.Constrained)
                {
                    // using stochastic gradient descent averaged over training examples.
                    this.In[edge].Weight = this.In[edge].Weight - properties.LearningRate * Delta;

                    // RMSProp (needs to move into a neural optimizer method)
                    //double mean = (properties.Epsilon * _DeltaL) + ((1.0 - properties.Epsilon) * (Delta * Delta));
                    //this.In[edge].Weight = this.In[edge].Weight - properties.LearningRate * (Delta / System.Math.Sqrt(mean * mean));
                }
                this.In[edge].Source.Update(properties);
            }
        }
Beispiel #11
0
 /// <summary>
 /// Propogates a reset event downstream through the network.
 /// </summary>
 /// <param name="properties">Network training properties.</param>
 public virtual void Reset(NetworkTrainingProperties properties)
 {
 }
Beispiel #12
0
 /// <summary>Backpropagates the errors through the network given the supplied label.</summary>
 /// <param name="y">Label to process.</param>
 /// <param name="properties">Network training properties for use in learning.</param>
 /// <param name="networkTrainer">Network training method.</param>
 public void Back(double y, NetworkTrainingProperties properties, INetworkTrainer networkTrainer)
 {
     this.Back(Vector.Create(this.Out.Length, () => y), properties, networkTrainer);
 }
Beispiel #13
0
        /// <summary>Propagates a weight update event upstream through the network using the supplied learning rate.</summary>
        /// <param name="properties">Network training properties.</param>
        public virtual void Update(NetworkTrainingProperties properties)
        {
            for (int edge = 0; edge < this.In.Count; edge++)
            {
                Delta = (1.0 / properties.Examples) * Delta;

                if (edge > 0)
                    Delta = Delta + ((properties.Lambda / properties.Examples) * this.In[edge].Weight);

                if (!this.Constrained)
                {
                    // using stochastic gradient descent averaged over training examples.
                    this.In[edge].Weight = this.In[edge].Weight - properties.LearningRate * Delta;

                    // RMSProp (needs to move into a neural optimizer method)
                    //double mean = (properties.Epsilon * _DeltaL) + ((1.0 - properties.Epsilon) * (Delta * Delta));
                    //this.In[edge].Weight = this.In[edge].Weight - properties.LearningRate * (Delta / System.Math.Sqrt(mean * mean));
                }
                this.In[edge].Source.Update(properties);
            }
        }
Beispiel #14
0
 /// <summary>
 /// Propogates a reset event downstream through the network.
 /// </summary>
 /// <param name="properties">Network training properties.</param>
 public virtual void Reset(NetworkTrainingProperties properties)
 {
 }
Beispiel #15
0
        /// <summary>Calculates and returns the error derivative (<see cref="Delta"/>) of this node.</summary>
        /// <param name="t">Error derivative from the previous layer (n + 1).</param>
        /// <param name="properties">Network training properties.</param>
        /// <returns>A double.</returns>
        public virtual double Error(double t, NetworkTrainingProperties properties)
        {
            _DeltaL = Delta;

            if (Out.Count == 0)
                Delta = delta = -(t - Output);

            else
            {
                if (In.Count > 0 && Out.Count > 0)
                {
                    double hp = this.ActivationFunction.Derivative(this.Input);
                    delta = Out.Sum(e => e.Weight * t) * hp;
                }

                Delta = Out.Sum(s => s.Target.delta * this.Output);
            }

            if (this.In.Count > 0)
            {
                for (int edge = 0; edge < this.In.Count; edge++)
                {
                    this.In[edge].Source.Error(this.Delta, properties);
                }
            }

            return Delta;
        }