/// <summary>
/// Resets the neurons in the entire graph (see <see cref="Neuron.Reset(NetworkTrainingProperties)"/>).
/// </summary>
/// <param name="properties">Network training properties object.</param>
public void ResetStates(NetworkTrainingProperties properties)
{
foreach (var node in this.GetVertices().OfType <Neuron>())
{
node.Reset(properties);
}
}
/// <summary>Calculates and returns the error derivative (<see cref="Delta"/>) of this node.</summary>
/// <param name="t">Error derivative from the previous layer (n + 1).</param>
/// <param name="properties">Network training properties.</param>
/// <returns>A double.</returns>
public virtual double Error(double t, NetworkTrainingProperties properties)
{
_DeltaL = Delta;
if (Out.Count == 0)
{
Delta = delta = -(t - Output);
}
else
{
if (In.Count > 0 && Out.Count > 0)
{
double hp = this.ActivationFunction.Derivative(this.Input);
delta = Out.Sum(e => e.Weight * t) * hp;
}
Delta = Out.Sum(s => s.Target.delta * this.Output);
}
if (this.In.Count > 0)
{
for (int edge = 0; edge < this.In.Count; edge++)
{
this.In[edge].Source.Error(this.Delta, properties);
}
}
return(Delta);
}
public virtual ISequenceModel Generate(Matrix X, Matrix Y)
{
this.Preprocess(X);
// because I said so...
if (MaxIterations == -1)
{
MaxIterations = 500;
}
var network = Network.New().Create(X.Cols, Y.Cols, Activation, OutputFunction, epsilon: Epsilon);
INetworkTrainer trainer = new GradientDescentTrainer();
var model = new NeuralNetworkModel
{
Descriptor = Descriptor,
NormalizeFeatures = base.NormalizeFeatures,
FeatureNormalizer = base.FeatureNormalizer,
FeatureProperties = base.FeatureProperties,
Network = network
};
OnModelChanged(this, ModelEventArgs.Make(model, "Initialized"));
NetworkTrainingProperties properties = NetworkTrainingProperties.Create(network, X.Rows, X.Cols, this.LearningRate, this.Lambda, this.MaxIterations);
Vector loss = Vector.Zeros(this.MaxIterations);
for (int i = 0; i < MaxIterations; i++)
{
properties.Iteration = i;
network.ResetStates(properties);
for (int x = 0; x < X.Rows; x++)
{
network.Forward(X[x, VectorType.Row]);
//OnModelChanged(this, ModelEventArgs.Make(model, "Forward"));
network.Back(Y[x, VectorType.Row], properties, trainer);
loss[i] += network.Cost;
}
var output = String.Format("Run ({0}/{1}): {2}", i, MaxIterations, network.Cost);
OnModelChanged(this, ModelEventArgs.Make(model, output));
if (this.LossMinimized(loss, i))
{
break;
}
}
return(model);
}
/// <summary>Backpropagates the errors through the network given the supplied label.</summary>
/// <param name="y">Label to process.</param>
/// <param name="properties">Network training properties for use in learning.</param>
public void Back(double y, NetworkTrainingProperties properties)
{
this.Cost = Score.ComputeRMSE(Vector.Create(this.Out.Length, () => y), this.Out.Select(s => s.Output).ToVector());
// propagate error gradients
for (int i = 0; i < Out.Length; i++)
{
Out[i].Error(y, properties);
}
// reset weights
for (int i = 0; i < Out.Length; i++)
{
Out[i].Update(properties);
}
}
/// <summary>Propagates a weight update event upstream through the network using the supplied learning rate.</summary>
/// <param name="properties">Network training properties.</param>
public virtual void Update(NetworkTrainingProperties properties)
{
for (var edge = 0; edge < In.Count; edge++)
{
Delta = 1.0 / properties.Examples * Delta;
if (edge > 0)
{
Delta = Delta + properties.Lambda / properties.Examples * In[edge].Weight;
}
if (!Constrained)
{
In[edge].Weight = In[edge].Weight - properties.LearningRate * Delta;
}
In[edge].Source.Update(properties);
}
}
/// <summary>
/// Propagates a weight update event upstream through the network.
/// </summary>
/// <param name="properties">Network training properties.</param>
/// <param name="networkTrainer">Network training method.</param>
public virtual void Update(NetworkTrainingProperties properties, INetworkTrainer networkTrainer)
{
for (int edge = 0; edge < this.In.Count; edge++)
{
Delta = (1.0 / properties.Examples) * Delta;
if (edge > 0)
{
Delta = Delta + ((properties.Lambda / properties.Examples) * this.In[edge].Weight);
}
if (!this.Constrained)
{
this.In[edge].Weight = networkTrainer.Update(this.In[edge].ParentId, this.In[edge].ChildId,
nameof(Edge.Weight), this.In[edge].Weight, Delta, properties);
}
this.In[edge].Source.Update(properties, networkTrainer);
}
}
/// <summary>Backpropagates the errors through the network given the supplied sequence label.</summary>
/// <param name="y">Output vector to process.</param>
/// <param name="properties">Network training properties for use in learning.</param>
/// <param name="update">Indicates whether to update the weights after computing the errors.</param>
public void Back(Vector y, NetworkTrainingProperties properties, bool update = true)
{
Cost = Score.ComputeRMSE(y, Out.Select(s => s.Output).ToVector());
// CK
// propagate error gradients
for (var i = 0; i < Out.Length; i++)
{
Out[i].Error(y[i], properties);
}
if (update)
{
for (var i = 0; i < Out.Length; i++)
{
Out[i].Update(properties);
}
}
}
/// <summary>Backpropagates the errors through the network given the supplied sequence label.</summary>
/// <param name="y">Output vector to process.</param>
/// <param name="properties">Network training properties for use in learning.</param>
/// <param name="networkTrainer">Network training method.</param>
/// <param name="update">Indicates whether to update the weights after computing the errors.</param>
public void Back(Vector y, NetworkTrainingProperties properties, INetworkTrainer networkTrainer, bool update = true)
{
this.Cost = this.LossFunction.Compute(this.Output(), y);
// CK
// propagate error gradients
for (int i = 0; i < Out.Length; i++)
{
Out[i].Error(y[i], properties);
}
if (update)
{
// reset weights
for (int i = 0; i < Out.Length; i++)
{
Out[i].Update(properties, networkTrainer);
}
}
}
public virtual ISequenceModel Generate(Matrix X, Matrix Y)
{
Preprocess(X);
// because I said so...
if (MaxIterations == -1)
{
MaxIterations = 500;
}
var network = Network.New().Create(X.Cols, Y.Cols, Activation, OutputFunction, epsilon: Epsilon);
var model = new NeuralNetworkModel
{
Descriptor = Descriptor,
NormalizeFeatures = NormalizeFeatures,
FeatureNormalizer = FeatureNormalizer,
FeatureProperties = FeatureProperties,
Network = network
};
OnModelChanged(this, ModelEventArgs.Make(model, "Initialized"));
var properties = NetworkTrainingProperties.Create(network, X.Rows, X.Cols, LearningRate, Lambda, MaxIterations);
for (var i = 0; i < MaxIterations; i++)
{
properties.Iteration = i;
for (var x = 0; x < X.Rows; x++)
{
network.Forward(X[x, VectorType.Row]);
//OnModelChanged(this, ModelEventArgs.Make(model, "Forward"));
network.Back(Y[x, VectorType.Row], properties);
}
var output = string.Format("Run ({0}/{1}): {2}", i, MaxIterations, network.Cost);
OnModelChanged(this, ModelEventArgs.Make(model, output));
}
return(model);
}
/// <summary>Propagates a weight update event upstream through the network using the supplied learning rate.</summary>
/// <param name="properties">Network training properties.</param>
public virtual void Update(NetworkTrainingProperties properties)
{
for (int edge = 0; edge < this.In.Count; edge++)
{
Delta = (1.0 / properties.Examples) * Delta;
if (edge > 0)
{
Delta = Delta + ((properties.Lambda / properties.Examples) * this.In[edge].Weight);
}
if (!this.Constrained)
{
// using stochastic gradient descent averaged over training examples.
this.In[edge].Weight = this.In[edge].Weight - properties.LearningRate * Delta;
// RMSProp (needs to move into a neural optimizer method)
//double mean = (properties.Epsilon * _DeltaL) + ((1.0 - properties.Epsilon) * (Delta * Delta));
//this.In[edge].Weight = this.In[edge].Weight - properties.LearningRate * (Delta / System.Math.Sqrt(mean * mean));
}
this.In[edge].Source.Update(properties);
}
}
/// <summary>
/// Propogates a reset event downstream through the network.
/// </summary>
/// <param name="properties">Network training properties.</param>
public virtual void Reset(NetworkTrainingProperties properties)
{
}
/// <summary>Backpropagates the errors through the network given the supplied label.</summary>
/// <param name="y">Label to process.</param>
/// <param name="properties">Network training properties for use in learning.</param>
/// <param name="networkTrainer">Network training method.</param>
public void Back(double y, NetworkTrainingProperties properties, INetworkTrainer networkTrainer)
{
this.Back(Vector.Create(this.Out.Length, () => y), properties, networkTrainer);
}
/// <summary>Propagates a weight update event upstream through the network using the supplied learning rate.</summary>
/// <param name="properties">Network training properties.</param>
public virtual void Update(NetworkTrainingProperties properties)
{
for (int edge = 0; edge < this.In.Count; edge++)
{
Delta = (1.0 / properties.Examples) * Delta;
if (edge > 0)
Delta = Delta + ((properties.Lambda / properties.Examples) * this.In[edge].Weight);
if (!this.Constrained)
{
// using stochastic gradient descent averaged over training examples.
this.In[edge].Weight = this.In[edge].Weight - properties.LearningRate * Delta;
// RMSProp (needs to move into a neural optimizer method)
//double mean = (properties.Epsilon * _DeltaL) + ((1.0 - properties.Epsilon) * (Delta * Delta));
//this.In[edge].Weight = this.In[edge].Weight - properties.LearningRate * (Delta / System.Math.Sqrt(mean * mean));
}
this.In[edge].Source.Update(properties);
}
}
/// <summary>
/// Propogates a reset event downstream through the network.
/// </summary>
/// <param name="properties">Network training properties.</param>
public virtual void Reset(NetworkTrainingProperties properties)
{
}
/// <summary>Calculates and returns the error derivative (<see cref="Delta"/>) of this node.</summary>
/// <param name="t">Error derivative from the previous layer (n + 1).</param>
/// <param name="properties">Network training properties.</param>
/// <returns>A double.</returns>
public virtual double Error(double t, NetworkTrainingProperties properties)
{
_DeltaL = Delta;
if (Out.Count == 0)
Delta = delta = -(t - Output);
else
{
if (In.Count > 0 && Out.Count > 0)
{
double hp = this.ActivationFunction.Derivative(this.Input);
delta = Out.Sum(e => e.Weight * t) * hp;
}
Delta = Out.Sum(s => s.Target.delta * this.Output);
}
if (this.In.Count > 0)
{
for (int edge = 0; edge < this.In.Count; edge++)
{
this.In[edge].Source.Error(this.Delta, properties);
}
}
return Delta;
}
