/// <summary>
/// Adds a nuron to the network at the given level and with the given
/// activation function. It returns null if it was unable to add the nuron.
/// </summary>
/// <param name="func">Activation function of the nuron</param>
/// <param name="level">Level of the nuron</param>
/// <returns>The inserted nuron</returns>
public NuronOld AddNuron(ActFunc func, int level)
{
//clamps the level to be within range
if (level > MAX_LV)
{
level = MAX_LV;
}
if (level < 0)
{
level = 0;
}
//tries to generate a random index
int index = RandIndex();
if (index < 0)
{
return(null);
}
//creates the node and adds it
NuronOld n = new NuronOld(this, func, level, index);
nurons.Add(index, n);
return(n);
}
internal override Vector <double> PropogateError(Vector <double> outputError, double errorWeight, Vector <double> inputCacheOverride = null, Vector <double> additionalError = null)
{
Vector <double> inputError;
if (inputCacheOverride == null)
{
inputError = outputError.PointwiseMultiply(ActFunc.Derivative(InputCache * Weights));
}
else
{
inputError = outputError.PointwiseMultiply(ActFunc.Derivative(inputCacheOverride * Weights));
}
if (additionalError != null)
{
inputError -= additionalError;
}
BiasErrorCache -= inputError * errorWeight;
WeightErrorCache -= errorWeight * InputCache.OuterProduct(inputError);
if (RegMode == RegularizationMode.L2)
{
WeightErrorCache -= errorWeight * NormalizationWeight * Weights;
}
return(Weights * inputError);
}
internal Nuron(NetworkAuto network, Nuron other)
{
this.network = network;
this.func = other.func;
this.index = other.index;
this.level = other.level;
this.value = 0.0;
this.vprev = 0.0;
}
/// <summary>
/// Creates a new nuron on a given network, with a given activation
/// function and level depth. An index is required to be able to
/// refrence the nuron from outside the network.
/// </summary>
/// <param name="network">Network on wich to create the nuron</param>
/// <param name="func">Activation funciton of the nuron</param>
/// <param name="index">Index of the nuron</param>
/// <param name="level">Level of the nuron</param>
internal Nuron(NetworkAuto network, ActFunc func, int index, int level)
{
this.network = network;
this.func = func;
this.index = index;
this.level = level;
this.value = 0.0;
this.vprev = 0.0;
}
/// <summary>
/// Creates a new nuron on a given network, with a given activation
/// function and level depth. An index is required to be able to
/// refrence the nuron from outside the network.
/// </summary>
/// <param name="network">Network on wich to create the nuron</param>
/// <param name="func">Activation funciton of the nuron</param>
/// <param name="level">Level of the nuron</param>
/// <param name="index">Index of the nuron</param>
internal NuronOld(NetworkCPP network, ActFunc func, int level, int index)
{
this.network = network;
this.func = func;
this.level = level;
this.index = index;
inputs = new VListArray <Axon>();
value = 0.0;
}
/// <summary>
/// Expands the curent nural net, in place, by either inserting a
/// new node along a pre-existing edge or creating a new edge. The
/// probibility of either event occoring is determined by the
/// existing topology.
/// </summary>
public void ExpandSelf()
{
NuronOld n1, n2;
//generates a pair of random nurons
bool test = RandPair(out n1, out n2);
Axon target = n2.GetAxon(n1.Index);
if (test && target == null)
{
//creates a new axon between the nurons
double weight = rng.RandGauss() * SDN;
n2.AddAxonInit(n1, weight);
}
else if (test && !target.Enabled)
{
//reinitilises the axon with a new weight
target.Weight = rng.RandGauss() * SDN;
target.Enabled = true;
}
else if ((n2.Level - n1.Level) > 3)
{
//disables the target edge
double weight = target.Weight;
target.Enabled = false;
//creates a new node with a random funciton
int index = RandIndex();
int level = (n2.Level + n1.Level) / 2;
ActFunc func = RandFunc();
//aborts the operation if we fail to insert
if (index < 0)
{
return;
}
//inserts the node into our data-structor
NuronOld nx = new NuronOld(this, func, level, index);
nurons.Add(index, nx);
//adds axons to replace the missing axon
nx.AddAxonInit(n1, 1.0);
n2.AddAxonInit(nx, weight);
}
}
internal override Vector <double> PropogateError(Vector <double> outputError, double errorWeight, Vector <double> inputCacheOverride = null, Vector <double> additionalError = null)
{
if (additionalError != null)
{
throw new NNException("Additional error is not supported for Convolutional Layers");
}
int nNodes = OutputHeight * OutputWidth;
int nInputNodes = InputHeight * InputWidth;
Vector <double> directInputError = 10 * outputError.PointwiseMultiply(ActFunc.Derivative(DirectInputCache)); // DEBUG 10*
Vector <double> inputError = new DenseVector(InputDimension);
for (var channel = 0; channel < InputDepth; channel++)
{
for (var f = 0; f < NFilters; f++)
{
int outputLayer = channel * NFilters + f;
for (var i = 0; i < OutputHeight; i++)
{
for (var j = 0; j < OutputWidth; j++)
{
double nodeError = directInputError[outputLayer * nNodes + i * OutputWidth + j];
BiasErrorCache[f] -= errorWeight * nodeError / nNodes; // Average the update over all instances of the same filter.
// Get the matrix of weight errors by multiplying the nodeError by the matrix in the input cache.
Matrix <double> weightError = new DenseMatrix(Weights[f].RowCount, Weights[f].ColumnCount);
for (var m = 0; m < weightError.RowCount; m++)
{
for (var n = 0; n < weightError.ColumnCount; n++)
{
int inputIndex = channel * nInputNodes + (m + i * Stride) * InputWidth + n + j * Stride;
weightError[m, n] = nodeError * InputCache[inputIndex];
inputError[inputIndex] = Weights[f][m, n] * nodeError;
}
}
WeightErrorCache[f] -= errorWeight * weightError / nNodes; // Average the update over all instances of the same filter.
}
}
}
}
return(inputError);
}
/// <summary>
/// Creates a copy of a given nuron for a given network. All of the
/// refrences to axons in the old network, now point to axons in
/// the new network. All other values remain the same.
/// </summary>
/// <param name="network">The network containing this nuron</param>
/// <param name="other">Another neuron to copy its values</param>
internal NuronOld(NetworkCPP network, NuronOld other)
{
//sets the network refrence to the new network
this.network = network;
//copies the other vlaues
func = other.func;
level = other.level;
value = other.value;
inputs = new VListArray <Axon>(other.inputs.Count);
//makes a deep copy of the list of inputs
foreach (Axon ax in other.inputs)
{
Axon clone = new Axon(ax);
this.inputs.Add(clone);
}
}
internal override Vector <double> Process(Vector <double> input)
{
InputCache = input;
for (var i = 0; i < OutputHeight; i++)
{
for (var j = 0; j < OutputWidth; j++)
{
for (var inputFrame = 0; inputFrame < InputDepth; inputFrame++)
{
for (var f = 0; f < NFilters; f++)
{
// (inputFrame * NFilters + f) = the output frame
// OutputHeight * OutputWidth = Number of pixels per frame
int outputIndex = (inputFrame * NFilters + f) * OutputHeight * OutputWidth + i * OutputWidth + j;
DirectInputCache[outputIndex] = ApplyFilter(input, inputFrame, f, i * Stride, j * Stride);
}
}
}
}
return(10 * (ActFunc.Of(DirectInputCache) - 0.5)); // DEBUG 10*
}
internal NuronComp(ActFunc func)
{
this.func = func;
this.value = 0.0;
}
internal override Vector <double> Process(Vector <double> input)
{
InputCache = input;
return(ActFunc.Of(input * Weights));
}
private void Expand(VRandom rng)
{
Nuron n1, n2;
//generates a pair of random nurons on diffrent levels
bool pass = GetRandomPair(rng, out n1, out n2);
if (!pass)
{
return;
}
//determins if this should be a recurent conneciton
bool testrec = recurent;
testrec = testrec && !n1.IsInput;
testrec = testrec && rng.RandBool(P_Recur);
if (testrec)
{
Nuron nx = n1;
n1 = n2;
n2 = nx;
}
//creates a temporary axon between the two nurons
double w = rng.RandGauss() * SD_NEW;
Axon temp = new Axon(n1.Index, n2.Index, w);
//atempts to gain the actual axon if it exists
Axon actual = axons.GetValue(temp.Index);
if (actual == null)
{
//adds the new axon into the network
axons.Add(temp.Index, temp);
}
else if (actual.Enabled == false)
{
//reneables the axon and updates it weight
actual.Enabled = true;
actual.Weight = temp.Weight;
}
else
{
//determins if we are able to insert a new node
int nid1 = GetRandomIndex(rng);
if (nid1 < 0)
{
return;
}
//sets the level of the new node in the middle
if (Math.Abs(n1.Level - n2.Level) < 3)
{
return;
}
int level = (n1.Level + n2.Level) / 2;
//adds a new node to the network
ActFunc act = GetRandomActivation(rng);
Nuron n3 = new Nuron(this, act, nid1, level);
nurons.Add(n3.Index, n3);
//inserts two new axons where the old one used to be
int nid0 = actual.Source;
int nid2 = actual.Target;
//Axon ax1 = new Axon(nid0, nid1, actual.Weight);
//Axon ax2 = new Axon(nid1, nid2, temp.Weight);
Axon ax1 = new Axon(nid0, nid1, actual.Weight);
Axon ax2 = new Axon(nid1, nid2, 1.0);
//we use the overwitre comand because there may be hash collisions
axons.Overwrite(ax1.Index, ax1);
axons.Overwrite(ax2.Index, ax2);
//deactivates the old axon
actual.Enabled = false;
}
}
