/// <summary>
/// Copy constructor.
/// </summary>
/// <param name="original">the NNetUnit object to be copied</param>
///
public NNetUnit(NNetUnit original)
{
mInput = original.mInput;
mSlope = original.mSlope;
mAmplify = original.mAmplify;
mActivationType = original.mActivationType;
}
/// <summary>
/// Calculates the error signal on each individual unit in the output layer.
///
/// Uses the gradient descent method to search the error surface.
/// </summary>
/// <param name="nNet">the network undergoing training</param>
/// <param name="outErr">the calculated output unit errors</param>
/// <param name="response">the network response values</param>
/// <param name="nTarget">the index of the corresponding target values in the training set</param>
///
private void CalcOutputError(NeuralNet nNet, List <double> outErr,
List <double> response, int nTarget)
{
List <double> unitInputs = new List <double>();
List <double> targetVec = mTrainTarget[nTarget];
// get the output layer activation unit details
ActiveT outType = nNet.OutputUnitType;
double outSlope = nNet.OutputUnitSlope;
double outAmplify = nNet.OutputUnitAmplify;
// get the output layer activation unit input values
nNet.GetUnitInputs(unitInputs, nNet.NumLayers);
for (int i = 0; i < response.Count; i++)
{
double yi = response[i];
double xi = unitInputs[i];
// follow the steepest path on the error function by moving along the gradient
// of the output units activation function - the gradient descent method
double err = (targetVec[i] - yi) * GetGradient(outType, outSlope, outAmplify, xi);
outErr.Add(err);
}
}
/// <summary>
/// Populates the form's Activation function list boxes with the available
/// values of the ActiveT enumerated type.
/// </summary>
///
private void PopulateActivationListBoxes()
{
int id = 0;
bool done = false;
while (!done)
{
ActiveT func = (ActiveT)id;
string funcID = func.ToString();
if (funcID == id.ToString())
{
// we have processed all the valid ActiveT enum values
done = true;
}
else
{
funcID = funcID.Substring(1);
this.OutFuncListBox.Items.Add(funcID);
this.HidFuncListBox.Items.Add(funcID);
id++;
}
}
// select the first entries in the list boxes
this.OutFuncListBox.SelectedIndex = 0;
this.HidFuncListBox.SelectedIndex = 0;
}
/// <summary>
/// Calculates the error signal on each individual unit within the networks hidden layers.
///
/// Uses the gradient descent method to search the error surface.
/// </summary>
/// <param name="hidErr">the calculated hidden unit errors</param>
/// <param name="outErr">the output unit errors</param>
/// <param name="nNet">the network undergoing training</param>
///
private void CalcHiddenError(List <List <double> > hidErr,
List <double> outErr, NeuralNet nNet)
{
int nHidden = nNet.NumLayers;
double slope = 1, amplify = 1;
ActiveT unitType = ActiveT.kUnipolar;
// initialise the the previous layer error with the output layer errors
List <double> prevErr = new List <double>();
prevErr.AddRange(outErr);
// start with the last hidden layer and work back to the first
for (int i = nHidden; i >= 1; i--)
{
List <double> unitInputs = new List <double>();
List <double> layerErr = new List <double>();
// get the weighted connections for the current hidden layer
NNetWeightedConnect wtConnect = nNet.GetWeightedConnect(i);
int nUnits = wtConnect.NumInputNodes;
int nConnect = wtConnect.NumOutputNodes;
// get the hidden layer activation unit details
nNet.GetLayerDetails(i - 1, ref unitType, ref slope, ref amplify);
// get the hidden layer activation unit input values
nNet.GetUnitInputs(unitInputs, i - 1);
// calculate the hidden layer errors
for (int j = 0; j < nUnits; j++)
{
double error = 0;
double xj = unitInputs[j];
for (int k = 0; k < nConnect; k++)
{
List <double> weights = new List <double>();
wtConnect.GetWeightVector(k, weights);
// follow the steepest path on the error function by moving along the gradient
// of the hidden layer units activation function - the gradient descent method
error += GetGradient(unitType, slope, amplify, xj) * prevErr[k] * weights[j];
}
layerErr.Add(error);
}
// update the hidden errors with the current layer error
// N.B. Since we start from the last hidden layer the
// hidden layer error signals are stored in reverse order
hidErr.Add(layerErr);
// back propagate the layer errors
prevErr.Clear();
prevErr = layerErr;
}
}
/// <summary>
/// Gets the details of the specified hidden layer.
/// </summary>
/// <param name="n">the specified hidden layer index</param>
/// <param name="unitType">the layer unit activation function type</param>
/// <param name="slope">the layer unit activation function slope value</param>
/// <param name="amplify">the layer unit activation function amplify value</param>
///
public void GetLayerDetails(int n, ref ActiveT unitType, ref double slope, ref double amplify)
{
if (n >= 0 && n < mNumLayers)
{
unitType = mActiveUnits[n];
slope = mActiveSlope[n];
amplify = mActiveAmplify[n];
}
}
/// <summary>
/// Constructs a neuron with the given activation function and settings.
/// </summary>
/// <param name="activationMode">the unit activation function type</param>
/// <param name="slope">the slope parameter value</param>
/// <param name="amplify">the amplify parameter value</param>
///
public NNetUnit(ActiveT activationMode, double slope, double amplify)
{
mActivationType = activationMode;
// ignore invalid values
if (slope > 0)
{
mSlope = slope;
}
// ignore invalid values
if (amplify > 0)
{
mAmplify = amplify;
}
}
/////////////////////////////////////////////////////////////////////
// Public Methods
/////////////////////////////////////////////////////////////////////
/// <summary>
/// Clears a NeuralNetwork object ready for re-use.
/// </summary>
///
public void ClearNeuralNetwork()
{
mNumInputs = 0;
NumOutputs = 0;
mNumLayers = 0;
mOutUnitType = ActiveT.kThreshold;
mOutUnitSlope = 1;
mOutUnitAmplify = 1;
mLayers.Clear();
mActivations.Clear();
mUnitInputs.Clear();
mActiveUnits.Clear();
mActiveSlope.Clear();
mActiveAmplify.Clear();
}
/// <summary>
/// Gets the gradient of the activation function at the given value of x.
/// </summary>
/// <param name="unitType">the activation function type</param>
/// <param name="slope">the activation function slope value</param>
/// <param name="amplify">the activation function amplify value</param>
/// <param name="x">calculates the gradient at this value</param>
/// <returns>the gradient of the activation function at the given value</returns>
///
private double GetGradient(ActiveT unitType, double slope, double amplify, double x)
{
double gradient = 0;
double expMX, expMX1, tanMX, absMX1, grad;
switch (unitType)
{
// 0 everywhere except the origin where the derivative is undefined!
case ActiveT.kThreshold:
// return the value of the slope parameter if x = 0
if (x == 0)
{
gradient = slope;
}
break;
case ActiveT.kUnipolar:
expMX = Math.Exp(-slope * x);
expMX1 = 1 + expMX;
gradient = (slope * expMX) / (expMX1 * expMX1);
break;
case ActiveT.kBipolar:
expMX = Math.Exp(-slope * x);
expMX1 = 1 + expMX;
gradient = (2 * slope * expMX) / (expMX1 * expMX1);
break;
case ActiveT.kTanh:
tanMX = Math.Tanh(slope * x);
gradient = slope * (1 - (tanMX * tanMX));
break;
case ActiveT.kGauss:
gradient = -2 * slope * x * Math.Exp(-slope * x * x);
break;
case ActiveT.kArctan:
gradient = slope / (1 + slope * slope * x * x);
break;
case ActiveT.kSin:
gradient = slope * Math.Cos(slope * x);
break;
case ActiveT.kCos:
gradient = -slope *Math.Sin(slope *x);
break;
case ActiveT.kSinC:
if (Math.Abs(x) < 0.00001)
{
gradient = 0;
}
else
{
gradient = (slope * x * Math.Cos(slope * x) - Math.Sin(slope * x)) / (slope * x * x);
}
break;
case ActiveT.kElliot:
absMX1 = 1 + Math.Abs(slope * x);
gradient = (0.5 * slope) / (absMX1 * absMX1);
break;
case ActiveT.kLinear:
gradient = slope;
break;
case ActiveT.kISRU:
grad = 1 / Math.Sqrt(1 + slope * x * x);
gradient = grad * grad * grad;
break;
case ActiveT.kSoftSign:
absMX1 = 1 + Math.Abs(slope * x);
gradient = slope / (absMX1 * absMX1);
break;
case ActiveT.kSoftPlus:
expMX = Math.Exp(slope * x);
gradient = (slope * expMX) / (1 + expMX);
break;
}
return(amplify * gradient);
}
/// <summary>
/// Instantiates this network from a string representation stored in a file.
/// </summary>
/// <param name="fname">the file containing a string representation of the network</param>
///
private void InitializeFromFile(string fname)
{
StreamReader inStream = new StreamReader(fname);
if (inStream != null)
{
int outUnitType;
// deserialize the main details
string sNumInputs = ReadString(inStream);
string sNumOutputs = ReadString(inStream);
string sNumLayers = ReadString(inStream);
string sOutUnitType = ReadString(inStream);
string sOutUnitSlope = ReadString(inStream);
string sOutUnitAmplify = ReadString(inStream);
mNumInputs = Convert.ToInt32(sNumInputs);
mNumOutputs = Convert.ToInt32(sNumOutputs);
mNumLayers = Convert.ToInt32(sNumLayers);
outUnitType = Convert.ToInt32(sOutUnitType);
mOutUnitSlope = Convert.ToDouble(sOutUnitSlope);
mOutUnitAmplify = Convert.ToDouble(sOutUnitAmplify);
mOutUnitType = (ActiveT)outUnitType;
// deserialize the layer data
for (int i = 0; i <= mNumLayers; i++) // use <= to include the output layer
{
string sDelim = ReadString(inStream);
string sNIn = ReadString(inStream);
string sNOut = ReadString(inStream);
string sNUnit = ReadString(inStream);
string sSUnit = ReadString(inStream);
string sAUnit = ReadString(inStream);
int nIn = Convert.ToInt32(sNIn);
int nOut = Convert.ToInt32(sNOut);
int nUnit = Convert.ToInt32(sNUnit);
double sUnit = Convert.ToDouble(sSUnit);
double aUnit = Convert.ToDouble(sAUnit);
NNetWeightedConnect connect = new NNetWeightedConnect(nIn, nOut);
for (int j = 0; j < nOut; j++)
{
List <double> weights = new List <double>();
for (int k = 0; k < nIn; k++)
{
string sWgt = ReadString(inStream);
double wgt = Convert.ToDouble(sWgt);
weights.Add(wgt);
}
connect.SetWeightVector(j, weights);
}
mLayers.Add(connect);
mActiveUnits.Add((ActiveT)nUnit);
mActiveSlope.Add(sUnit);
mActiveAmplify.Add(aUnit);
}
// tidy up
inStream.Close();
}
}
/// <summary>
/// Adds a new hidden layer.
///
/// The hidden layers are stored in the order of calls to this method
/// so the first call to AddLayer creates the first hidden layer, the
/// second call creates the second layer and so on.
/// </summary>
/// <param name="numUnits">the number of units in the hidden layer</param>
/// <param name="unitType">the layer unit activation function type (defaults to unipolar)</param>
/// <param name="initRange">the range of the initial weighted connection values (defaults to 2 coressponding to -1 to +1)</param>
/// <param name="slope">the layer unit activation function slope value (defaults to 1.0)</param>
/// <param name="amplify">the layer unit activation function amplify value (defaults to 1.0)</param>
/// <returns>0 if the layer is successfully added otherwise -1</returns>
///
public int AddLayer(int numUnits, ActiveT unitType = ActiveT.kUnipolar,
double initRange = 2, double slope = 1, double amplify = 1)
{
NNetWeightedConnect connect = new NNetWeightedConnect();
NNetWeightedConnect output = new NNetWeightedConnect();
// ignore invalid values
if (numUnits > 0 && initRange > 0 && slope > 0 && amplify > 0)
{
if (mNumLayers == 0)
{
// configure the first hidden layer
if (mNumInputs > 0)
{
// set up the weighted connections between the input and the first layer
// the weighted connections are initialised with random values in the
// range: -(initRange / 2) to +(initRange / 2)
connect.SetNumNodes(mNumInputs, numUnits, initRange);
// store the unit type for the layer
mActiveUnits.Add(unitType);
// store the steepness of the activation function's slope
mActiveSlope.Add(slope);
// store the amplification factor of the activation function
mActiveAmplify.Add(amplify);
mNumLayers++;
}
else
{
return(-1);
}
}
else
{
// configure subsequent hidden layers
if (mNumLayers > 0)
{
int nInputs = mLayers[mNumLayers - 1].NumOutputNodes;
// set up the weighted connections between the previous layer and the new one
// the weighted connections are initialised with random values in the
// range: -(initRange / 2) to +(initRange / 2)
connect.SetNumNodes(nInputs, numUnits, initRange);
// store the unit type for the layer
mActiveUnits.Add(unitType);
// store the steepness of the activation function's slope
mActiveSlope.Add(slope);
// store the amplification factor of the activation function
mActiveAmplify.Add(amplify);
mNumLayers++;
}
else
{
return(-1);
}
}
// connect the last hidden layer to the output layer
if (mNumLayers > 1)
{
// overwrite the old output connections
mLayers[mNumLayers - 1] = connect;
}
else
{
// add the connections for the first layer
mLayers.Add(connect);
}
// set up the weighted connections between the last layer and the output
output.SetNumNodes(numUnits, mNumOutputs, initRange);
// add the output connections
mLayers.Add(output);
}
else
{
return(-1);
}
return(0);
}
/// <summary>
/// Copies the contents of the source neural network into this neural network.
/// </summary>
/// <param name="src">the source net, to be copied</param>
///
public void CopyNeuralNet(NeuralNet src)
{
mLayers.Clear();
mActivations.Clear();
mUnitInputs.Clear();
mActiveUnits.Clear();
mActiveSlope.Clear();
mActiveAmplify.Clear();
mNumInputs = src.mNumInputs;
mNumOutputs = src.mNumOutputs;
mNumLayers = src.mNumLayers;
mOutUnitType = src.mOutUnitType;
mOutUnitSlope = src.mOutUnitSlope;
mOutUnitAmplify = src.mOutUnitAmplify;
// the weighted connections linking the network layers
for (int i = 0; i < src.mLayers.Count; i++)
{
NNetWeightedConnect wCnct = new NNetWeightedConnect(src.mLayers[i]);
mLayers.Add(wCnct);
}
// the activation values for each of the network layers
int rowLen = src.mActivations.Count;
for (int row = 0; row < rowLen; row++)
{
List <double> vec = new List <double>();
int colLen = src.mActivations[row].Count;
for (int col = 0; col < colLen; col++)
{
vec.Add(src.mActivations[row][col]);
}
mActivations.Add(vec);
}
// the input values for the layer activation functions
rowLen = src.mUnitInputs.Count;
for (int row = 0; row < rowLen; row++)
{
List <double> vec = new List <double>();
int colLen = src.mUnitInputs[row].Count;
for (int col = 0; col < colLen; col++)
{
vec.Add(src.mUnitInputs[row][col]);
}
mUnitInputs.Add(vec);
}
// the hidden layer unit activation function types
for (int i = 0; i < src.mActiveUnits.Count; i++)
{
mActiveUnits.Add(src.mActiveUnits[i]);
}
// the hidden layer unit activation function slope values
for (int i = 0; i < src.mActiveSlope.Count; i++)
{
mActiveSlope.Add(src.mActiveSlope[i]);
}
// the hidden layer unit activation function amplify values
for (int i = 0; i < src.mActiveAmplify.Count; i++)
{
mActiveAmplify.Add(src.mActiveAmplify[i]);
}
}
