/// <summary>
/// Sets the weighted connections for a specified layer.
///
/// This method is typically called by the training process to update the weighted connections.
/// </summary>
/// <param name="wtConnect">the weighted connections between the specified layer and the next sequential layer in the network</param>
/// <param name="layer">the specified layer</param>
///
public void SetWeightedConnect(NNetWeightedConnect wtConnect, int layer)
{
if (layer >= 0 && layer < mLayers.Count)
{
mLayers[layer] = wtConnect;
}
}
/// <summary>
/// Calculates the weight adjustments for the connections into the output layer.
/// </summary>
/// <param name="outErr">the output unit errors</param>
/// <param name="nNet">the network undergoing training</param>
///
private void CalcOutputWtAdjust(List <double> outErr, NeuralNet nNet)
{
int n = nNet.NumLayers, prevIdx = 0;
List <double> xVec = new List <double>();
// get the weighted connections between the last hidden layer and the output layer
NNetWeightedConnect wtConnect = nNet.GetWeightedConnect(n);
// get the input values for the weighted connections
nNet.GetActivations(xVec, n - 1);
int nOut = wtConnect.NumOutputNodes;
// calculate the weight adjustments for each weighted connection output unit
for (int i = 0; i < nOut; i++)
{
double ei = outErr[i];
List <double> weights = new List <double>();
// get the output units weight vector
wtConnect.GetWeightVector(i, weights);
// calculate the total weight adjustment
for (int j = 0; j < xVec.Count; j++)
{
// the weight adjustment calculation
double dW = mLearnConst * ei * xVec[j];
// if the momentum term is greater than 0
// the previous weighting needs to be taken into account
if (mMomentum > 0)
{
if (mPrevOutWt.Count > prevIdx)
{
double dWPrev = mPrevOutWt[prevIdx];
// include a percentage of the previous weighting
dW += mMomentum * dWPrev;
// store the weighting
mPrevOutWt[prevIdx] = dW;
}
else
{
// store the first weighting
mPrevOutWt.Add(dW);
}
}
// the total weight adjustment
weights[j] += dW;
prevIdx++;
}
wtConnect.SetWeightVector(i, weights);
}
nNet.SetWeightedConnect(wtConnect, n);
}
/// <summary>
/// Calculates the error signal on each individual unit within the networks hidden layers.
///
/// Uses the gradient descent method to search the error surface.
/// </summary>
/// <param name="hidErr">the calculated hidden unit errors</param>
/// <param name="outErr">the output unit errors</param>
/// <param name="nNet">the network undergoing training</param>
///
private void CalcHiddenError(List <List <double> > hidErr,
List <double> outErr, NeuralNet nNet)
{
int nHidden = nNet.NumLayers;
double slope = 1, amplify = 1;
ActiveT unitType = ActiveT.kUnipolar;
// initialise the the previous layer error with the output layer errors
List <double> prevErr = new List <double>();
prevErr.AddRange(outErr);
// start with the last hidden layer and work back to the first
for (int i = nHidden; i >= 1; i--)
{
List <double> unitInputs = new List <double>();
List <double> layerErr = new List <double>();
// get the weighted connections for the current hidden layer
NNetWeightedConnect wtConnect = nNet.GetWeightedConnect(i);
int nUnits = wtConnect.NumInputNodes;
int nConnect = wtConnect.NumOutputNodes;
// get the hidden layer activation unit details
nNet.GetLayerDetails(i - 1, ref unitType, ref slope, ref amplify);
// get the hidden layer activation unit input values
nNet.GetUnitInputs(unitInputs, i - 1);
// calculate the hidden layer errors
for (int j = 0; j < nUnits; j++)
{
double error = 0;
double xj = unitInputs[j];
for (int k = 0; k < nConnect; k++)
{
List <double> weights = new List <double>();
wtConnect.GetWeightVector(k, weights);
// follow the steepest path on the error function by moving along the gradient
// of the hidden layer units activation function - the gradient descent method
error += GetGradient(unitType, slope, amplify, xj) * prevErr[k] * weights[j];
}
layerErr.Add(error);
}
// update the hidden errors with the current layer error
// N.B. Since we start from the last hidden layer the
// hidden layer error signals are stored in reverse order
hidErr.Add(layerErr);
// back propagate the layer errors
prevErr.Clear();
prevErr = layerErr;
}
}
/// <summary>
/// Gets the weighted connections for a specified layer.
///
/// This method is typically called by the training process to access the weighted connections.
/// </summary>
/// <param name="layer">the specified layer</param>
/// <returns>the weighted connections between the specified layer and the next sequential layer in the network</returns>
///
public NNetWeightedConnect GetWeightedConnect(int layer)
{
NNetWeightedConnect wtConnect = null;
if (layer >= 0 && layer < mLayers.Count)
{
wtConnect = new NNetWeightedConnect(mLayers[layer]);
}
return(wtConnect);
}
/// <summary>
/// Calculates the weight adjustments for the connections into the hidden layers.
/// </summary>
/// <param name="hidErrSig">the hidden unit errors</param>
/// <param name="inputVec">the current training set input values</param>
/// <param name="nNet">the network undergoing training</param>
///
private void CalcHiddenWtAdjust(List <List <double> > hidErrSig,
List <double> inputVec, NeuralNet nNet)
{
List <double> xVec = new List <double>();
int maxHidLayIdx = nNet.NumLayers - 1, prevIdx = 0;
// calculate the weight adjustments for the hidden layers
for (int n = maxHidLayIdx; n >= 0; n--)
{
// get the weighted connections between the current layer and the previous hidden layer
NNetWeightedConnect wtConnect = nNet.GetWeightedConnect(n);
// get the hidden unit errors for the previous hidden layer
// N.B. the hidden error signals are stored in reverse order
List <double> outErr = hidErrSig[maxHidLayIdx - n];
if (n == 0)
{
// we are dealing with the input layer
xVec = inputVec;
}
else
{
// we are dealing with a hidden layer
nNet.GetActivations(xVec, n - 1);
}
int nOut = wtConnect.NumOutputNodes;
// calculate the weight adjustments for each weighted connection output unit
for (int i = 0; i < nOut; i++)
{
double ei = outErr[i];
List <double> weights = new List <double>();
// get the output units weight vector
wtConnect.GetWeightVector(i, weights);
// calculate the total weight adjustment
for (int j = 0; j < xVec.Count; j++)
{
// the weight adjustment calculation
double dW = mLearnConst * ei * xVec[j];
// if the momentum term is greater than 0
// the previous weighting needs to be taken into account
if (mMomentum > 0)
{
if (mPrevHidWt.Count > prevIdx)
{
double dWPrev = mPrevHidWt[prevIdx];
// include a percentage of the previous weighting
dW += mMomentum * dWPrev;
// store the weighting
mPrevHidWt[prevIdx] = dW;
}
else
{
// store the first weighting
mPrevHidWt.Add(dW);
}
}
// the total weight adjustment
weights[j] += dW;
prevIdx++;
}
wtConnect.SetWeightVector(i, weights);
}
nNet.SetWeightedConnect(wtConnect, n);
}
}
/// <summary>
/// Instantiates this network from a string representation stored in a file.
/// </summary>
/// <param name="fname">the file containing a string representation of the network</param>
///
private void InitializeFromFile(string fname)
{
StreamReader inStream = new StreamReader(fname);
if (inStream != null)
{
int outUnitType;
// deserialize the main details
string sNumInputs = ReadString(inStream);
string sNumOutputs = ReadString(inStream);
string sNumLayers = ReadString(inStream);
string sOutUnitType = ReadString(inStream);
string sOutUnitSlope = ReadString(inStream);
string sOutUnitAmplify = ReadString(inStream);
mNumInputs = Convert.ToInt32(sNumInputs);
mNumOutputs = Convert.ToInt32(sNumOutputs);
mNumLayers = Convert.ToInt32(sNumLayers);
outUnitType = Convert.ToInt32(sOutUnitType);
mOutUnitSlope = Convert.ToDouble(sOutUnitSlope);
mOutUnitAmplify = Convert.ToDouble(sOutUnitAmplify);
mOutUnitType = (ActiveT)outUnitType;
// deserialize the layer data
for (int i = 0; i <= mNumLayers; i++) // use <= to include the output layer
{
string sDelim = ReadString(inStream);
string sNIn = ReadString(inStream);
string sNOut = ReadString(inStream);
string sNUnit = ReadString(inStream);
string sSUnit = ReadString(inStream);
string sAUnit = ReadString(inStream);
int nIn = Convert.ToInt32(sNIn);
int nOut = Convert.ToInt32(sNOut);
int nUnit = Convert.ToInt32(sNUnit);
double sUnit = Convert.ToDouble(sSUnit);
double aUnit = Convert.ToDouble(sAUnit);
NNetWeightedConnect connect = new NNetWeightedConnect(nIn, nOut);
for (int j = 0; j < nOut; j++)
{
List <double> weights = new List <double>();
for (int k = 0; k < nIn; k++)
{
string sWgt = ReadString(inStream);
double wgt = Convert.ToDouble(sWgt);
weights.Add(wgt);
}
connect.SetWeightVector(j, weights);
}
mLayers.Add(connect);
mActiveUnits.Add((ActiveT)nUnit);
mActiveSlope.Add(sUnit);
mActiveAmplify.Add(aUnit);
}
// tidy up
inStream.Close();
}
}
/////////////////////////////////////////////////////////////////////
// Private Methods
/////////////////////////////////////////////////////////////////////
/// <summary>
/// Generates a string representation of this network.
/// </summary>
/// <returns>a string representation of this network</returns>
///
private string Serialize()
{
List <double> weights = new List <double>();
StringBuilder outStr = new StringBuilder();
// serialize the main details
int outUnitType = (int)mOutUnitType; // cast the output unit type enum to an int
outStr.Append(mNumInputs.ToString());
outStr.Append(" ");
outStr.Append(mNumOutputs.ToString());
outStr.Append(" ");
outStr.Append(mNumLayers.ToString());
outStr.Append(" ");
outStr.Append(outUnitType.ToString());
outStr.Append(" ");
outStr.Append(mOutUnitSlope.ToString());
outStr.Append(" ");
outStr.Append(mOutUnitAmplify.ToString());
outStr.Append(" ");
// serialize the layer data
for (int i = 0; i <= mNumLayers; i++) // use <= to include the output layer
{
NNetWeightedConnect connect = mLayers[i];
int nIn = connect.NumInputNodes;
int nOut = connect.NumOutputNodes;
int nUnit = 0;
double sUnit = 0.0, aUnit = 0.0;
// get the unit type, slope and amplification for the hidden layer
if (i < mNumLayers)
{
nUnit = (int)mActiveUnits[i];
}
if (i < mNumLayers)
{
sUnit = mActiveSlope[i];
}
if (i < mNumLayers)
{
aUnit = mActiveAmplify[i];
}
outStr.Append("L ");
outStr.Append(nIn.ToString());
outStr.Append(" ");
outStr.Append(nOut.ToString());
outStr.Append(" ");
outStr.Append(nUnit.ToString());
outStr.Append(" ");
outStr.Append(sUnit.ToString());
outStr.Append(" ");
outStr.Append(aUnit.ToString());
outStr.Append(" ");
for (int j = 0; j < nOut; j++)
{
connect.GetWeightVector(j, weights);
for (int k = 0; k < nIn; k++)
{
outStr.Append(weights[k].ToString("G16"));
outStr.Append(" ");
}
}
}
// terminate the output string
outStr.AppendLine();
return(outStr.ToString());
}
/// <summary>
/// Gets the response of the network to the given input.
///
/// The number of elements in the inputs vector should correspond to
/// the number of the input units. If the inputs vector contains
/// more elements than this, the additional input values are ignored.
/// </summary>
/// <param name="inputs">the network input values</param>
/// <param name="outputs">the network output values</param>
///
public void GetResponse(List <double> inputs, List <double> outputs)
{
if (inputs.Count >= mNumInputs && mNumLayers > 0)
{
List <double> inputVec = new List <double>();
List <double> outputVec = new List <double>();
NNetWeightedConnect connect = new NNetWeightedConnect();
// clear any old activation and unit input values
mActivations.Clear();
mUnitInputs.Clear();
// 'load' the input vector
for (int i = 0; i < mNumInputs; i++)
{
inputVec.Add(inputs[i]);
}
// get the weighted connections between the input layer and first layer
connect = mLayers[0];
// apply the weighted connections
connect.SetInputs(inputVec);
connect.GetOutputs(outputVec);
// store the output vector - this contains the unit input values
mUnitInputs.Add(outputVec);
// clear the input vector so it can be used to hold the input for the next layer
inputVec.Clear();
// set the unit type, slope and amplification for the first layer
NNetUnit unit = new NNetUnit(mActiveUnits[0], mActiveSlope[0], mActiveAmplify[0]);
// activate the net units
for (int i = 0; i < (int)outputVec.Count; i++)
{
unit.Input = outputVec[i];
inputVec.Add(unit.GetActivation());
}
// store the activations
mActivations.Add(inputVec);
// propagate the data through the remaining layers
for (int i = 1; i <= mNumLayers; i++) // use <= to include the output layer
{
// get the weighted connections linking the next layer
connect = mLayers[i];
// apply the weighted connections
outputVec = new List <double>();
connect.SetInputs(inputVec);
connect.GetOutputs(outputVec);
inputVec = new List <double>();
// store the output vector - this contains the unit input values
mUnitInputs.Add(outputVec);
if (i < mNumLayers)
{
// set the unit type, slope and amplification for the next hidden layer
unit.ActivationType = mActiveUnits[i];
unit.Slope = mActiveSlope[i];
unit.Amplify = mActiveAmplify[i];
}
else
{
// set the unit type, slope and amplification for the output layer
unit.ActivationType = mOutUnitType;
unit.Slope = mOutUnitSlope;
unit.Amplify = mOutUnitAmplify;
}
// activate the net units
for (int j = 0; j < (int)outputVec.Count; j++)
{
unit.Input = outputVec[j];
inputVec.Add(unit.GetActivation());
}
// store the activations
mActivations.Add(inputVec);
}
// copy the results into the output vector
outputs.Clear();
outputs.AddRange(inputVec);
}
}
/// <summary>
/// Adds a new hidden layer.
///
/// The hidden layers are stored in the order of calls to this method
/// so the first call to AddLayer creates the first hidden layer, the
/// second call creates the second layer and so on.
/// </summary>
/// <param name="numUnits">the number of units in the hidden layer</param>
/// <param name="unitType">the layer unit activation function type (defaults to unipolar)</param>
/// <param name="initRange">the range of the initial weighted connection values (defaults to 2 coressponding to -1 to +1)</param>
/// <param name="slope">the layer unit activation function slope value (defaults to 1.0)</param>
/// <param name="amplify">the layer unit activation function amplify value (defaults to 1.0)</param>
/// <returns>0 if the layer is successfully added otherwise -1</returns>
///
public int AddLayer(int numUnits, ActiveT unitType = ActiveT.kUnipolar,
double initRange = 2, double slope = 1, double amplify = 1)
{
NNetWeightedConnect connect = new NNetWeightedConnect();
NNetWeightedConnect output = new NNetWeightedConnect();
// ignore invalid values
if (numUnits > 0 && initRange > 0 && slope > 0 && amplify > 0)
{
if (mNumLayers == 0)
{
// configure the first hidden layer
if (mNumInputs > 0)
{
// set up the weighted connections between the input and the first layer
// the weighted connections are initialised with random values in the
// range: -(initRange / 2) to +(initRange / 2)
connect.SetNumNodes(mNumInputs, numUnits, initRange);
// store the unit type for the layer
mActiveUnits.Add(unitType);
// store the steepness of the activation function's slope
mActiveSlope.Add(slope);
// store the amplification factor of the activation function
mActiveAmplify.Add(amplify);
mNumLayers++;
}
else
{
return(-1);
}
}
else
{
// configure subsequent hidden layers
if (mNumLayers > 0)
{
int nInputs = mLayers[mNumLayers - 1].NumOutputNodes;
// set up the weighted connections between the previous layer and the new one
// the weighted connections are initialised with random values in the
// range: -(initRange / 2) to +(initRange / 2)
connect.SetNumNodes(nInputs, numUnits, initRange);
// store the unit type for the layer
mActiveUnits.Add(unitType);
// store the steepness of the activation function's slope
mActiveSlope.Add(slope);
// store the amplification factor of the activation function
mActiveAmplify.Add(amplify);
mNumLayers++;
}
else
{
return(-1);
}
}
// connect the last hidden layer to the output layer
if (mNumLayers > 1)
{
// overwrite the old output connections
mLayers[mNumLayers - 1] = connect;
}
else
{
// add the connections for the first layer
mLayers.Add(connect);
}
// set up the weighted connections between the last layer and the output
output.SetNumNodes(numUnits, mNumOutputs, initRange);
// add the output connections
mLayers.Add(output);
}
else
{
return(-1);
}
return(0);
}
/// <summary>
/// Copies the contents of the source neural network into this neural network.
/// </summary>
/// <param name="src">the source net, to be copied</param>
///
public void CopyNeuralNet(NeuralNet src)
{
mLayers.Clear();
mActivations.Clear();
mUnitInputs.Clear();
mActiveUnits.Clear();
mActiveSlope.Clear();
mActiveAmplify.Clear();
mNumInputs = src.mNumInputs;
mNumOutputs = src.mNumOutputs;
mNumLayers = src.mNumLayers;
mOutUnitType = src.mOutUnitType;
mOutUnitSlope = src.mOutUnitSlope;
mOutUnitAmplify = src.mOutUnitAmplify;
// the weighted connections linking the network layers
for (int i = 0; i < src.mLayers.Count; i++)
{
NNetWeightedConnect wCnct = new NNetWeightedConnect(src.mLayers[i]);
mLayers.Add(wCnct);
}
// the activation values for each of the network layers
int rowLen = src.mActivations.Count;
for (int row = 0; row < rowLen; row++)
{
List <double> vec = new List <double>();
int colLen = src.mActivations[row].Count;
for (int col = 0; col < colLen; col++)
{
vec.Add(src.mActivations[row][col]);
}
mActivations.Add(vec);
}
// the input values for the layer activation functions
rowLen = src.mUnitInputs.Count;
for (int row = 0; row < rowLen; row++)
{
List <double> vec = new List <double>();
int colLen = src.mUnitInputs[row].Count;
for (int col = 0; col < colLen; col++)
{
vec.Add(src.mUnitInputs[row][col]);
}
mUnitInputs.Add(vec);
}
// the hidden layer unit activation function types
for (int i = 0; i < src.mActiveUnits.Count; i++)
{
mActiveUnits.Add(src.mActiveUnits[i]);
}
// the hidden layer unit activation function slope values
for (int i = 0; i < src.mActiveSlope.Count; i++)
{
mActiveSlope.Add(src.mActiveSlope[i]);
}
// the hidden layer unit activation function amplify values
for (int i = 0; i < src.mActiveAmplify.Count; i++)
{
mActiveAmplify.Add(src.mActiveAmplify[i]);
}
}
