/// <summary> /// Sets the weighted connections for a specified layer. /// /// This method is typically called by the training process to update the weighted connections. /// </summary> /// <param name="wtConnect">the weighted connections between the specified layer and the next sequential layer in the network</param> /// <param name="layer">the specified layer</param> /// public void SetWeightedConnect(NNetWeightedConnect wtConnect, int layer) { if (layer >= 0 && layer < mLayers.Count) { mLayers[layer] = wtConnect; } }
/// <summary> /// Calculates the weight adjustments for the connections into the output layer. /// </summary> /// <param name="outErr">the output unit errors</param> /// <param name="nNet">the network undergoing training</param> /// private void CalcOutputWtAdjust(List <double> outErr, NeuralNet nNet) { int n = nNet.NumLayers, prevIdx = 0; List <double> xVec = new List <double>(); // get the weighted connections between the last hidden layer and the output layer NNetWeightedConnect wtConnect = nNet.GetWeightedConnect(n); // get the input values for the weighted connections nNet.GetActivations(xVec, n - 1); int nOut = wtConnect.NumOutputNodes; // calculate the weight adjustments for each weighted connection output unit for (int i = 0; i < nOut; i++) { double ei = outErr[i]; List <double> weights = new List <double>(); // get the output units weight vector wtConnect.GetWeightVector(i, weights); // calculate the total weight adjustment for (int j = 0; j < xVec.Count; j++) { // the weight adjustment calculation double dW = mLearnConst * ei * xVec[j]; // if the momentum term is greater than 0 // the previous weighting needs to be taken into account if (mMomentum > 0) { if (mPrevOutWt.Count > prevIdx) { double dWPrev = mPrevOutWt[prevIdx]; // include a percentage of the previous weighting dW += mMomentum * dWPrev; // store the weighting mPrevOutWt[prevIdx] = dW; } else { // store the first weighting mPrevOutWt.Add(dW); } } // the total weight adjustment weights[j] += dW; prevIdx++; } wtConnect.SetWeightVector(i, weights); } nNet.SetWeightedConnect(wtConnect, n); }
/// <summary> /// Calculates the error signal on each individual unit within the networks hidden layers. /// /// Uses the gradient descent method to search the error surface. /// </summary> /// <param name="hidErr">the calculated hidden unit errors</param> /// <param name="outErr">the output unit errors</param> /// <param name="nNet">the network undergoing training</param> /// private void CalcHiddenError(List <List <double> > hidErr, List <double> outErr, NeuralNet nNet) { int nHidden = nNet.NumLayers; double slope = 1, amplify = 1; ActiveT unitType = ActiveT.kUnipolar; // initialise the the previous layer error with the output layer errors List <double> prevErr = new List <double>(); prevErr.AddRange(outErr); // start with the last hidden layer and work back to the first for (int i = nHidden; i >= 1; i--) { List <double> unitInputs = new List <double>(); List <double> layerErr = new List <double>(); // get the weighted connections for the current hidden layer NNetWeightedConnect wtConnect = nNet.GetWeightedConnect(i); int nUnits = wtConnect.NumInputNodes; int nConnect = wtConnect.NumOutputNodes; // get the hidden layer activation unit details nNet.GetLayerDetails(i - 1, ref unitType, ref slope, ref amplify); // get the hidden layer activation unit input values nNet.GetUnitInputs(unitInputs, i - 1); // calculate the hidden layer errors for (int j = 0; j < nUnits; j++) { double error = 0; double xj = unitInputs[j]; for (int k = 0; k < nConnect; k++) { List <double> weights = new List <double>(); wtConnect.GetWeightVector(k, weights); // follow the steepest path on the error function by moving along the gradient // of the hidden layer units activation function - the gradient descent method error += GetGradient(unitType, slope, amplify, xj) * prevErr[k] * weights[j]; } layerErr.Add(error); } // update the hidden errors with the current layer error // N.B. Since we start from the last hidden layer the // hidden layer error signals are stored in reverse order hidErr.Add(layerErr); // back propagate the layer errors prevErr.Clear(); prevErr = layerErr; } }
/// <summary> /// Gets the weighted connections for a specified layer. /// /// This method is typically called by the training process to access the weighted connections. /// </summary> /// <param name="layer">the specified layer</param> /// <returns>the weighted connections between the specified layer and the next sequential layer in the network</returns> /// public NNetWeightedConnect GetWeightedConnect(int layer) { NNetWeightedConnect wtConnect = null; if (layer >= 0 && layer < mLayers.Count) { wtConnect = new NNetWeightedConnect(mLayers[layer]); } return(wtConnect); }
/// <summary> /// Calculates the weight adjustments for the connections into the hidden layers. /// </summary> /// <param name="hidErrSig">the hidden unit errors</param> /// <param name="inputVec">the current training set input values</param> /// <param name="nNet">the network undergoing training</param> /// private void CalcHiddenWtAdjust(List <List <double> > hidErrSig, List <double> inputVec, NeuralNet nNet) { List <double> xVec = new List <double>(); int maxHidLayIdx = nNet.NumLayers - 1, prevIdx = 0; // calculate the weight adjustments for the hidden layers for (int n = maxHidLayIdx; n >= 0; n--) { // get the weighted connections between the current layer and the previous hidden layer NNetWeightedConnect wtConnect = nNet.GetWeightedConnect(n); // get the hidden unit errors for the previous hidden layer // N.B. the hidden error signals are stored in reverse order List <double> outErr = hidErrSig[maxHidLayIdx - n]; if (n == 0) { // we are dealing with the input layer xVec = inputVec; } else { // we are dealing with a hidden layer nNet.GetActivations(xVec, n - 1); } int nOut = wtConnect.NumOutputNodes; // calculate the weight adjustments for each weighted connection output unit for (int i = 0; i < nOut; i++) { double ei = outErr[i]; List <double> weights = new List <double>(); // get the output units weight vector wtConnect.GetWeightVector(i, weights); // calculate the total weight adjustment for (int j = 0; j < xVec.Count; j++) { // the weight adjustment calculation double dW = mLearnConst * ei * xVec[j]; // if the momentum term is greater than 0 // the previous weighting needs to be taken into account if (mMomentum > 0) { if (mPrevHidWt.Count > prevIdx) { double dWPrev = mPrevHidWt[prevIdx]; // include a percentage of the previous weighting dW += mMomentum * dWPrev; // store the weighting mPrevHidWt[prevIdx] = dW; } else { // store the first weighting mPrevHidWt.Add(dW); } } // the total weight adjustment weights[j] += dW; prevIdx++; } wtConnect.SetWeightVector(i, weights); } nNet.SetWeightedConnect(wtConnect, n); } }
/// <summary> /// Instantiates this network from a string representation stored in a file. /// </summary> /// <param name="fname">the file containing a string representation of the network</param> /// private void InitializeFromFile(string fname) { StreamReader inStream = new StreamReader(fname); if (inStream != null) { int outUnitType; // deserialize the main details string sNumInputs = ReadString(inStream); string sNumOutputs = ReadString(inStream); string sNumLayers = ReadString(inStream); string sOutUnitType = ReadString(inStream); string sOutUnitSlope = ReadString(inStream); string sOutUnitAmplify = ReadString(inStream); mNumInputs = Convert.ToInt32(sNumInputs); mNumOutputs = Convert.ToInt32(sNumOutputs); mNumLayers = Convert.ToInt32(sNumLayers); outUnitType = Convert.ToInt32(sOutUnitType); mOutUnitSlope = Convert.ToDouble(sOutUnitSlope); mOutUnitAmplify = Convert.ToDouble(sOutUnitAmplify); mOutUnitType = (ActiveT)outUnitType; // deserialize the layer data for (int i = 0; i <= mNumLayers; i++) // use <= to include the output layer { string sDelim = ReadString(inStream); string sNIn = ReadString(inStream); string sNOut = ReadString(inStream); string sNUnit = ReadString(inStream); string sSUnit = ReadString(inStream); string sAUnit = ReadString(inStream); int nIn = Convert.ToInt32(sNIn); int nOut = Convert.ToInt32(sNOut); int nUnit = Convert.ToInt32(sNUnit); double sUnit = Convert.ToDouble(sSUnit); double aUnit = Convert.ToDouble(sAUnit); NNetWeightedConnect connect = new NNetWeightedConnect(nIn, nOut); for (int j = 0; j < nOut; j++) { List <double> weights = new List <double>(); for (int k = 0; k < nIn; k++) { string sWgt = ReadString(inStream); double wgt = Convert.ToDouble(sWgt); weights.Add(wgt); } connect.SetWeightVector(j, weights); } mLayers.Add(connect); mActiveUnits.Add((ActiveT)nUnit); mActiveSlope.Add(sUnit); mActiveAmplify.Add(aUnit); } // tidy up inStream.Close(); } }
///////////////////////////////////////////////////////////////////// // Private Methods ///////////////////////////////////////////////////////////////////// /// <summary> /// Generates a string representation of this network. /// </summary> /// <returns>a string representation of this network</returns> /// private string Serialize() { List <double> weights = new List <double>(); StringBuilder outStr = new StringBuilder(); // serialize the main details int outUnitType = (int)mOutUnitType; // cast the output unit type enum to an int outStr.Append(mNumInputs.ToString()); outStr.Append(" "); outStr.Append(mNumOutputs.ToString()); outStr.Append(" "); outStr.Append(mNumLayers.ToString()); outStr.Append(" "); outStr.Append(outUnitType.ToString()); outStr.Append(" "); outStr.Append(mOutUnitSlope.ToString()); outStr.Append(" "); outStr.Append(mOutUnitAmplify.ToString()); outStr.Append(" "); // serialize the layer data for (int i = 0; i <= mNumLayers; i++) // use <= to include the output layer { NNetWeightedConnect connect = mLayers[i]; int nIn = connect.NumInputNodes; int nOut = connect.NumOutputNodes; int nUnit = 0; double sUnit = 0.0, aUnit = 0.0; // get the unit type, slope and amplification for the hidden layer if (i < mNumLayers) { nUnit = (int)mActiveUnits[i]; } if (i < mNumLayers) { sUnit = mActiveSlope[i]; } if (i < mNumLayers) { aUnit = mActiveAmplify[i]; } outStr.Append("L "); outStr.Append(nIn.ToString()); outStr.Append(" "); outStr.Append(nOut.ToString()); outStr.Append(" "); outStr.Append(nUnit.ToString()); outStr.Append(" "); outStr.Append(sUnit.ToString()); outStr.Append(" "); outStr.Append(aUnit.ToString()); outStr.Append(" "); for (int j = 0; j < nOut; j++) { connect.GetWeightVector(j, weights); for (int k = 0; k < nIn; k++) { outStr.Append(weights[k].ToString("G16")); outStr.Append(" "); } } } // terminate the output string outStr.AppendLine(); return(outStr.ToString()); }
/// <summary> /// Gets the response of the network to the given input. /// /// The number of elements in the inputs vector should correspond to /// the number of the input units. If the inputs vector contains /// more elements than this, the additional input values are ignored. /// </summary> /// <param name="inputs">the network input values</param> /// <param name="outputs">the network output values</param> /// public void GetResponse(List <double> inputs, List <double> outputs) { if (inputs.Count >= mNumInputs && mNumLayers > 0) { List <double> inputVec = new List <double>(); List <double> outputVec = new List <double>(); NNetWeightedConnect connect = new NNetWeightedConnect(); // clear any old activation and unit input values mActivations.Clear(); mUnitInputs.Clear(); // 'load' the input vector for (int i = 0; i < mNumInputs; i++) { inputVec.Add(inputs[i]); } // get the weighted connections between the input layer and first layer connect = mLayers[0]; // apply the weighted connections connect.SetInputs(inputVec); connect.GetOutputs(outputVec); // store the output vector - this contains the unit input values mUnitInputs.Add(outputVec); // clear the input vector so it can be used to hold the input for the next layer inputVec.Clear(); // set the unit type, slope and amplification for the first layer NNetUnit unit = new NNetUnit(mActiveUnits[0], mActiveSlope[0], mActiveAmplify[0]); // activate the net units for (int i = 0; i < (int)outputVec.Count; i++) { unit.Input = outputVec[i]; inputVec.Add(unit.GetActivation()); } // store the activations mActivations.Add(inputVec); // propagate the data through the remaining layers for (int i = 1; i <= mNumLayers; i++) // use <= to include the output layer { // get the weighted connections linking the next layer connect = mLayers[i]; // apply the weighted connections outputVec = new List <double>(); connect.SetInputs(inputVec); connect.GetOutputs(outputVec); inputVec = new List <double>(); // store the output vector - this contains the unit input values mUnitInputs.Add(outputVec); if (i < mNumLayers) { // set the unit type, slope and amplification for the next hidden layer unit.ActivationType = mActiveUnits[i]; unit.Slope = mActiveSlope[i]; unit.Amplify = mActiveAmplify[i]; } else { // set the unit type, slope and amplification for the output layer unit.ActivationType = mOutUnitType; unit.Slope = mOutUnitSlope; unit.Amplify = mOutUnitAmplify; } // activate the net units for (int j = 0; j < (int)outputVec.Count; j++) { unit.Input = outputVec[j]; inputVec.Add(unit.GetActivation()); } // store the activations mActivations.Add(inputVec); } // copy the results into the output vector outputs.Clear(); outputs.AddRange(inputVec); } }
/// <summary> /// Adds a new hidden layer. /// /// The hidden layers are stored in the order of calls to this method /// so the first call to AddLayer creates the first hidden layer, the /// second call creates the second layer and so on. /// </summary> /// <param name="numUnits">the number of units in the hidden layer</param> /// <param name="unitType">the layer unit activation function type (defaults to unipolar)</param> /// <param name="initRange">the range of the initial weighted connection values (defaults to 2 coressponding to -1 to +1)</param> /// <param name="slope">the layer unit activation function slope value (defaults to 1.0)</param> /// <param name="amplify">the layer unit activation function amplify value (defaults to 1.0)</param> /// <returns>0 if the layer is successfully added otherwise -1</returns> /// public int AddLayer(int numUnits, ActiveT unitType = ActiveT.kUnipolar, double initRange = 2, double slope = 1, double amplify = 1) { NNetWeightedConnect connect = new NNetWeightedConnect(); NNetWeightedConnect output = new NNetWeightedConnect(); // ignore invalid values if (numUnits > 0 && initRange > 0 && slope > 0 && amplify > 0) { if (mNumLayers == 0) { // configure the first hidden layer if (mNumInputs > 0) { // set up the weighted connections between the input and the first layer // the weighted connections are initialised with random values in the // range: -(initRange / 2) to +(initRange / 2) connect.SetNumNodes(mNumInputs, numUnits, initRange); // store the unit type for the layer mActiveUnits.Add(unitType); // store the steepness of the activation function's slope mActiveSlope.Add(slope); // store the amplification factor of the activation function mActiveAmplify.Add(amplify); mNumLayers++; } else { return(-1); } } else { // configure subsequent hidden layers if (mNumLayers > 0) { int nInputs = mLayers[mNumLayers - 1].NumOutputNodes; // set up the weighted connections between the previous layer and the new one // the weighted connections are initialised with random values in the // range: -(initRange / 2) to +(initRange / 2) connect.SetNumNodes(nInputs, numUnits, initRange); // store the unit type for the layer mActiveUnits.Add(unitType); // store the steepness of the activation function's slope mActiveSlope.Add(slope); // store the amplification factor of the activation function mActiveAmplify.Add(amplify); mNumLayers++; } else { return(-1); } } // connect the last hidden layer to the output layer if (mNumLayers > 1) { // overwrite the old output connections mLayers[mNumLayers - 1] = connect; } else { // add the connections for the first layer mLayers.Add(connect); } // set up the weighted connections between the last layer and the output output.SetNumNodes(numUnits, mNumOutputs, initRange); // add the output connections mLayers.Add(output); } else { return(-1); } return(0); }
/// <summary> /// Copies the contents of the source neural network into this neural network. /// </summary> /// <param name="src">the source net, to be copied</param> /// public void CopyNeuralNet(NeuralNet src) { mLayers.Clear(); mActivations.Clear(); mUnitInputs.Clear(); mActiveUnits.Clear(); mActiveSlope.Clear(); mActiveAmplify.Clear(); mNumInputs = src.mNumInputs; mNumOutputs = src.mNumOutputs; mNumLayers = src.mNumLayers; mOutUnitType = src.mOutUnitType; mOutUnitSlope = src.mOutUnitSlope; mOutUnitAmplify = src.mOutUnitAmplify; // the weighted connections linking the network layers for (int i = 0; i < src.mLayers.Count; i++) { NNetWeightedConnect wCnct = new NNetWeightedConnect(src.mLayers[i]); mLayers.Add(wCnct); } // the activation values for each of the network layers int rowLen = src.mActivations.Count; for (int row = 0; row < rowLen; row++) { List <double> vec = new List <double>(); int colLen = src.mActivations[row].Count; for (int col = 0; col < colLen; col++) { vec.Add(src.mActivations[row][col]); } mActivations.Add(vec); } // the input values for the layer activation functions rowLen = src.mUnitInputs.Count; for (int row = 0; row < rowLen; row++) { List <double> vec = new List <double>(); int colLen = src.mUnitInputs[row].Count; for (int col = 0; col < colLen; col++) { vec.Add(src.mUnitInputs[row][col]); } mUnitInputs.Add(vec); } // the hidden layer unit activation function types for (int i = 0; i < src.mActiveUnits.Count; i++) { mActiveUnits.Add(src.mActiveUnits[i]); } // the hidden layer unit activation function slope values for (int i = 0; i < src.mActiveSlope.Count; i++) { mActiveSlope.Add(src.mActiveSlope[i]); } // the hidden layer unit activation function amplify values for (int i = 0; i < src.mActiveAmplify.Count; i++) { mActiveAmplify.Add(src.mActiveAmplify[i]); } }