/// <summary>
/// Randomize the connections between two layers.
/// </summary>
/// <param name="network">The network to randomize.</param>
/// <param name="fromLayer">The starting layer.</param>
private void RandomizeSynapse(BasicNetwork network, int fromLayer)
{
int toLayer = fromLayer + 1;
int toCount = network.GetLayerNeuronCount(toLayer);
int fromCount = network.GetLayerNeuronCount(fromLayer);
int fromCountTotalCount = network.GetLayerTotalNeuronCount(fromLayer);
IActivationFunction af = network.GetActivation(toLayer);
double low = CalculateRange(af, Double.NegativeInfinity);
double high = CalculateRange(af, Double.PositiveInfinity);
double b = 0.7d * Math.Pow(toCount, (1d / fromCount)) / (high - low);
for (int toNeuron = 0; toNeuron < toCount; toNeuron++)
{
if (fromCount != fromCountTotalCount)
{
double w = RangeRandomizer.Randomize(-b, b);
network.SetWeight(fromLayer, fromCount, toNeuron, w);
}
for (int fromNeuron = 0; fromNeuron < fromCount; fromNeuron++)
{
double w = RangeRandomizer.Randomize(0, b);
network.SetWeight(fromLayer, fromNeuron, toNeuron, w);
}
}
}
/// <summary>
/// Determine the significance of the neuron. The higher the return value,
/// the more significant the neuron is.
/// </summary>
///
/// <param name="layer">The layer to query.</param>
/// <param name="neuron">The neuron to query.</param>
/// <returns>How significant is this neuron.</returns>
public double DetermineNeuronSignificance(int layer,
int neuron)
{
_network.ValidateNeuron(layer, neuron);
// calculate the bias significance
double result = 0;
// calculate the inbound significance
if (layer > 0)
{
int prevLayer = layer - 1;
int prevCount = _network
.GetLayerTotalNeuronCount(prevLayer);
for (int i = 0; i < prevCount; i++)
{
result += _network.GetWeight(prevLayer, i, neuron);
}
}
// calculate the outbound significance
if (layer < _network.LayerCount - 1)
{
int nextLayer = layer + 1;
int nextCount = _network.GetLayerNeuronCount(nextLayer);
for (int i = 0; i < nextCount; i++)
{
result += _network.GetWeight(layer, neuron, i);
}
}
return(Math.Abs(result));
}
/// <summary>
/// Randomize one level of a neural network.
/// </summary>
///
/// <param name="network">The network to randomize</param>
/// <param name="fromLayer">The from level to randomize.</param>
public override void Randomize(BasicNetwork network, int fromLayer)
{
int fromCount = network.GetLayerTotalNeuronCount(fromLayer);
int toCount = network.GetLayerNeuronCount(fromLayer + 1);
for (int fromNeuron = 0; fromNeuron < fromCount; fromNeuron++)
{
for (int toNeuron = 0; toNeuron < toCount; toNeuron++)
{
double v = CalculateValue(toCount);
network.SetWeight(fromLayer, fromNeuron, toNeuron, v);
}
}
}
/// <summary>
/// The Xaiver initialization works layer by layer.
/// </summary>
/// <param name="network">The network.</param>
/// <param name="fromLayer">The source layer.</param>
private void RandomizeLayer(BasicNetwork network, int fromLayer)
{
var fromCount = network.GetLayerTotalNeuronCount(fromLayer);
var toCount = network.Layers[fromLayer + 1].Count;
for (var fromNeuron = 0; fromNeuron < fromCount; fromNeuron++)
{
for (var toNeuron = 0; toNeuron < toCount; toNeuron++)
{
var sigma = Math.Sqrt(2.0 / (fromCount + toCount));
var w = Rnd.NextGaussian() * sigma;
network.SetWeight(fromLayer, fromNeuron, toNeuron, w);
}
}
}
/// <summary>
/// Randomize one level of a neural network.
/// </summary>
///
/// <param name="network">The network to randomize</param>
/// <param name="fromLayer">The from level to randomize.</param>
public virtual void Randomize(BasicNetwork network, int fromLayer)
{
int fromCount = network.GetLayerTotalNeuronCount(fromLayer);
int toCount = network.GetLayerNeuronCount(fromLayer + 1);
for (int fromNeuron = 0; fromNeuron < fromCount; fromNeuron++)
{
for (int toNeuron = 0; toNeuron < toCount; toNeuron++)
{
double v = network.GetWeight(fromLayer, fromNeuron, toNeuron);
v = Randomize(v);
network.SetWeight(fromLayer, fromNeuron, toNeuron, v);
}
}
}
/// <summary>
/// Apply a regularization penalty, such as that from L1/L2 regularization.
/// </summary>
/// <param name="fromLayer">The from layer.</param>
/// <param name="l">The penalty.</param>
public void LayerRegularizationPenalty(int fromLayer, double[] l)
{
var fromCount = _network.GetLayerTotalNeuronCount(fromLayer);
var toCount = _network.Layers[fromLayer + 1].Count;
for (var fromNeuron = 0; fromNeuron < fromCount; fromNeuron++)
{
for (var toNeuron = 0; toNeuron < toCount; toNeuron++)
{
var w = _network.GetWeight(fromLayer, fromNeuron, toNeuron);
l[0] += Math.Abs(w);
l[1] += w * w;
}
}
}
/// <summary>
/// The Xaiver initialization works layer by layer.
/// </summary>
/// <param name="network">The network.</param>
/// <param name="fromLayer">The source layer.</param>
private void RandomizeLayer(BasicNetwork network, int fromLayer)
{
var fromCount = network.GetLayerTotalNeuronCount(fromLayer);
var toCount = network.Layers[fromLayer + 1].Count;
for (var fromNeuron = 0; fromNeuron < fromCount; fromNeuron++)
{
for (var toNeuron = 0; toNeuron < toCount; toNeuron++)
{
var sigma = Math.Sqrt(2.0/(fromCount + toCount));
var w = Rnd.NextGaussian()*sigma;
network.SetWeight(fromLayer, fromNeuron, toNeuron, w);
}
}
}
/// <summary>
/// Randomize one level of a neural network.
/// </summary>
///
/// <param name="network">The network to randomize</param>
/// <param name="fromLayer">The from level to randomize.</param>
public override void Randomize(BasicNetwork network, int fromLayer)
{
int fromCount = network.GetLayerTotalNeuronCount(fromLayer);
int toCount = network.GetLayerNeuronCount(fromLayer + 1);
for (int toNeuron = 0; toNeuron < toCount; toNeuron++)
{
double n = 0.0;
for (int fromNeuron = 0; fromNeuron < fromCount; fromNeuron++)
{
double w = network.GetWeight(fromLayer, fromNeuron, toNeuron);
n += w * w;
}
n = Math.Sqrt(n);
for (int fromNeuron = 0; fromNeuron < fromCount; fromNeuron++)
{
double w = network.GetWeight(fromLayer, fromNeuron, toNeuron);
w = _beta * w / n;
network.SetWeight(fromLayer, fromNeuron, toNeuron, w);
}
}
}
/// <summary>
/// Construct a network analyze class. Analyze the specified network.
/// </summary>
///
/// <param name="network">The network to analyze.</param>
public AnalyzeNetwork(BasicNetwork network)
{
int assignDisabled = 0;
int assignedTotal = 0;
IList<Double> biasList = new List<Double>();
IList<Double> weightList = new List<Double>();
IList<Double> allList = new List<Double>();
for (int layerNumber = 0; layerNumber < network.LayerCount - 1; layerNumber++)
{
int fromCount = network.GetLayerNeuronCount(layerNumber);
int fromBiasCount = network
.GetLayerTotalNeuronCount(layerNumber);
int toCount = network.GetLayerNeuronCount(layerNumber + 1);
// weights
for (int fromNeuron = 0; fromNeuron < fromCount; fromNeuron++)
{
for (int toNeuron = 0; toNeuron < toCount; toNeuron++)
{
double v = network.GetWeight(layerNumber, fromNeuron,
toNeuron);
if (network.Structure.ConnectionLimited )
{
if (Math.Abs(v) < network.Structure.ConnectionLimit )
{
assignDisabled++;
}
}
weightList.Add(v);
allList.Add(v);
assignedTotal++;
}
}
// bias
if (fromCount != fromBiasCount)
{
int biasNeuron = fromCount;
for (int toNeuron = 0; toNeuron < toCount; toNeuron++)
{
double v = network.GetWeight(layerNumber, biasNeuron,
toNeuron);
if (network.Structure.ConnectionLimited)
{
if (Math.Abs(v) < network.Structure.ConnectionLimit)
{
assignDisabled++;
}
}
biasList.Add(v);
allList.Add(v);
assignedTotal++;
}
}
}
_disabledConnections = assignDisabled;
_totalConnections = assignedTotal;
_weights = new NumericRange(weightList);
_bias = new NumericRange(biasList);
_weightsAndBias = new NumericRange(allList);
_weightValues = EngineArray.ListToDouble(weightList);
_allValues = EngineArray.ListToDouble(allList);
_biasValues = EngineArray.ListToDouble(biasList);
}
/// <summary>
/// Construct a network analyze class. Analyze the specified network.
/// </summary>
///
/// <param name="network">The network to analyze.</param>
public AnalyzeNetwork(BasicNetwork network)
{
int assignDisabled = 0;
int assignedTotal = 0;
IList <Double> biasList = new List <Double>();
IList <Double> weightList = new List <Double>();
IList <Double> allList = new List <Double>();
for (int layerNumber = 0; layerNumber < network.LayerCount - 1; layerNumber++)
{
int fromCount = network.GetLayerNeuronCount(layerNumber);
int fromBiasCount = network
.GetLayerTotalNeuronCount(layerNumber);
int toCount = network.GetLayerNeuronCount(layerNumber + 1);
// weights
for (int fromNeuron = 0; fromNeuron < fromCount; fromNeuron++)
{
for (int toNeuron = 0; toNeuron < toCount; toNeuron++)
{
double v = network.GetWeight(layerNumber, fromNeuron,
toNeuron);
if (network.Structure.ConnectionLimited)
{
if (Math.Abs(v) < network.Structure.ConnectionLimit)
{
assignDisabled++;
}
}
weightList.Add(v);
allList.Add(v);
assignedTotal++;
}
}
// bias
if (fromCount != fromBiasCount)
{
int biasNeuron = fromCount;
for (int toNeuron = 0; toNeuron < toCount; toNeuron++)
{
double v = network.GetWeight(layerNumber, biasNeuron,
toNeuron);
if (network.Structure.ConnectionLimited)
{
if (Math.Abs(v) < network.Structure.ConnectionLimit)
{
assignDisabled++;
}
}
biasList.Add(v);
allList.Add(v);
assignedTotal++;
}
}
}
_disabledConnections = assignDisabled;
_totalConnections = assignedTotal;
_weights = new NumericRange(weightList);
_bias = new NumericRange(biasList);
_weightsAndBias = new NumericRange(allList);
_weightValues = EngineArray.ListToDouble(weightList);
_allValues = EngineArray.ListToDouble(allList);
_biasValues = EngineArray.ListToDouble(biasList);
}
/// <summary>
/// Construct a network analyze class. Analyze the specified network.
/// </summary>
///
/// <param name="network">The network to analyze.</param>
public AnalyzeNetwork(BasicNetwork network)
{
IList<Double> biasList = new List<Double>();
IList<Double> weightList = new List<Double>();
IList<Double> allList = new List<Double>();
for (int layerNumber = 0; layerNumber < network.LayerCount - 1; layerNumber++)
{
int fromCount = network.GetLayerNeuronCount(layerNumber);
int fromBiasCount = network
.GetLayerTotalNeuronCount(layerNumber);
int toCount = network.GetLayerNeuronCount(layerNumber + 1);
// weights
for (int fromNeuron = 0; fromNeuron < fromCount; fromNeuron++)
{
for (int toNeuron = 0; toNeuron < toCount; toNeuron++)
{
double v = network.GetWeight(layerNumber, fromNeuron,
toNeuron);
weightList.Add(v);
allList.Add(v);
}
}
// bias
if (fromCount != fromBiasCount)
{
int biasNeuron = fromCount;
for (int toNeuron = 0; toNeuron < toCount; toNeuron++)
{
double v = network.GetWeight(layerNumber, biasNeuron,
toNeuron);
biasList.Add(v);
allList.Add(v);
}
}
}
_disabledConnections = 0;
_totalConnections = 0;
_weights = new NumericRange(weightList);
_bias = new NumericRange(biasList);
_weightsAndBias = new NumericRange(allList);
_weightValues = EngineArray.ListToDouble(weightList);
_allValues = EngineArray.ListToDouble(allList);
_biasValues = EngineArray.ListToDouble(biasList);
}
/// <summary>
/// Calculate the derivatives for this training set element.
/// </summary>
///
/// <param name="pair">The training set element.</param>
/// <returns>The sum squared of errors.</returns>
private double CalculateDerivatives(IMLDataPair pair)
{
// error values
double e = 0.0d;
double sum = 0.0d;
_network.Compute(pair.Input);
int fromLayer = _network.LayerCount - 2;
int toLayer = _network.LayerCount - 1;
int fromNeuronCount = _network.GetLayerTotalNeuronCount(fromLayer);
int toNeuronCount = _network.GetLayerNeuronCount(toLayer);
double output = _network.Structure.Flat.LayerOutput[0];
e = pair.Ideal[0] - output;
for (int i = 0; i < fromNeuronCount; i++)
{
double lastOutput = _network.GetLayerOutput(fromLayer, i);
_jacobian[_jacobianRow][_jacobianCol++] = CalcDerivative(
_network.GetActivation(toLayer), output) * lastOutput;
}
while (fromLayer > 0)
{
fromLayer--;
toLayer--;
fromNeuronCount = _network.GetLayerTotalNeuronCount(fromLayer);
toNeuronCount = _network.GetLayerNeuronCount(toLayer);
// this.network.getLayerOutput(fromLayer, neuronNumber) holder.getResult().get(lastSynapse);
// for each neuron in the input layer
for (int neuron = 0; neuron < toNeuronCount; neuron++)
{
output = _network.GetLayerOutput(toLayer, neuron);
IActivationFunction function = _network.GetActivation(toLayer);
double w = _network.GetWeight(toLayer, neuron, 0);
double val = CalcDerivative(function, output)
* CalcDerivative2(function, sum) * w;
// for each weight of the input neuron
for (int i = 0; i < fromNeuronCount; i++)
{
sum = 0.0d;
// for each neuron in the next layer
for (int j = 0; j < toNeuronCount; j++)
{
// for each weight of the next neuron
for (int k = 0; k < fromNeuronCount; k++)
{
sum += _network.GetWeight(fromLayer, k, j) * output;
}
}
_jacobian[_jacobianRow][_jacobianCol++] = val
* _network.GetLayerOutput(fromLayer, i);
}
}
}
// return error
return(e);
}