public void ActivationLayer_CopyLayerForPredictionModel()
{
var batchSize = 1;
var sut = new ActivationLayer(Activation.Relu);
sut.Initialize(3, 3, 1, batchSize, Initialization.GlorotUniform, new Random(232));
var layers = new List <ILayer>();
sut.CopyLayerForPredictionModel(layers);
var actual = (ActivationLayer)layers.Single();
Assert.AreEqual(sut.Width, actual.Width);
Assert.AreEqual(sut.Height, actual.Height);
Assert.AreEqual(sut.Depth, actual.Depth);
Assert.AreEqual(sut.ActivationFunc, actual.ActivationFunc);
Assert.AreEqual(sut.OutputActivations.RowCount, actual.OutputActivations.RowCount);
Assert.AreEqual(sut.OutputActivations.ColumnCount, actual.OutputActivations.ColumnCount);
Assert.AreEqual(sut.ActivationDerivative.RowCount, actual.ActivationDerivative.RowCount);
Assert.AreEqual(sut.ActivationDerivative.ColumnCount, actual.ActivationDerivative.ColumnCount);
}
/// <summary>
/// Creates the initial weight vector w
/// </summary>
///
/// <returns>The sum of squared weights divided by 2.</returns>
///
private double saveNetworkToArray()
{
double w, sumOfSquaredWeights = 0.0;
// for each layer in the network
for (int li = 0, cur = 0; li < network.Layers.Length; li++)
{
ActivationLayer layer = network.Layers[li] as ActivationLayer;
// for each neuron in the layer
for (int ni = 0; ni < network.Layers[li].Neurons.Length; ni++, cur++)
{
ActivationNeuron neuron = layer.Neurons[ni] as ActivationNeuron;
// for each weight in the neuron
for (int wi = 0; wi < neuron.InputsCount; wi++, cur++)
{
// We copy it to the starting weights vector
w = weights[cur] = (float)neuron.Weights[wi];
sumOfSquaredWeights += w * w;
}
// and also for the threshold value (bias):
w = weights[cur] = (float)neuron.Threshold;
sumOfSquaredWeights += w * w;
}
}
return(sumOfSquaredWeights / 2.0);
}
/// <summary>
/// Update network's weights.
/// </summary>
///
/// <returns>The sum of squared weights divided by 2.</returns>
///
private double loadArrayIntoNetwork()
{
double w, sumOfSquaredWeights = 0.0;
// For each layer in the network
for (int li = 0, cur = 0; li < network.Layers.Length; li++)
{
ActivationLayer layer = network.Layers[li] as ActivationLayer;
// for each neuron in the layer
for (int ni = 0; ni < layer.Neurons.Length; ni++, cur++)
{
ActivationNeuron neuron = layer.Neurons[ni] as ActivationNeuron;
// for each weight in the neuron
for (int wi = 0; wi < neuron.Weights.Length; wi++, cur++)
{
neuron.Weights[wi] = w = weights[cur] + deltas[cur];
sumOfSquaredWeights += w * w;
}
// for each threshold value (bias):
neuron.Threshold = w = weights[cur] + deltas[cur];
sumOfSquaredWeights += w * w;
}
}
return(sumOfSquaredWeights / 2.0);
}
public void backward()
{
int count_examples = 2;
int layerInSize = 5;
var layer = new ActivationLayer(layerInSize);
var inputs = MatrixD.Build.DenseOfArray(
new double[, ] {
{ -100, -1, 0, 1, 100 }, { -500, -0.1, 0, 0.1, 500 }
});
var nextGradients = MatrixD.Build.DenseOfArray(
new double[, ] {
{ 0, 0, 0, 1, 100 }, { 0, 0, 0, 0.1, 500 }
});
var expectedGradients = MatrixD.Build.repeat(count_examples, layerInSize, 0);
FuncDD[] df = layer.DerivateActivations;
for (var i = 0; i < count_examples; i++)
{
for (var j = 0; j < layerInSize; j++)
{
expectedGradients[i, j] = nextGradients[i, j] * df[j](inputs[i, j]);
}
}
Assert.IsTrue(layer.backward(inputs, nextGradients).EEquals(expectedGradients));
}
public FCSuperResolution()
{
superres_enc_front = InputLayer.Create(StartSide, 3);
superres_enc_back = ActivationLayer.Create <LeakyReLU>();
superres_enc_front.Append(
FCLayer.Create(16, 16).Append(
ActivationLayer.Create <LeakyReLU>().Append(
FCLayer.Create(8, 8).Append(
ActivationLayer.Create <LeakyReLU>().Append(
FCLayer.Create(4, 4).Append(
superres_enc_back
))))));
superres_dec_front = InputLayer.Create(4, 4);
superres_dec_back = ActivationLayer.Create <Tanh>();
superres_dec_front.Append(
FCLayer.Create(8, 8).Append(
ActivationLayer.Create <LeakyReLU>().Append(
FCLayer.Create(16, 8).Append(
ActivationLayer.Create <LeakyReLU>().Append(
FCLayer.Create(32, 3).Append(
superres_dec_back
))))));
superres_enc_back.Append(superres_dec_front);
//Initialize Weights
superres_enc_front.SetupInternalState();
superres_enc_front.InitializeWeights(new UniformWeightInitializer(0, 0.001f));
}
private void CreateDNet()
{
ConvolutionLayer conv0 = new ConvolutionLayer(inputDimension, filterSize: 3, filterCount: 32, zeroPadding: true);
ActivationLayer activation0 = new ActivationLayer(new Relu(leaky: true));
MaxPooling2DLayer pool0 = new MaxPooling2DLayer();
ConvolutionLayer conv1 = new ConvolutionLayer(inputDimension, filterSize: 3, filterCount: 32, zeroPadding: true);
ActivationLayer activation1 = new ActivationLayer(new Relu(leaky: true));
MaxPooling2DLayer pool1 = new MaxPooling2DLayer();
FlattenLayer flatten = new FlattenLayer();
LinearLayer linear0 = new LinearLayer(numNeurons: 128);
ActivationLayer activation2 = new ActivationLayer(new Relu(leaky: true));
LinearLayer linear1 = new LinearLayer(numNeurons: 1);
ActivationLayer activation3 = new ActivationLayer(new Sigmoid());
dNet.Add(conv0);
dNet.Add(activation0);
dNet.Add(pool0);
dNet.Add(conv1);
dNet.Add(activation1);
dNet.Add(pool1);
dNet.Add(flatten);
dNet.Add(linear0);
dNet.Add(activation2);
dNet.Add(linear1);
dNet.Add(activation3);
dNet.Compile(new BinaryCrossEntropy(), new Adam(0.001d));
}
public DGAN()
{
discriminator = InputLayer.Create(StartSide, 1);
discriminator_back = ActivationLayer.Create <Sigmoid>();
discriminator.Append(
FCLayer.Create(1, 256).Append(
ActivationLayer.Create <LeakyReLU>().Append(
FCLayer.Create(1, 1).Append(
discriminator_back
))));
generator = InputLayer.Create(1, LatentSize);
generator_back = ActivationLayer.Create <Tanh>();
generator.Append(
FCLayer.Create(1, 256).Append(
ActivationLayer.Create <LeakyReLU>().Append(
DropoutLayer.Create(0.5f).Append(
FCLayer.Create(1, 512).Append(
ActivationLayer.Create <LeakyReLU>().Append(
DropoutLayer.Create(0.5f).Append(
FCLayer.Create(1, OutputSize).Append(
generator_back
))))))));
//Initialize Weights
discriminator.SetupInternalState();
discriminator.InitializeWeights(new UniformWeightInitializer(3, 0));
generator.SetupInternalState();
generator.InitializeWeights(new UniformWeightInitializer(1, 0));
}
/// <summary>
/// Initializes bias values of activation neurons in the activation layer.
/// </summary>
/// <param name="activationLayer">
/// The activation layer to initialize
/// </param>
/// <exception cref="ArgumentNullException">
/// If <c>activationLayer</c> is <c>null</c>
/// </exception>
public void Initialize(ActivationLayer activationLayer)
{
Helper.ValidateNotNull(activationLayer, "layer");
foreach (ActivationNeuron neuron in activationLayer.Neurons)
{
neuron.bias = Helper.GetRandom(minLimit, maxLimit);
}
}
/// <summary>
/// Initializes bias values of activation neurons in the activation layer.
/// </summary>
/// <param name="activationLayer">
/// The activation layer to initialize
/// </param>
/// <exception cref="ArgumentNullException">
/// If <c>activationLayer</c> is <c>null</c>
/// </exception>
public void Initialize(ActivationLayer activationLayer)
{
Helper.ValidateNotNull(activationLayer, "layer");
foreach (ActivationNeuron neuron in activationLayer.Neurons)
{
neuron.bias = constant;
}
}
public void ActivationLayer_ReLu_GradientCheck()
{
const int fanIn = 5;
const int batchSize = 10;
var sut = new ActivationLayer(Activation.Relu);
GradientCheckTools.CheckLayer(sut, fanIn, 1, 1, batchSize, 1e-4f, new Random(21));
}
public ConvSuperResolution()
{
superres_enc_front = InputLayer.Create(StartSide, 3);
superres_enc_back = ActivationLayer.Create <ReLU>();
var pooling_0 = PoolingLayer.Create(2, 2);
var pooling_1 = PoolingLayer.Create(2, 2);
var pooling_2 = PoolingLayer.Create(2, 2);
var pooling_3 = PoolingLayer.Create(2, 2);
superres_enc_front.Append(
ConvLayer.Create(5, 128, 2).Append( //o = 96
ActivationLayer.Create <ReLU>().Append(
pooling_0.Append( //o = 48
ConvLayer.Create(3, 128, 1).Append( //o = 48
ActivationLayer.Create <ReLU>().Append(
pooling_1.Append( //o = 24
ConvLayer.Create(3, 64, 1).Append( //o = 24
ActivationLayer.Create <ReLU>().Append(
pooling_2.Append( //o = 12
ConvLayer.Create(3, 32, 1).Append( //o = 12
ActivationLayer.Create <ReLU>().Append(
pooling_3.Append( //o = 6
ConvLayer.Create(3, 32, 1).Append( //o = 6
superres_enc_back
))))))))))))));
superres_dec_front = InputLayer.Create(6, 32);
superres_dec_back = ActivationLayer.Create <Tanh>();
superres_dec_front.Append(
ConvLayer.Create(3, 32, 1).Append( //o = 6
ActivationLayer.Create <ReLU>().Append(
UnpoolingLayer.Create(pooling_3).Append( //o = 12
ConvLayer.Create(3, 64, 1).Append( //o = 12
ActivationLayer.Create <ReLU>().Append(
UnpoolingLayer.Create(pooling_2).Append( //o = 24
ConvLayer.Create(3, 128, 1).Append( //o = 24
ActivationLayer.Create <ReLU>().Append(
UnpoolingLayer.Create(pooling_1).Append( //o = 48
ConvLayer.Create(3, 128, 1).Append( //o = 48
ActivationLayer.Create <ReLU>().Append(
UnpoolingLayer.Create(pooling_0).Append( //o = 96
ConvLayer.Create(5, 3, 2).Append( //o = 96
superres_dec_back
))))))))))))));
superres_enc_back.Append(superres_dec_front);
//TODO: come up with an approach that saves the convolution/multiplication indexes and rearranges the weights etc so they fit into cache better
//TODO: unpooling layer tied to pooling layers
//Initialize Weights
superres_enc_front.SetupInternalState();
superres_enc_front.InitializeWeights(new UniformWeightInitializer(0, 0.001f));
}
/// <summary>
/// Create a new activation neuron
/// </summary>
/// <param name="parent">
/// The parent layer containing this neuron
/// </param>
/// <exception cref="System.ArgumentNullException">
/// If <c>parent</c> is <c>null</c>
/// </exception>
public ActivationNeuron(ActivationLayer parent)
{
Helper.ValidateNotNull(parent, "parent");
this.input = 0d;
this.output = 0d;
this.error = 0d;
this.bias = 0d;
this.parent = parent;
}
/// <summary>
/// Runs learning epoch.
/// </summary>
///
/// <param name="input">Array of input vectors.</param>
/// <param name="output">Array of output vectors.</param>
///
/// <returns>Returns summary squared learning error for the entire epoch.</returns>
///
/// <remarks><para><note>While running the neural network's learning process, it is required to
/// pass the same <paramref name="input"/> and <paramref name="output"/> values for each
/// epoch. On the very first run of the method it will initialize evolutionary fitness
/// function with the given input/output. So, changing input/output in middle of the learning
/// process, will break it.</note></para></remarks>
///
public double RunEpoch(double[][] input, double[][] output)
{
Debug.Assert(input.Length > 0);
Debug.Assert(output.Length > 0);
Debug.Assert(input.Length == output.Length);
Debug.Assert(network.InputsCount == input.Length);
// check if it is a first run and create population if so
if (population == null)
{
// sample chromosome
DoubleArrayChromosome chromosomeExample = new DoubleArrayChromosome(
chromosomeGenerator, mutationMultiplierGenerator, mutationAdditionGenerator,
numberOfNetworksWeights);
// create population ...
population = new Population(populationSize, chromosomeExample,
new EvolutionaryFitness(network, input, output), selectionMethod);
// ... and configure it
population.CrossoverRate = crossOverRate;
population.MutationRate = mutationRate;
population.RandomSelectionPortion = randomSelectionRate;
}
// run genetic epoch
population.RunEpoch( );
// get best chromosome of the population
DoubleArrayChromosome chromosome = (DoubleArrayChromosome)population.BestChromosome;
double[] chromosomeGenes = chromosome.Value;
// put best chromosome's value into neural network's weights
int v = 0;
for (int i = 0, layersCount = network.LayersCount; i < layersCount; i++)
{
ActivationLayer layer = network[i];
for (int j = 0, neuronsCount = layer.NeuronsCount; j < neuronsCount; j++)
{
ActivationNeuron neuron = layer[j];
for (int k = 0, weightsCount = neuron.InputsCount; k < weightsCount; k++)
{
neuron[k] = chromosomeGenes[v++];
}
neuron.Threshold = chromosomeGenes[v++];
}
}
Debug.Assert(v == numberOfNetworksWeights);
return(1.0 / chromosome.Fitness);
}
/// <summary>
/// Initializes a new instance of the <see cref="LevenbergMarquardtLearning"/> class.
/// </summary>
///
/// <param name="network">Network to teach.</param>
/// <param name="useRegularization">True to use Bayesian regularization, false otherwise.</param>
/// <param name="method">The method by which the Jacobian matrix will be calculated.</param>
///
public LevenbergMarquardtLearning(ActivationNetwork network, bool useRegularization, JacobianMethod method)
{
this.ParallelOptions = new ParallelOptions();
this.network = network;
this.numberOfParameters = getNumberOfParameters(network);
this.outputCount = network.Layers[network.Layers.Length - 1].Neurons.Length;
this.useBayesianRegularization = useRegularization;
this.method = method;
this.weights = new float[numberOfParameters];
this.hessian = new float[numberOfParameters][];
for (int i = 0; i < hessian.Length; i++)
{
hessian[i] = new float[numberOfParameters];
}
this.diagonal = new float[numberOfParameters];
this.gradient = new float[numberOfParameters];
this.jacobian = new float[numberOfParameters][];
// Will use Backpropagation method for Jacobian computation
if (method == JacobianMethod.ByBackpropagation)
{
// create weight derivatives arrays
this.weightDerivatives = new float[network.Layers.Length][][];
this.thresholdsDerivatives = new float[network.Layers.Length][];
// initialize arrays
for (int i = 0; i < network.Layers.Length; i++)
{
ActivationLayer layer = (ActivationLayer)network.Layers[i];
this.weightDerivatives[i] = new float[layer.Neurons.Length][];
this.thresholdsDerivatives[i] = new float[layer.Neurons.Length];
for (int j = 0; j < layer.Neurons.Length; j++)
{
this.weightDerivatives[i][j] = new float[layer.InputsCount];
}
}
}
else // Will use finite difference method for Jacobian computation
{
// create differential coefficient arrays
this.differentialCoefficients = createCoefficients(3);
this.derivativeStepSize = new double[numberOfParameters];
// initialize arrays
for (int i = 0; i < numberOfParameters; i++)
{
this.derivativeStepSize[i] = derivativeStep;
}
}
}
/// <summary>
/// Initializes bias values of activation neurons in the activation layer.
/// </summary>
/// <param name="activationLayer">
/// The activation layer to initialize
/// </param>
/// <exception cref="ArgumentNullException">
/// If <c>activationLayer</c> is <c>null</c>
/// </exception>
public void Initialize(ActivationLayer activationLayer)
{
Helper.ValidateNotNull(activationLayer, "activationLayer");
int i = 0;
double[] normalized = Helper.GetRandomVector(activationLayer.NeuronCount, 1d);
foreach (ActivationNeuron neuron in activationLayer.Neurons)
{
neuron.bias = normalized[i++];
}
}
public ActivationNetwork(IActivationFunction function, int inputsCount, params int[] neuronsCount)
: base(inputsCount, neuronsCount.Length)
{
for (int i = 0; i < layersCount; i++)
{
layers[i] = new ActivationLayer(
neuronsCount[i],
(i == 0) ? inputsCount : neuronsCount[i - 1],
function);
}
}
/// <summary>
/// Initializes bias values of activation neurons in the activation layer.
/// </summary>
/// <param name="activationLayer">
/// The activation layer to initialize
/// </param>
/// <exception cref="ArgumentNullException">
/// If <c>activationLayer</c> is <c>null</c>
/// </exception>
public void Initialize(ActivationLayer activationLayer)
{
Helper.ValidateNotNull(activationLayer, "activationLayer");
int i = 0;
double[] normalized = Helper.GetRandomVector(activationLayer.NeuronCount, 1d);
foreach (ActivationNeuron neuron in activationLayer.Neurons)
{
neuron.bias = normalized[i++];
}
}
/// <summary>
/// 点击计算按钮
/// </summary>
/// <param name="sender"></param>
/// <param name="e"></param>
private void tsmiCalculate_Click(object sender, EventArgs e)
{
// 创建输入层、隐层和输出层
ActivationLayer inputLayer = GetLayer(cboInputLayerType.SelectedItem.ToString(), 2);
ActivationLayer hiddenLayer = GetLayer(cboHiddenLayerType.SelectedItem.ToString(), int.Parse(txtHiddenLayerCount.Text));
ActivationLayer outputLayer = GetLayer(cboOutputLayerType.SelectedItem.ToString(), 1);
// 创建层之间的关联
new BackpropagationConnector(inputLayer, hiddenLayer, ConnectionMode.Complete).Initializer = new RandomFunction(0, 0.3);
new BackpropagationConnector(hiddenLayer, outputLayer, ConnectionMode.Complete).Initializer = new RandomFunction(0, 0.3);
// 创建神经网络
var network = new BackpropagationNetwork(inputLayer, outputLayer);
network.SetLearningRate(double.Parse(txtInitialLearningRate.Text), double.Parse(txtFinalLearningRate.Text));
// 进行训练
var trainingSet = new TrainingSet(2, 1);
for (var i = 0; i < 17; i++)
{
var x1 = data[i, 0];
var x2 = data[i, 1];
var y = data[i, 2];
var inputVector = new double[] { x1, x2 };
var outputVector = new double[] { y };
var trainingSample = new TrainingSample(inputVector, outputVector);
trainingSet.Add(trainingSample);
}
network.SetLearningRate(0.3, 0.1);
network.Learn(trainingSet, int.Parse(txtTrainingEpochs.Text));
network.StopLearning();
// 进行预测
for (var i = 0; i < 17; i++)
{
var x1 = data[i, 0];
var x2 = data[i, 1];
var y = data[i, 2];
var testInput = new double[] { x1, x2 };
var testOutput = network.Run(testInput)[0];
var absolute = testOutput - y;
var relative = Math.Abs((testOutput - y) / testOutput);
dgvData.Rows[i].Cells[3].Value = testOutput.ToString("f3");
dgvData.Rows[i].Cells[4].Value = absolute.ToString("f3");
dgvData.Rows[i].Cells[5].Value = (relative * 100).ToString("f1") + "%";
}
}
/// <summary>
/// Creates a new Back Propagation Network, with the specified input and output layers. (You
/// are required to connect all layers using appropriate synapses, before using the constructor.
/// Any changes made to the structure of the network after its creation may lead to complete
/// malfunctioning)
/// </summary>
/// <param name="inputLayer">
/// The input layer
/// </param>
/// <param name="outputLayer">
/// The output layer
/// </param>
/// <exception cref="ArgumentNullException">
/// If <c>inputLayer</c> or <c>outputLayer</c> is <c>null</c>
/// </exception>
public PSONetwork(ActivationLayer inputLayer, ActivationLayer outputLayer)
: base(inputLayer, outputLayer, TrainingMethod.Supervised)
{
this.meanSquaredError = 0d;
this.isValidMSE = false;
// Re-Initialize the network
Initialize();
double[] weights = getAllWeights();
PsoProblem = new Problem(OptimisationProblem.Neural_Network, new Problem.FitnessHandler(getFitness), weights);
}
/// <summary>
/// 神经网络
/// </summary>
/// <param name="function"></param>
/// <param name="inputsCount"></param>
/// <param name="neuronsCount">指定神经网络每层中的神经元数量</param>
public ActivationNetwork(IActivationFunction function, int inputsCount, params int[] neuronsCount)
: base(inputsCount, neuronsCount.Length)
{
// neuronsCount 指定神经网络每层中的神经元数量。
for (int i = 0; i < Layers.Length; i++)
{
Layers[i] = new ActivationLayer(
neuronsCount[i],
// 每个神经元只有一个输出，上一层有多少个神经元也就有多少个输出，也就是这一层需要有多少输入
(i == 0) ? inputsCount : neuronsCount[i - 1],
function);
}
}
/// <summary>
/// Evaluates chromosome.
/// </summary>
///
/// <param name="chromosome">Chromosome to evaluate.</param>
///
/// <returns>Returns chromosome's fitness value.</returns>
///
/// <remarks>The method calculates fitness value of the specified
/// chromosome.</remarks>
///
public double Evaluate(IChromosome chromosome)
{
DoubleArrayChromosome daChromosome = (DoubleArrayChromosome)chromosome;
double[] chromosomeGenes = daChromosome.Value;
// total number of weight in neural network
int totalNumberOfWeights = 0;
// asign new weights and thresholds to network from the given chromosome
for (int i = 0, layersCount = network.LayersCount; i < layersCount; i++)
{
ActivationLayer layer = network[i];
for (int j = 0, neuronsCount = layer.NeuronsCount; j < neuronsCount; j++)
{
ActivationNeuron neuron = layer[j];
for (int k = 0, weightsCount = neuron.InputsCount; k < weightsCount; k++)
{
neuron[k] = chromosomeGenes[totalNumberOfWeights++];
}
neuron.Threshold = chromosomeGenes[totalNumberOfWeights++];
}
}
// post check if all values are processed and lenght of chromosome
// is equal to network size
Debug.Assert(totalNumberOfWeights == daChromosome.Length);
double totalError = 0;
for (int i = 0, inputVectorsAmount = input.Length; i < inputVectorsAmount; i++)
{
double[] computedOutput = network.Compute(input[i]);
for (int j = 0, outputLength = output[0].Length; j < outputLength; j++)
{
double error = output[i][j] - computedOutput[j];
totalError += error * error;
}
}
if (totalError > 0)
{
return(1.0 / totalError);
}
// zero error means the best fitness
return(double.MaxValue);
}
/// <summary>
/// Calculates the Jacobian Matrix using Finite Differences
/// </summary>
///
/// <returns>Returns the sum of squared errors of the network divided by 2.</returns>
///
private double JacobianByFiniteDifference(double[][] input, double[][] desiredOutput)
{
double e, sumOfSquaredErrors = 0;
// for each input training sample
for (int i = 0, row = 0; i < input.Length; i++)
{
// Compute a forward pass
double[] networkOutput = network.Compute(input[i]);
// for each output respective to the input
for (int j = 0; j < networkOutput.Length; j++, row++)
{
// Calculate network error to build the residuals vector
e = errors[row] = desiredOutput[i][j] - networkOutput[j];
sumOfSquaredErrors += e * e;
// Computation of one of the Jacobian Matrix rows by numerical differentiation:
// for each weight w_j in the network, we have to compute its partial derivative
// to build the Jacobian matrix.
// So, for each layer:
for (int li = 0, col = 0; li < network.Layers.Length; li++)
{
ActivationLayer layer = network.Layers[li] as ActivationLayer;
// for each neuron:
for (int ni = 0; ni < layer.Neurons.Length; ni++, col++)
{
ActivationNeuron neuron = layer.Neurons[ni] as ActivationNeuron;
// for each weight:
for (int wi = 0; wi < neuron.InputsCount; wi++, col++)
{
// Compute its partial derivative
jacobian[col][row] = (float)ComputeDerivative(input[i], li, ni,
wi, ref derivativeStepSize[col], networkOutput[j], j);
}
// and also for each threshold value (bias)
jacobian[col][row] = (float)ComputeDerivative(input[i], li, ni,
-1, ref derivativeStepSize[col], networkOutput[j], j);
}
}
}
}
// returns the sum of squared errors / 2
return(sumOfSquaredErrors / 2.0);
}
/// <summary>
/// Calculates error values for all neurons of the network.
/// </summary>
///
/// <param name="desiredOutput">Desired output vector.</param>
///
/// <returns>Returns summary squared error of the last layer divided by 2.</returns>
///
private double CalculateError(double[] desiredOutput)
{
double sumOfSquaredErrors = 0.0;
int layersCount = network.Layers.Length;
double[][] networkErrors = this.networkErrors.Value;
double[][] networkOutputs = this.networkOutputs.Value;
// Assume that all network neurons have the same activation function
var function = (this.network.Layers[0].Neurons[0] as ActivationNeuron)
.ActivationFunction;
// 1. Calculate error values for last layer first.
double[] layerOutputs = networkOutputs[layersCount - 1];
double[] errors = networkErrors[layersCount - 1];
for (int i = 0; i < errors.Length; i++)
{
double output = layerOutputs[i];
double e = output - desiredOutput[i];
errors[i] = e * function.Derivative2(output);
sumOfSquaredErrors += e * e;
}
// 2. Calculate errors for all other layers
for (int j = layersCount - 2; j >= 0; j--)
{
errors = networkErrors[j];
layerOutputs = networkOutputs[j];
ActivationLayer nextLayer = network.Layers[j + 1] as ActivationLayer;
double[] nextErrors = networkErrors[j + 1];
// For all neurons of this layer
for (int i = 0; i < errors.Length; i++)
{
double sum = 0.0;
// For all neurons of the next layer
for (int k = 0; k < nextErrors.Length; k++)
{
sum += nextErrors[k] * nextLayer.Neurons[k].Weights[i];
}
errors[i] = sum * function.Derivative2(layerOutputs[i]);
}
}
return(sumOfSquaredErrors / 2.0);
}
public void initNet(int fLayer, int sLayer, int tLayer)
{
ActivationLayer inputLayer = createLayer(fLayer, inputNeurons);
ActivationLayer hiddenLayer = createLayer(sLayer, hiddenNeurons);
ActivationLayer outputLayer = createLayer(tLayer, outputNeurons);
//var hiddenLayer = new LinearLayer(hiddenNeurons);
//var outputLayer = new LinearLayer(outputNeurons);
new BackpropagationConnector(inputLayer, hiddenLayer).Initializer = new RandomFunction(0d, 0.3d);
new BackpropagationConnector(hiddenLayer, outputLayer).Initializer = new RandomFunction(0d, 0.3d);
network = new BackpropagationNetwork(inputLayer, outputLayer);
network.SetLearningRate(learningRate);
SampleInput = new double[inputNeurons + outputNeurons];
SampleOutput = new double[outputNeurons];
}
/// <summary>
/// Initializes a new instance of the <see cref="ActivationNetwork"/> class
/// </summary>
/// <param name="function">Activation function of neurons of the network</param>
/// <param name="inputsCount">Network's inputs count</param>
/// <param name="neuronsCount">Array, which specifies the amount of neurons in
/// each layer of the neural network</param>
///
/// <remarks>The new network will be randomized (see <see cref="ActivationNeuron.Randomize"/>
/// method) after it is created.</remarks>
///
/// <example>The following sample illustrates the usage of <c>ActivationNetwork</c> class:
/// <code>
/// // create activation network
/// ActivationNetwork network = new ActivationNetwork(
/// new SigmoidFunction( ), // sigmoid activation function
/// 3, // 3 inputs
/// 4, 1 ); // 2 layers:
/// // 4 neurons in the firs layer
/// // 1 neuron in the second layer
/// </code>
/// </example>
///
public ActivationNetwork(IActivationFunction function, int inputsCount, params int[] neuronsCount)
: base(inputsCount, neuronsCount.Length)
{
// create each layer
for (int i = 0; i < LayersCount; i++)
{
layers[i] = new ActivationLayer(
// neurons count in the layer
neuronsCount[i],
// inputs count of the layer
(i == 0) ? inputsCount : neuronsCount[i - 1],
// activation function of the layer
function);
}
}
/// <summary>
/// Calculate weights updates
/// </summary>
///
/// <param name="input">Network's input vector.</param>
///
private void CalculateGradient(double[] input)
{
// 1. calculate updates for the first layer
ActivationLayer layer = network.Layers[0] as ActivationLayer;
double[] weightErrors = neuronErrors[0];
double[][] layerWeightsDerivatives = weightsDerivatives[0];
double[] layerThresholdDerivatives = thresholdsDerivatives[0];
// So, for each neuron of the first layer:
for (int i = 0; i < layer.Neurons.Length; i++)
{
ActivationNeuron neuron = layer.Neurons[i] as ActivationNeuron;
double[] neuronWeightDerivatives = layerWeightsDerivatives[i];
// for each weight of the neuron:
for (int j = 0; j < neuron.InputsCount; j++)
{
neuronWeightDerivatives[j] += weightErrors[i] * input[j];
}
layerThresholdDerivatives[i] += weightErrors[i];
}
// 2. for all other layers
for (int k = 1; k < network.Layers.Length; k++)
{
layer = network.Layers[k] as ActivationLayer;
weightErrors = neuronErrors[k];
layerWeightsDerivatives = weightsDerivatives[k];
layerThresholdDerivatives = thresholdsDerivatives[k];
ActivationLayer layerPrev = network.Layers[k - 1] as ActivationLayer;
// for each neuron of the layer
for (int i = 0; i < layer.Neurons.Length; i++)
{
ActivationNeuron neuron = layer.Neurons[i] as ActivationNeuron;
double[] neuronWeightDerivatives = layerWeightsDerivatives[i];
// for each weight of the neuron
for (int j = 0; j < layerPrev.Neurons.Length; j++)
{
neuronWeightDerivatives[j] += weightErrors[i] * layerPrev.Neurons[j].Output;
}
layerThresholdDerivatives[i] += weightErrors[i];
}
}
}
/// <summary>
/// Calculates error values for all neurons of the network.
/// </summary>
///
/// <param name="desiredOutput">Desired output vector.</param>
///
/// <returns>Returns summary squared error of the last layer divided by 2.</returns>
///
private double CalculateError(double[] desiredOutput)
{
double error = 0;
int layersCount = network.Layers.Length;
// assume, that all neurons of the network have the same activation function
IActivationFunction function = (network.Layers[0].Neurons[0] as ActivationNeuron).ActivationFunction;
// calculate error values for the last layer first
ActivationLayer layer = network.Layers[layersCount - 1] as ActivationLayer;
double[] layerDerivatives = neuronErrors[layersCount - 1];
for (int i = 0; i < layer.Neurons.Length; i++)
{
double output = layer.Neurons[i].Output;
double e = output - desiredOutput[i];
layerDerivatives[i] = e * function.Derivative2(output);
error += (e * e);
}
// calculate error values for other layers
for (int j = layersCount - 2; j >= 0; j--)
{
layer = network.Layers[j] as ActivationLayer;
layerDerivatives = neuronErrors[j];
ActivationLayer layerNext = network.Layers[j + 1] as ActivationLayer;
double[] nextDerivatives = neuronErrors[j + 1];
// for all neurons of the layer
for (int i = 0, n = layer.Neurons.Length; i < n; i++)
{
double sum = 0.0;
for (int k = 0; k < layerNext.Neurons.Length; k++)
{
sum += nextDerivatives[k] * layerNext.Neurons[k].Weights[i];
}
layerDerivatives[i] = sum * function.Derivative2(layer.Neurons[i].Output);
}
}
// return squared error of the last layer divided by 2
return(error / 2.0);
}
public void forward()
{
var inputs = MatrixD.Build.DenseOfArray(
new double[, ] {
{ -100, -1, 0, 1, 100 }, { -500, -0.1, 0, 0.1, 500 }
});
var expectedOutputs = MatrixD.Build.DenseOfArray(
new double[, ] {
{ 0, 0, 0, 1, 100 }, { 0, 0, 0, 0.1, 500 }
});
var layer = new ActivationLayer(5);
Assert.IsTrue(layer.forward(inputs).EEquals(expectedOutputs));
}
private void CreateGNet()
{
ConvolutionLayer conv1 = new ConvolutionLayer(inputDimension, filterSize: 3, filterCount: 4, zeroPadding: true);
ActivationLayer act = new ActivationLayer(new Relu(leaky: true));
ConvolutionLayer conv = new ConvolutionLayer(inputDimension, filterSize: 3, filterCount: 4, zeroPadding: true);
ActivationLayer act1 = new ActivationLayer(new Relu(leaky: true));
ConvolutionLayer convolutionOut = new ConvolutionLayer(inputDimension, filterSize: 3, filterCount: 4, zeroPadding: true);
ActivationLayer actOut = new ActivationLayer(new Relu(leaky: true));
gNet.Add(conv1);
gNet.Add(act);
gNet.Add(conv);
gNet.Add(act1);
gNet.Add(convolutionOut);
gNet.Compile(new BinaryCrossEntropy(), new Adam(0.001d));
}
public void Treinar(double[][] dados)
{
double[][] output = new double[dados.Length][];
double[][] input = new double[dados.Length][];
for (int i = 0; i < dados.Length; i++)
{
input[i] = new double[Variaveis];
output[i] = new double[TotalNeuronios];
for (int j = 0; j < Variaveis; j++)
{
input[i][j] = dados[i][j];
}
int classe = Convert.ToInt32(dados[i][4]) - 1;
output[i][classe] = 1;
}
ActivationNetwork network = new ActivationNetwork(new SigmoidFunction(ValorAlpha), Variaveis, TotalNeuronios);
ActivationLayer layer = network[0];
PerceptronLearning teacher = new PerceptronLearning(network);
teacher.LearningRate = CamadaRate;
int interacao = 1000;
int count = 0;
List <double> errolist = new List <double>();
while (count <= interacao)
{
double erro = teacher.RunEpoch(input, output) / dados.Length;
errolist.Add(erro);
count++;
}
CriaColunaFuncao();
for (int i = 0; i < TotalNeuronios; i++)
{
this.FuncaoNeuronio.Rows.Add("Neuronio [" + (i + 1) + "]", Convert.ToInt32(layer[i].Threshold), layer[i][0]);
}
}
// Create and initialize genetic population
private int CalculateNetworkSize(ActivationNetwork activationNetwork)
{
// caclculate total amount of weight in neural network
int networkSize = 0;
for (int i = 0, layersCount = network.LayersCount; i < layersCount; i++)
{
ActivationLayer layer = network[i];
for (int j = 0, neuronsCount = layer.NeuronsCount; j < neuronsCount; j++)
{
// sum all weights and threshold
networkSize += layer[j].InputsCount + 1;
}
}
return(networkSize);
}
/// <summary>
/// Initializes bias values of activation neurons in the activation layer.
/// </summary>
/// <param name="activationLayer">
/// The activation layer to initialize
/// </param>
/// <exception cref="ArgumentNullException">
/// If <c>activationLayer</c> is <c>null</c>
/// </exception>
public void Initialize(ActivationLayer activationLayer)
{
Helper.ValidateNotNull(activationLayer, "activationLayer");
int hiddenNeuronCount = 0;
foreach (IConnector targetConnector in activationLayer.TargetConnectors)
{
hiddenNeuronCount += targetConnector.TargetLayer.NeuronCount;
}
double nGuyenWidrowFactor = NGuyenWidrowFactor(activationLayer.NeuronCount, hiddenNeuronCount);
foreach (ActivationNeuron neuron in activationLayer.Neurons)
{
neuron.bias = Helper.GetRandom(-nGuyenWidrowFactor, nGuyenWidrowFactor);
}
}
public void CreateActivationLayerTest()
{
LayerFactory factory = new LayerFactory();
IActivationFunction ActivationThresHold = new ThresholdFunction();
ActivationLayer layer = (ActivationLayer)factory.CreateLayer(NETWORK_TYPE_ACTIVATION, INPUT_COUNT, ActivationThresHold);
Assert.Equal(3, layer.GetNeuronCount());
ActivationNeuron actNeuron = (ActivationNeuron)layer.GetNeuron(1);
Assert.NotNull(actNeuron);
List <ISynapse> inputs = actNeuron.FetchInputs();
Assert.NotNull(inputs);
Assert.Equal(INPUT_COUNT, inputs.Count);
ISynapse input = inputs[1];
Assert.True(input.Weight != 0);
}
/// <summary>
/// Initializes bias values of activation neurons in the activation layer.
/// </summary>
/// <param name="activationLayer">
/// The activation layer to initialize
/// </param>
/// <exception cref="ArgumentNullException">
/// If <c>activationLayer</c> is <c>null</c>
/// </exception>
public void Initialize(ActivationLayer activationLayer)
{
Helper.ValidateNotNull(activationLayer, "layer");
foreach (ActivationNeuron neuron in activationLayer.Neurons)
{
neuron.bias = constant;
}
}
/// <summary>
/// Initializes bias values of activation neurons in the activation layer.
/// </summary>
/// <param name="activationLayer">
/// The activation layer to initialize
/// </param>
/// <exception cref="ArgumentNullException">
/// If <c>activationLayer</c> is <c>null</c>
/// </exception>
public void Initialize(ActivationLayer activationLayer)
{
Helper.ValidateNotNull(activationLayer, "activationLayer");
int hiddenNeuronCount = 0;
foreach (IConnector targetConnector in activationLayer.TargetConnectors)
{
hiddenNeuronCount += targetConnector.TargetLayer.NeuronCount;
}
double nGuyenWidrowFactor = NGuyenWidrowFactor(activationLayer.NeuronCount, hiddenNeuronCount);
foreach (ActivationNeuron neuron in activationLayer.Neurons)
{
neuron.bias = Helper.GetRandom(-nGuyenWidrowFactor, nGuyenWidrowFactor);
}
}
/// <summary>
/// Initializes bias values of activation neurons in the activation layer.
/// </summary>
/// <param name="activationLayer">
/// The activation layer to initialize
/// </param>
/// <exception cref="ArgumentNullException">
/// If <c>activationLayer</c> is <c>null</c>
/// </exception>
public void Initialize(ActivationLayer activationLayer)
{
Helper.ValidateNotNull(activationLayer, "layer");
foreach (ActivationNeuron neuron in activationLayer.Neurons)
{
neuron.bias = Helper.GetRandom(minLimit, maxLimit);
}
}
/// <summary>
/// Creates a new Back Propagation Network, with the specified input and output layers. (You
/// are required to connect all layers using appropriate synapses, before using the constructor.
/// Any changes made to the structure of the network after its creation may lead to complete
/// malfunctioning)
/// </summary>
/// <param name="inputLayer">
/// The input layer
/// </param>
/// <param name="outputLayer">
/// The output layer
/// </param>
/// <exception cref="ArgumentNullException">
/// If <c>inputLayer</c> or <c>outputLayer</c> is <c>null</c>
/// </exception>
public BackpropagationNetwork(ActivationLayer inputLayer, ActivationLayer outputLayer)
: base(inputLayer, outputLayer, TrainingMethod.Supervised)
{
this.meanSquaredError = 0d;
this.isValidMSE = false;
}
