private static NeuralNetwork InitializeNeuralNetwork(int seed)
{
Random random = new Random(seed == 0 ? new Random().Next() : seed);
float RandomWeight() => (float)(random.NextDouble() * 2 - 1);
Layer prevLayer;
InputLayer li = new InputLayer(3, 5);
prevLayer = li;
ConvolutionalLayer l0 = new ConvolutionalLayer(8, 2, 1, 0, prevLayer, ActivationFunctions.ReLU(true));
prevLayer = l0;
prevLayer.InitializeWeights(RandomWeight);
ConvolutionalLayer l2 = new ConvolutionalLayer(16, 2, 1, 0, prevLayer, ActivationFunctions.ReLU(true));
prevLayer = l2;
prevLayer.InitializeWeights(RandomWeight);
FullyConnectedLayer l7 = new FullyConnectedLayer(16, prevLayer, ActivationFunctions.Sigmoid(1));
prevLayer = l7;
prevLayer.InitializeWeights(RandomWeight);
FullyConnectedLayer l8 = new FullyConnectedLayer(10, prevLayer, ActivationFunctions.SoftMax(1));
prevLayer = l8;
prevLayer.InitializeWeights(RandomWeight);
return(new NeuralNetwork(li, l0, l2, l7, l8));
}
/// <summary>
/// This method calculates the results of the network at the output layer
/// </summary>
/// <param name="input">input layer data</param>
/// <param name="numOfFeatures">number of input neurons</param>
/// <param name="output">output parameter to store outputs at the output layer</param>
/// <param name="outputSum">output sum of the last layer.</param>
private void CalculateResultatOutputlayer(double[] input, int numOfFeatures, bool softmax, out double[] output, out double[] outputSum)
{
output = new double[m_OutputLayerNeurons];
outputSum = new double[m_OutputLayerNeurons];
int numOfHiddenNeuronsInLastHiddenLayer = m_HiddenLayerNeurons[m_HiddenLayerNeurons.Length - 1];
for (int j = 0; j < m_OutputLayerNeurons; j++)
{
outputSum[j] = 0.0;
for (int i = 0; i < numOfHiddenNeuronsInLastHiddenLayer; i++)
{
outputSum[j] += m_Weights[m_HiddenLayerNeurons.Length][j, i] * input[i];
}
outputSum[j] += m_Biases[m_HiddenLayerNeurons.Length][j];
if (softmax == false)
{
output[j] = ActivationFunctions.Sigmoid(outputSum[j]);
}
else
{
// Do nothing
}
}
if (softmax == true)
{
output = ActivationFunctions.SoftMaxClassifier(outputSum);
}
}
public float activation(bool derivate = false)
{
if (type == PerceptronType.Type.input || type == PerceptronType.Type.bias)
{
return(state);
}
if (activation_type == ActivationType.Type.sigmoid)
{
return(ActivationFunctions.Sigmoid(state, derivate));
}
else if (activation_type == ActivationType.Type.relu)
{
return(ActivationFunctions.RelU(state, derivate));
}
else if (activation_type == ActivationType.Type.tanh)
{
return(ActivationFunctions.TanH(state, derivate));
}
else if (activation_type == ActivationType.Type.identity)
{
return(ActivationFunctions.Identity(state, derivate));
}
else if (activation_type == ActivationType.Type.lrelu)
{
return(ActivationFunctions.LeakyReLU(state, derivate));
}
else
{
return(ActivationFunctions.Sigmoid(state, derivate));
}
}
private static NeuralNetwork InitializeNeuralNetwork(int seed)
{
Random random = new Random(seed == 0 ? new Random().Next() : seed);
float RandomWeight() => (float)(random.NextDouble() * 2 - 1);
Layer prevLayer;
InputLayer li = new InputLayer(28, 28);
prevLayer = li;
ConvolutionalLayer l0 = new ConvolutionalLayer(15, 5, 1, 0, prevLayer, ActivationFunctions.ReLU(true));
prevLayer = l0;
prevLayer.InitializeWeights(RandomWeight);
MaxPoolingLayer l1 = new MaxPoolingLayer(2, 2, prevLayer);
prevLayer = l1;
ConvolutionalLayer l2 = new ConvolutionalLayer(30, 4, 1, 0, prevLayer, ActivationFunctions.ReLU(true));
prevLayer = l2;
prevLayer.InitializeWeights(RandomWeight);
MaxPoolingLayer l3 = new MaxPoolingLayer(3, 2, prevLayer);
prevLayer = l3;
ConvolutionalLayer l4 = new ConvolutionalLayer(45, 2, 2, 0, prevLayer, ActivationFunctions.ReLU(true));
prevLayer = l4;
prevLayer.InitializeWeights(RandomWeight);
MaxPoolingLayer l5 = new MaxPoolingLayer(2, 1, prevLayer);
prevLayer = l5;
FullyConnectedLayer l6 = new FullyConnectedLayer(64, prevLayer, ActivationFunctions.Sigmoid(1));
prevLayer = l6;
prevLayer.InitializeWeights(RandomWeight);
FullyConnectedLayer l7 = new FullyConnectedLayer(32, prevLayer, ActivationFunctions.Sigmoid(1));
prevLayer = l7;
prevLayer.InitializeWeights(RandomWeight);
FullyConnectedLayer l8 = new FullyConnectedLayer(10, prevLayer, ActivationFunctions.SoftMax(1));
prevLayer = l8;
prevLayer.InitializeWeights(RandomWeight);
return(new NeuralNetwork(li, l0, l1, l2, l3, l4, l5, l6, l7, l8));
}
internal static void Run()
{
NeuralNetwork nn = InitializeNeuralNetwork(0, ActivationFunctions.Sigmoid(1));
CalculateXOR(nn);
TrainXOR(nn, 0.5f, 1000000);
CalculateXOR(nn);
}
public void TestSigmoid()
{
Matrix matrix = new Matrix(new double[, ] {
{ -1, 1, 2 }
});
Matrix matrix1 = new Matrix(new double[, ] {
{ 0.26894142, 0.73105858, 0.88079708 }
});
matrix = ActivationFunctions.Sigmoid(matrix);
Assert.IsTrue(matrix.ToString() == matrix1.ToString());
}
public override Volume ForwardPass(Volume volume)
{
var raw = volume.WeightsRaw;
var destRaw = OutVolume.WeightsRaw;
for (var i = 0; i < raw.Length; i++)
{
destRaw[i] = ActivationFunctions.Sigmoid(raw[i]);
}
return(OutVolume);
}
public virtual double CalculateValue()
{
double weightedSum = 0;
foreach (Synapse synapse in InputSynapses)
{
weightedSum += synapse.InputNeuron.Value * synapse.Weight;
}
Value = ActivationFunctions.Sigmoid(weightedSum + Bias);
return(Value);
}
void Calculation()
{
hiddenLayer1ActivationWithoutSig = matrixOperations.MatrixMultiplication(weightIH1, HL1N, IN, inputActivation, CM);
for (int i = 0; i < HL1N; i++)
{
hiddenLayer1Activation[i, 0] = activationFunctions.Sigmoid(hiddenLayer1ActivationWithoutSig[i, 0]);
hiddenLayer1ActivationTranspose[0, i] = hiddenLayer1Activation[i, 0];
}
hiddenLayer2ActivationWithoutSig = matrixOperations.MatrixMultiplication(weightH1H2, HL2N, HL1N, hiddenLayer1Activation, CM);
for (int i = 0; i < HL2N; i++)
{
hiddenLayer2Activation[i, 0] = activationFunctions.Sigmoid(hiddenLayer2ActivationWithoutSig[i, 0]);
hiddenLayer2ActivationTranspose[0, i] = hiddenLayer2Activation[i, 0];
}
output = matrixOperations.MatrixMultiplication(weightH2O, ON, HL2N, hiddenLayer2Activation, CM);
for (int i = 0; i < ON; i++)
{
outputActivation[i, 0] = activationFunctions.Sigmoid(output[i, 0]);
outputActivationTranspose[0, i] = outputActivation[i, 0];
}
errorOutput = matrixOperations.MatrixMultiplication(errorOutput, ON, CM, outputActivationTranspose, ON);
//errorOutput = matrixOperations.MatrixMultiplication(errorOutput, ON, CM, matrixOperations.MatrixSubstraction(matrixOperations.Ones(ON, CM), ON, CM, output), ON);
//deltaweightH2O = matrixOperations.MatrixMultiplication(errorOutput, ON, CM, hiddenLayer1ActivationTranspose, ON);
}
public void TestThreeLayerNeuralNetwork()
{
Matrix inputMatrix = new Matrix(new double[, ] {
{ 1, 0.5 }
});
//1x2 * ?? = 1x3
// 1x2 * 2x3 =1x3;
Matrix w1 = new Matrix(new double[, ] {
{ 0.1, 0.3, 0.5 },
{ 0.2, 0.4, 0.6 }
});
Matrix b1 = new Matrix(new double[, ] {
{ 0.1, 0.2, 0.3 },
});
//1x3 * 3 *2
Matrix w2 = new Matrix(new double[, ] {
{ 0.1, 0.4 },
{ 0.2, 0.5 },
{ 0.3, 0.6 }
});
Matrix b2 = new Matrix(new double[, ] {
{ 0.1, 0.2 },
});
Matrix w3 = new Matrix(new double[, ] {
{ 0.1, 0.3 },
{ 0.2, 0.4 },
});
Matrix b3 = new Matrix(new double[, ] {
{ 0.1, 0.2 },
});
inputMatrix = Matrix.Dot(inputMatrix, w1) + b1;
inputMatrix = ActivationFunctions.Sigmoid(inputMatrix);
inputMatrix = Matrix.Dot(inputMatrix, w2) + b2;
inputMatrix = ActivationFunctions.Sigmoid(inputMatrix);
inputMatrix = Matrix.Dot(inputMatrix, w3) + b3;
// inputMatrix = ActivationFunctions.Sigmoid(inputMatrix);
}
static EliminationGA()
{
Ffann = new FFANN(new[] { 2, 8, 3 }, new[] { ActivationFunctions.Sigmoid(), ActivationFunctions.Sigmoid() });
ChromosomeSize = Ffann.WeightCount();
Evaluator = new Evaluator(DataPath, Ffann);
Selection = new TournamentSelection(TournamentSize);
var discreteRecombination = new DiscreteRecombination();
var simpleArithmetic = new SimpleArithmeticRecombination();
var singleArithmetic = new SingleArithmeticRecombination();
Crossover = new MixedCrossover(discreteRecombination, simpleArithmetic, singleArithmetic);
var gausMutation = new GausMutation(Sigma1, P1);
var newGausMutation = new NewGausMutation(Sigma2, P2);
Mutation = new MixedMutation(gausMutation, newGausMutation, P);
}
public double[] CalculateOutput(double x, double y, double[] parameters)
{
var paramIdx = 0;
var outputs = new double[_layers.Length][];
const int likenessLayerIdx = 1;
// 1 - likeness neuron layer
outputs[likenessLayerIdx] = new double[_layers[likenessLayerIdx]];
// Activate the likeness layer
for (var likenessNeuronIdx = 0; likenessNeuronIdx < _layers[likenessLayerIdx]; likenessNeuronIdx++)
{
var weightX = parameters[paramIdx++];
var varianceX = parameters[paramIdx++];
var weightY = parameters[paramIdx++];
var varianceY = parameters[paramIdx++];
var xy = new[] { x, y };
var w = new[] { weightX, weightY };
var s = new[] { varianceX, varianceY };
outputs[likenessLayerIdx][likenessNeuronIdx] = ActivationFunctions.Likeness(xy, w, s);
}
// Pass through the hidden layer
for (var hiddenLayerIdx = 2; hiddenLayerIdx < _layers.Length; hiddenLayerIdx++)
{
var hidden = _layers[hiddenLayerIdx];
outputs[hiddenLayerIdx] = new double[hidden];
var previousOutput = outputs[hiddenLayerIdx - 1];
// Pass through every neuron
for (var hiddenNeuronIdx = 0; hiddenNeuronIdx < hidden; hiddenNeuronIdx++)
{
// Collect previous outputs
// Bias + previous output
var output = parameters[paramIdx++] + previousOutput.Sum(o => o * parameters[paramIdx++]);
outputs[hiddenLayerIdx][hiddenNeuronIdx] = ActivationFunctions.Sigmoid(output);
}
}
return(outputs[_layers.Length - 1]);
}
//Сигмоида
private void button3_Click(object sender, EventArgs e)
{
tensor = ActivationFunctions.Sigmoid(tensor - tensor.Mean(), 9);// Гамма-фильтрация
pictureBox2.Image = ImgConverter.ToBitmap(tensor);
Decomposition();
}
public Matrix Forward(Matrix x, Matrix t = null)
{
outMatrix = ActivationFunctions.Sigmoid(x);
return(outMatrix);
}
public double ActivationFunction_Sigmoid_ExpectedResults(double value)
{
return(ActivationFunctions.Sigmoid(value));
}
