public void SoftmaxFunctionReturnsCorrectResult()
{
List<Node> nodes = new List<Node>()
{
new Node(new SoftmaxActivationFunction()),
new Node(new SoftmaxActivationFunction()),
new Node(new SoftmaxActivationFunction())
};
nodes[0].Input = 2.0;
nodes[1].Input = -1.0;
nodes[2].Input = 4.0;
Layer layer = new Layer(LayerType.Hidden, new SoftmaxActivationFunction(), 1, 3);
layer.Nodes = nodes;
double scalingFactor = Math.Exp(2 - 4) + Math.Exp(-1 - 4) + Math.Exp(4 - 4);
double result = Math.Exp(2 - 4) / scalingFactor;
List<double> inputs = new List<double>();
foreach(Node node in nodes)
{
inputs.Add(node.Input);
}
layer.ActivateNodes();
//nodes[0].Softmax(inputs.ToArray());
Console.WriteLine("expected {0}, actual {1}", result, nodes[0].ActivatedSum);
Assert.AreEqual(result, nodes[0].ActivatedSum);
}
public Layer(Layer layer)
{
m_inputsCount = layer.m_inputsCount;
m_neuronsCount = layer.m_neuronsCount;
m_function = layer.m_function;
m_neurons = new Neuron[m_neuronsCount];
for (int i = 0; i < m_neuronsCount; i++)
m_neurons[i] = new Neuron(layer[i]);
m_output = (double[])layer.m_output.Clone();
}
//public Network(int[] layersTopology, IActivationFunction[] functions)
//{
// m_layers = new Layer[layersTopology.Length];
// int layer_input = layersTopology[0];
// for (int i = 0; i < layersTopology.Length; i++)
// {
// m_layers[i] = new Layer(layer_input, layersTopology[i], functions[i]);
// layer_input = layersTopology[i];
// }
//}
public Network(int inputsCount, int[] layersTopology, double[][] thresholds, IActivationFunction[] functions)
{
m_layers = new Layer[layersTopology.Length];
int layer_input = inputsCount;
for (int i = 0; i < layersTopology.Length; i++)
{
m_layers[i] = new Layer(layer_input, layersTopology[i], thresholds[i], functions[i]);
layer_input = layersTopology[i];
}
}
public void Activate(Layer layer = null)
{
if (this.ActivationFunction == null)
{
this.ActivatedSum = this.Input;
}
else
{
this.ActivatedSum = this.ActivationFunction.Activate(this, layer);
}
}
private static void ChangeWeights(double n, Layer[] layers, int l)
{
double dw;
for (int i = 0; i < layers[l].cellCount; i++)
{
for (int j = 0; j < layers[l - 1].cellCount; j++)
{
dw = -n * layers[l].errors[i] * layers[l - 1].output[j];
layers[l].weights[j, i] += dw;
}
dw = -n * layers[l].errors[i];
layers[l].bias[i] += dw;
}
}
public void ActivateTanhSettingNodeValuesCorrectly()
{
Layer layer = new Layer(LayerType.Hidden, new HyperbolicTangentActivationFunction(), 1, 3);
layer.Nodes[0].Input = 2.0;
layer.Nodes[1].Input = -1.0;
layer.Nodes[2].Input = 4.0;
double result1 = Math.Tanh(layer.Nodes[0].Input);
double result2 = Math.Tanh(layer.Nodes[1].Input);
double result3 = Math.Tanh(layer.Nodes[2].Input);
layer.ActivateNodes();
Assert.AreEqual(result1, layer.Nodes[0].ActivatedSum);
Assert.AreEqual(result2, layer.Nodes[1].ActivatedSum);
Assert.AreEqual(result3, layer.Nodes[2].ActivatedSum);
}
public void ActivateSigmoidSettingNodeValuesCorrectly()
{
Layer layer = new Layer(LayerType.Hidden, new LogisticSigmoidActivationFunction(), 1, 3);
layer.Nodes[0].Input = 2.0;
layer.Nodes[1].Input = -1.0;
layer.Nodes[2].Input = 4.0;
double result1 = (1.0 / (1.0 + Math.Exp(-(layer.Nodes[0].Input))));
double result2 = (1.0 / (1.0 + Math.Exp(-(layer.Nodes[1].Input))));
double result3 = (1.0 / (1.0 + Math.Exp(-(layer.Nodes[2].Input))));
layer.ActivateNodes();
Assert.AreEqual(result1, layer.Nodes[0].ActivatedSum);
Assert.AreEqual(result2, layer.Nodes[1].ActivatedSum);
Assert.AreEqual(result3, layer.Nodes[2].ActivatedSum);
}
public void ActivateSoftmaxSettingNodeValuesCorrectly()
{
Layer layer = new Layer(LayerType.Hidden, new SoftmaxActivationFunction(), 1, 3);
layer.Nodes[0].Input = 2.0;
layer.Nodes[1].Input = -1.0;
layer.Nodes[2].Input = 4.0;
double scalingFactor = Math.Exp(2 - 4) + Math.Exp(-1 - 4) + Math.Exp(4 - 4);
double result1 = Math.Exp(2 - 4) / scalingFactor;
double result2 = Math.Exp(-1 - 4) / scalingFactor;
double result3 = Math.Exp(4 - 4) / scalingFactor;
layer.ActivateNodes();
Assert.AreEqual(result1, layer.Nodes[0].ActivatedSum);
Assert.AreEqual(result2, layer.Nodes[1].ActivatedSum);
Assert.AreEqual(result3, layer.Nodes[2].ActivatedSum);
}
public FeedForwardNeuralNetwork(int nrOfInputs, int nrOfOutputs, int nrHiddenLayers, int nrOfNeuronsPerHiddenLayer, float bias, float[] weights = null)
{
this.bias = bias;
this.NrOfInputs = nrOfInputs;
this.NrOfHiddenLayers = nrHiddenLayers;
this.NrOfOutputs = nrOfOutputs;
this.NrOfNeuronsPerHiddenLayer = nrOfNeuronsPerHiddenLayer;
layers = new Layer[nrHiddenLayers + 1];
layers[0] = new Layer(nrOfInputs, nrOfNeuronsPerHiddenLayer);
for (int i = 1; i < nrHiddenLayers; i++)
layers[i] = new Layer(nrOfNeuronsPerHiddenLayer, nrOfNeuronsPerHiddenLayer);
// last layer is the output layer
layers[nrHiddenLayers] = new Layer(nrOfNeuronsPerHiddenLayer, nrOfOutputs);
if (weights == null)
weights = GetRandomWeights(weights);
SetWeights(weights);
}
public void LayerConstructorCreatesCorrectNumberOfNodes()
{
Layer layer = new Layer(LayerType.Input, null, 0, 4);
Assert.AreEqual(4, layer.Nodes.Count);
}
/// <summary>
/// Create a new neural network
/// with "inputs" inputs and size of "layers"
/// layers of neurones.
/// The layer i is made with layers_desc[i] neurones.
/// The activation function of each neuron is set to default (Sigmoid with beta = 1).
/// The lerning algorithm is set to default (Back Propagation).
/// </summary>
/// <param name="inputs">Number of inputs of the network</param>
/// <param name="layers_desc">Number of neurons for each layer of the network</param>
public NeuralNetwork(int inputs, int[] layers_desc)
{
if (layers_desc.Length<1)
throw new Exception("PERCEPTRON : cannot build perceptron, it must have at least 1 layer of neurone");
if (inputs<1)
throw new Exception("PERCEPTRON : cannot build perceptron, it must have at least 1 input");
la = new BackPropagationLearningAlgorithm(this);
ni = inputs;
ActivationFunction n_act = new SigmoidActivationFunction();
layers = new Layer[layers_desc.Length];
layers[0] = new Layer(layers_desc[0], ni);
for(int i=1; i<layers_desc.Length; i++)
layers[i] = new Layer(layers_desc[i],layers_desc[i-1],n_act);
}
public void FeedForwardSetsNextLayerInputsCorrectly()
{
List<double> biases = new List<double>()
{
1,
1,
1,
1
};
List<double> weights = new List<double>()
{
1,
1,
1,
1,
1,
1,
1,
1,
1,
1,
1,
1
};
Layer input = new Layer(LayerType.Input, null, 0, 3);
Layer hidden = new Layer(LayerType.Hidden, new LogisticSigmoidActivationFunction(), 1, 4);
input.Weights = weights;
input.Biases = biases;
input.Nodes[0].Input = 1;
input.Nodes[1].Input = 2;
input.Nodes[2].Input = 3;
input.Nodes[0].ActivatedSum = 1;
input.Nodes[1].ActivatedSum = 2;
input.Nodes[2].ActivatedSum = 3;
input.ActivateNodes();
input.FeedForward(hidden);
Assert.AreEqual(7, hidden.Nodes[0].Input);
}
public void LayerConstructorThrowsIfNumberOfNodesLessThanOne()
{
Layer layer = new Layer(LayerType.Input, null, 0, 0);
}
public InputNeuron(Layer l) : base(l)
{
}
static void createNeuralNetwork(String[] words)
{
Random r = Lib.NeuralNetwork.random;
try
{
if (Char.IsDigit(words[1].ToCharArray()[0]))
{
int[] nodes = new int[words.Length - 1];
for (int i = 0; i < nodes.Length; i++)
{
nodes[i] = int.Parse(words[i + 1]);
}
nn = new Lib.NeuralNetwork(r, nodes);
}
else
{
Layer[] layers = new Layer[(words.Length - 1) / 2];
for (int i = 1; i < words.Length; i += 2)
{
if (words[i].Equals("relu"))
{
int nodes = int.Parse(words[i + 1]);
layers[(i - 1) / 2] = new ReluLayer(nodes);
}
else if (words[i].Equals("do"))
{
int nodes = int.Parse(words[i + 1]);
layers[(i - 1) / 2] = new DropoutLayer(nodes);
}
else if (words[i].Equals("lrelu"))
{
int nodes = int.Parse(words[i + 1]);
layers[(i - 1) / 2] = new Lib.Layers.LeakyReluLayer(nodes);
}
else
{
Console.WriteLine("There is no layer type like \"" + words[i] + "\"");
return;
}
}
for (int i = 0; i < layers.Length; i++)
{
if (i != layers.Length - 1)
{
layers[i].weights = new Matrix(layers[i + 1].nodes, layers[i].nodes);
layers[i].weights.randomize(r);
}
layers[i].bias = new Matrix(layers[i].nodes, 1);
layers[i].bias.randomize(r);
}
nn = new Lib.NeuralNetwork(Lib.NeuralNetwork.random, layers);
}
} catch (Exception e)
{
printError(e);
return;
}
Console.WriteLine("Neural Network was created succesfully!");
}
public void NeuralNetConstructorAssignsCorrectNumberOfWeightsToEachLayer()
{
Layer input = new Layer(LayerType.Input, null, 0, 3);
Layer hidden = new Layer(LayerType.Hidden, new LogisticSigmoidActivationFunction(), 1, 4);
Layer output = new Layer(LayerType.Output, new SoftmaxActivationFunction(), 2, 2);
input.Nodes[0].Input = 1;
input.Nodes[1].Input = 2;
input.Nodes[2].Input = 3;
input.Nodes[0].ActivatedSum = 1;
input.Nodes[1].ActivatedSum = 2;
input.Nodes[2].ActivatedSum = 3;
List<Layer> layers = new List<Layer>();
layers.Add(input);
layers.Add(hidden);
layers.Add(output);
//24 weights
List<double> weights = new List<double>()
{
1,
1,
1,
1,
1,
1,
1,
1,
1,
1,
1,
1,
1,
1,
1,
1,
1,
1,
1,
1,
1,
1,
1,
1,
1
};
//6 biases
List<double> biases = new List<double>()
{
1,
1,
1,
1,
1,
1
};
NeuralNet nn = new NeuralNet(layers, weights, biases);
Assert.AreEqual(nn.Layers[0].Nodes.Count * nn.Layers[1].Nodes.Count, nn.Layers[0].Weights.Count);
Assert.AreEqual(nn.Layers[1].Nodes.Count * nn.Layers[2].Nodes.Count, nn.Layers[1].Weights.Count);
}
public static void Main(string[] args)
{
//// inputs
Byte[] inputByte = MyMath.GetRandomBytes(2);
double n = 0.1;
int layersCount = 2;
Layer[] layers = new Layer[layersCount + 1];
layers[0] = new Layer();
layers[0].cellCount = inputByte.Length;
layers[0].output = new double[inputByte.Length];
for (int i = 0; i < inputByte.Length; i++)
layers[0].output[i] = inputByte[i] / 255.0;
layers[1] = new Layer();
layers[1].cellCount = 3;
layers[1].function = TransferFunction.Linear;
layers[1].bias = MyMath.GetRandomDoubles(layers[1].cellCount);
layers[1].weights = MyMath.GetRandom(layers[0].cellCount, layers[1].cellCount);
layers[2] = new Layer();
layers[2].cellCount = 1;
layers[2].function = TransferFunction.Linear;
layers[2].bias = MyMath.GetRandomDoubles(layers[2].cellCount);
layers[2].weights = MyMath.GetRandom(layers[1].cellCount, layers[2].cellCount);
for (int z = 0; z < 50; z++)
{
//feed forward all
for (int i = 1; i < layers.Length; i++)
layers[i].FeedForward(layers[i - 1].output);
//backpropagate last layer
int l = layers.Length - 1;
double[] targetOutput = new double[layers[l].cellCount];
//calculate errors
layers[l].errors = new double[layers[l].cellCount];
for (int i = 0; i < targetOutput.Length; i++)
layers[l].errors[i] = layers[l].output[i] - targetOutput[i];
Console.WriteLine("FeedForward error: " + Math.Round(MyMath.Sum(layers[l].errors),4));
//calculate weights
ChangeWeights(n, layers, l);
//backpropagate other layers
for (l = layers.Length - 2; l > 0; l--)
{
//calculate errors
layers[l].errors = new double[layers[l].cellCount];
for (int i = 0; i < layers[l].cellCount; i++)
for (int j = 0; j < layers[l + 1].cellCount; j++)
layers[l].errors[i] = layers[l + 1].errors[j] * layers[l + 1].weights[i, j];
//calculate weights
ChangeWeights(n, layers, l);
}
}
Console.ReadKey();
}
public void LayerConstructorAllowsLayersWithActivationTypeNone()
{
Layer input = new Layer(LayerType.Input, null, 0, 4);
Layer hidden = new Layer(LayerType.Hidden, null, 1, 4);
Layer output = new Layer(LayerType.Output, null, 2, 4);
}
public void FeedForwardThrowsIfLayerActivatedSumsAreNull()
{
Layer hidden = new Layer(LayerType.Hidden, new LogisticSigmoidActivationFunction(), 1, 4);
Layer output = new Layer(LayerType.Output, new SoftmaxActivationFunction(), 2, 3);
hidden.Nodes[0].Input = 1;
hidden.Nodes[1].Input = 2;
hidden.Nodes[2].Input = 3;
hidden.Nodes[3].Input = 4;
hidden.Biases = new List<double>() { 1, 1, 1 };
hidden.Weights = new List<double>() {1,1,1,1,1,1,1,1,1,1,1,1};
hidden.FeedForward(output);
}
public void FeedForwardThrowsIfBiasesDoesNotEqualNextLayerNodes()
{
Layer input = new Layer(LayerType.Input, null, 0, 5);
Layer hidden = new Layer( LayerType.Hidden, new LogisticSigmoidActivationFunction(), 1, 4);
input.Nodes[0].Input = 1;
input.Nodes[1].Input = 2;
input.Nodes[2].Input = 3;
input.Nodes[3].Input = 4;
input.Nodes[0].ActivatedSum = 1;
input.Nodes[1].ActivatedSum = 2;
input.Nodes[2].ActivatedSum = 3;
input.Nodes[3].ActivatedSum = 4;
input.ActivateNodes();
input.FeedForward(hidden);
}
public void NeuralNetConstructorOrdersLayersAscendingByLayerOrder()
{
Layer input = new Layer(LayerType.Input, null, 0, 3);
Layer output = new Layer(LayerType.Output, new SoftmaxActivationFunction(), 2, 2);
Layer hidden = new Layer(LayerType.Hidden, new LogisticSigmoidActivationFunction(), 1, 4);
input.Nodes[0].Input = 1;
input.Nodes[1].Input = 2;
input.Nodes[2].Input = 3;
input.Nodes[0].ActivatedSum = 1;
input.Nodes[1].ActivatedSum = 2;
input.Nodes[2].ActivatedSum = 3;
List<Layer> layers = new List<Layer>();
layers.Add(input);
layers.Add(output);
layers.Add(hidden);
//24 weights
List<double> weights = new List<double>()
{
1,
1,
1,
1,
1,
1,
1,
1,
1,
1,
1,
1,
1,
1,
1,
1,
1,
1,
1,
1,
1,
1,
1,
1,
1
};
//6 biases
List<double> biases = new List<double>()
{
1,
1,
1,
1,
1,
1
};
NeuralNet nn = new NeuralNet(layers, weights, biases);
//nn.Run();
Assert.AreEqual(0, nn.Layers[0].LayerOrder);
Assert.AreEqual(1, nn.Layers[1].LayerOrder);
Assert.AreEqual(2, nn.Layers[2].LayerOrder);
}
public void LayerConstructorThrowsIfLayerTypeInputAndActivationFunctionNotNull()
{
Layer layer = new Layer(LayerType.Input, new LogisticSigmoidActivationFunction(), 0, 5);
}
public void LayerConstructorThrowsIfLayerTypeInputAndLayerOrderNotZero()
{
Layer layer = new Layer(LayerType.Input, null, 1, 5);
}
public void LayerConstructorThrowsIfLayerTypeNotInputAndLayerOrderZero()
{
Layer layer = new Layer(LayerType.Hidden, new LogisticSigmoidActivationFunction(), 0, 5);
}
public Network(Network network)
{
m_layers = new Layer[network.m_layers.Length];
for (int i = 0; i < m_layers.Length; i++)
m_layers[i] = new Layer(network[i]);
}
public void FeedForward(Layer nextLayer)
{
if (this.Biases.Count != nextLayer.Nodes.Count || this.Weights.Count != nextLayer.Nodes.Count * this.Nodes.Count)
{
throw new Exception("Wrong number of weights or biases.");
}
foreach(Node node in this.Nodes)
{
if (node.ActivatedSum == null)
{
throw new Exception("Nodes must be activated before FeedForward is called.");
}
}
for(int i = 0; i < nextLayer.Nodes.Count; i++)
{
double input = 0;
for(int p = 0; p < this.Nodes.Count; p++)
{
input += Convert.ToDouble(this.Nodes[p].ActivatedSum) * this.Weights[(i * this.Nodes.Count) + p];
}
input += this.Biases[i];
nextLayer.Nodes[i].Input = input;
}
}
