public void TrainBasicNetworksWithMomentumSortofWell()
{
var input = new Layer(5, Array.Empty <Layer>(), ActivationFunctionType.Linear, InitialisationFunctionType.None);
var h1 = new Layer(10, new Layer[] { input }, ActivationFunctionType.Linear, InitialisationFunctionType.HeUniform);
var output = new Layer(5, new Layer[] { h1 }, ActivationFunctionType.Sigmoid, InitialisationFunctionType.HeEtAl);
output.AddMomentumRecursively();
output.Initialise(new Random());
var learningRate = 0.01;
var momentum = 0.9;
for (var i = 0; i < 1000; i++)
{
output.NegativeSample(0, 0, false, ErrorFunctionType.CrossEntropy, learningRate, momentum);
output.NegativeSample(1, 1, false, ErrorFunctionType.CrossEntropy, learningRate, momentum);
output.NegativeSample(2, 2, true, ErrorFunctionType.CrossEntropy, learningRate, momentum);
output.NegativeSample(3, 3, false, ErrorFunctionType.CrossEntropy, learningRate, momentum);
output.NegativeSample(4, 4, false, ErrorFunctionType.CrossEntropy, learningRate, momentum);
}
Assert.True(output.GetResult(0, 0) < 0.05);
Assert.True(output.GetResult(1, 1) < 0.05);
Assert.True(output.GetResult(2, 2) > 0.95);
Assert.True(output.GetResult(3, 3) < 0.05);
Assert.True(output.GetResult(4, 4) < 0.05);
}
public void TrainFilterToFeatureSortOfWell()
{
var inputLayer = new Layer2D((3, 3), Array.Empty <Layer>(), ActivationFunctionType.RELU, InitialisationFunctionType.None);
var filter1 = new Filter2D(new[] { inputLayer }, (2, 2), ActivationFunctionType.RELU, InitialisationFunctionType.GlorotUniform);
var filter2 = new Filter2D(new[] { inputLayer }, (2, 2), ActivationFunctionType.RELU, InitialisationFunctionType.GlorotUniform);
var filter3 = new Filter2D(new[] { inputLayer }, (2, 2), ActivationFunctionType.RELU, InitialisationFunctionType.GlorotUniform);
var output = new Layer(1, new Layer[] { filter1, filter2, filter3 }, ActivationFunctionType.Sigmoid, InitialisationFunctionType.HeEtAl);
output.AddMomentumRecursively();
output.Initialise(new Random());
// feature is horizontal line
var inputMatch1 = new double[] { 1, 1, 1, 0, 0, 0, 0, 0, 0 };
var inputMatch2 = new double[] { 0, 0, 0, 1, 1, 1, 0, 0, 0 };
var inputMatch3 = new double[] { 0, 0, 0, 0, 0, 0, 1, 1, 1 };
// vertical lines
var inputNoMatch1 = new double[] { 1, 0, 0, 1, 0, 0, 1, 0, 0 };
var inputNoMatch2 = new double[] { 0, 1, 0, 0, 1, 0, 0, 1, 0 };
var inputNoMatch3 = new double[] { 0, 0, 1, 0, 0, 1, 0, 0, 1 };
for (var i = 0; i < 10000; i++)
{
output.Backpropagate(inputMatch1, new double[] { 1 }, ErrorFunctionType.CrossEntropy, 0.01, 0.9);
output.Backpropagate(inputMatch2, new double[] { 1 }, ErrorFunctionType.CrossEntropy, 0.01, 0.9);
output.Backpropagate(inputMatch3, new double[] { 1 }, ErrorFunctionType.CrossEntropy, 0.01, 0.9);
output.Backpropagate(inputNoMatch1, new double[] { 0 }, ErrorFunctionType.CrossEntropy, 0.01, 0.9);
output.Backpropagate(inputNoMatch2, new double[] { 0 }, ErrorFunctionType.CrossEntropy, 0.01, 0.9);
output.Backpropagate(inputNoMatch3, new double[] { 0 }, ErrorFunctionType.CrossEntropy, 0.01, 0.9);
}
output.CalculateOutputs(inputMatch2);
output.CalculateOutputs(inputMatch1);
Assert.True(output.Nodes[0].Output > 0.95);
output.CalculateOutputs(inputMatch2);
Assert.True(output.Nodes[0].Output > 0.95);
output.CalculateOutputs(inputMatch3);
Assert.True(output.Nodes[0].Output > 0.95);
output.CalculateOutputs(inputNoMatch1);
Assert.True(output.Nodes[0].Output < 0.05);
output.CalculateOutputs(inputNoMatch2);
Assert.True(output.Nodes[0].Output < 0.05);
output.CalculateOutputs(inputNoMatch3);
Assert.True(output.Nodes[0].Output < 0.05);
}
public void ReduceDimensions(IEnumerable <IEmbedding> embeddings)
{
// TODO: make this work
var embeddingsList = embeddings.ToList();
var probabilityMatrix = CalculateProbabilityMatrix(embeddingsList, _distanceFunctionType, _perplexity);
var inputLayer = new Layer(embeddingsList.Count, Array.Empty <Layer>(), ActivationFunctionType.Linear, InitialisationFunctionType.None, false);
var outputLayer = new Layer(_dimensions, new[] { inputLayer }, ActivationFunctionType.Linear, InitialisationFunctionType.RandomGuassian, false);
outputLayer.AddMomentumRecursively();
outputLayer.Initialise(new Random());
OptimiseSNE(probabilityMatrix, outputLayer);
var results = new List <IEmbedding>();
for (var i = 0; i < embeddingsList.Count; i++)
{
var inputNode = inputLayer.Nodes[i];
var vector = outputLayer.Nodes.Select(outputNode => outputNode.Weights[inputNode].Value);
embeddingsList[i].Vector = vector.ToArray();
}
}