static void ThirdNN()
{
var r = new Random();
var data = new Tensor((Matrix) new double[, ] {
{ 0, 0 }, { 0, 1 }, { 1, 0 }, { 1, 1 }
}, true);
var target = new Tensor((Matrix) new double[, ] {
{ 0 }, { 1 }, { 0 }, { 1 }
}, true);
var seq = new Sequential();
seq.Layers.Add(new Linear(2, 3, r));
seq.Layers.Add(new Linear(3, 1, r));
var sgd = new StochasticGradientDescent(seq.Parameters, 0.1f);
for (var i = 0; i < 10; i++)
{
var pred = seq.Forward(data);
var diff = pred.Sub(target);
var loss = diff.Mul(diff).Sum(AxisZero.vertical);
loss.Backward(new Tensor(Matrix.Ones(loss.Data.X, loss.Data.Y)));
sgd.Step();
Console.WriteLine($"Epoch: {i} Loss: {loss}");
}
}
public void ShouldTrainUsingRegularizationParam()
{
var stochasticGradientDescent = new StochasticGradientDescent(new CrossEntropy(), _layers, 3000, 4, 5D, 0.01D);
stochasticGradientDescent.Train(_trainingData);
var neuralNetwork = new NeuralNetwork(_layers, _layers.First().PrimaryNeuronsCount);
var result1 = neuralNetwork.Run(_trainingData[0].Inputs);
result1.Should().HaveCount(1);
Math.Round(result1[0], 10).Should().Be(0.0285179059D);
var result2 = neuralNetwork.Run(_trainingData[1].Inputs);
result2.Should().HaveCount(1);
Math.Round(result2[0], 10).Should().Be(0.9714820792D);
var result3 = neuralNetwork.Run(_trainingData[2].Inputs);
result3.Should().HaveCount(1);
Math.Round(result3[0], 10).Should().Be(0.9714820797D);
var result4 = neuralNetwork.Run(_trainingData[3].Inputs);
result4.Should().HaveCount(1);
Math.Round(result4[0], 10).Should().Be(0.0285163624D);
}
static void FourthNN()
{
var r = new Random();
var data = new Tensor((Matrix) new double[, ] {
{ 0, 0 }, { 0, 1 }, { 1, 0 }, { 1, 1 }
}, true);
var target = new Tensor((Matrix) new double[, ] {
{ 0 }, { 1 }, { 0 }, { 1 }
}, true);
var seq = new Sequential();
seq.Layers.Add(new Linear(2, 3, r));
seq.Layers.Add(new Linear(3, 1, r));
var sgd = new StochasticGradientDescent(seq.Parameters, 0.1f);
var mse = new MeanSquaredError();
for (var i = 0; i < 10; i++)
{
var pred = seq.Forward(data);
var loss = mse.Forward(pred, target);
loss.Backward(new Tensor(Matrix.Ones(loss.Data.X, loss.Data.Y)));
sgd.Step();
Console.WriteLine($"Epoch: {i} Loss: {loss}");
}
}
public void Run()
{
Console.WriteLine("Building random neural network");
var layers = new ILayer[] { new FullyConnectedLayer(new Sigmoid(), 30, 784), new FullyConnectedLayer(new Sigmoid(), 10, 30) };
Randomiser.Randomise(layers);
Console.WriteLine("Evaluating untrained neural network");
var untrainedAccuracy = Statistics.GetAccuracyByMax(_validationData, new NeuralNetwork(layers, 784));
Console.WriteLine($"Untrained network accuracy: {untrainedAccuracy.ToString("N2")}%");
var stochasticGradientDescent = new StochasticGradientDescent(new CrossEntropy(), layers, 1, 20, 1D, 0D);
var maxAccuracy = 0D;
for (int i = 0; i < _epochs; i++)
{
Console.WriteLine($"Epoch {i + 1} started");
var trainingLength = Statistics.GetTrainingLength(stochasticGradientDescent, _trainingData);
var trainingAccuracy = Statistics.GetAccuracyByMax(_validationData, new NeuralNetwork(layers, 784));
Console.WriteLine($"Results after epoch {i + 1}:");
Console.WriteLine($"Training length in miliseconds: {trainingLength}, Accuracy: {trainingAccuracy.ToString("N2")}%");
if (maxAccuracy < trainingAccuracy)
{
maxAccuracy = trainingAccuracy;
}
}
Console.WriteLine($"End of training. Best accuracy {maxAccuracy.ToString("N2")}%");
}
public void ShouldTrainUsingQuadraticCostFunction()
{
var stochasticGradientDescent = new StochasticGradientDescent(new Quadratic(), _layers, 3000, 4, 5D, 0);
stochasticGradientDescent.Train(_trainingData);
var neuralNetwork = new NeuralNetwork(_layers, _layers.First().PrimaryNeuronsCount);
var result1 = neuralNetwork.Run(_trainingData[0].Inputs);
result1.Should().HaveCount(1);
Math.Round(result1[0], 10).Should().Be(0.0245579310D);
var result2 = neuralNetwork.Run(_trainingData[1].Inputs);
result2.Should().HaveCount(1);
Math.Round(result2[0], 10).Should().Be(0.9661695582D);
var result3 = neuralNetwork.Run(_trainingData[2].Inputs);
result3.Should().HaveCount(1);
Math.Round(result3[0], 10).Should().Be(0.9852113647D);
var result4 = neuralNetwork.Run(_trainingData[3].Inputs);
result4.Should().HaveCount(1);
Math.Round(result4[0], 10).Should().Be(0.0320611480D);
}
/// <summary>
/// Initializes a new Optimizer using the default values.
/// <param name="theta">Theta to optimize.</param>
/// <param name="maxIterations">Maximum number of iterations.</param>
/// <param name="learningRate">Learning Rate (alpha) (Optional).</param>
/// <param name="momentum">Momentum parameter for use in accelerated methods (Optional).</param>
/// <param name="optimizationMethod">Type of optimization method to use (Optional).</param>
/// <param name="optimizer">An external typed optimization method to use (Optional).</param>
/// </summary>
public Optimizer(Vector theta, int maxIterations, double learningRate = 1.0, double momentum = 0.9,
OptimizationMethods optimizationMethod = OptimizationMethods.StochasticGradientDescent, OptimizationMethod optimizer = null)
{
this.Completed = false;
if (optimizationMethod != OptimizationMethods.External)
{
switch (optimizationMethod)
{
case OptimizationMethods.FastGradientDescent: optimizer = new FastGradientDescent()
{
Momentum = momentum
}; break;
case OptimizationMethods.StochasticGradientDescent: optimizer = new StochasticGradientDescent(); break;
case OptimizationMethods.NAGDescent: optimizer = new NAGDescent()
{
Momentum = momentum
}; break;
}
}
this.OpimizationMethod = optimizer;
this.Properties = new OptimizerProperties()
{
Iteration = 0,
MaxIterations = maxIterations,
Cost = double.MaxValue,
Gradient = Vector.Zeros(theta.Length),
Theta = theta,
LearningRate = learningRate,
Momentum = momentum
};
}
public void ShouldTrainUsingCrossEntropyCostFunction()
{
var stochasticGradientDescent = new StochasticGradientDescent(new CrossEntropy(), _layers, 3000, 4, 5D, 0);
stochasticGradientDescent.Train(_trainingData);
var neuralNetwork = new NeuralNetwork(_layers, _layers.First().PrimaryNeuronsCount);
var result1 = neuralNetwork.Run(_trainingData[0].Inputs);
result1.Should().HaveCount(1);
Math.Round(result1[0], 10).Should().Be(0.0005468953D);
var result2 = neuralNetwork.Run(_trainingData[1].Inputs);
result2.Should().HaveCount(1);
Math.Round(result2[0], 10).Should().Be(0.9993728892D);
var result3 = neuralNetwork.Run(_trainingData[2].Inputs);
result3.Should().HaveCount(1);
Math.Round(result3[0], 10).Should().Be(0.9994636693D);
var result4 = neuralNetwork.Run(_trainingData[3].Inputs);
result4.Should().HaveCount(1);
Math.Round(result4[0], 10).Should().Be(0.0008765251D);
}
static void SecondNN()
{
var r = new Random();
var data = new Tensor((Matrix) new double[, ] {
{ 0, 0 }, { 0, 1 }, { 1, 0 }, { 1, 1 }
}, true);
var target = new Tensor((Matrix) new double[, ] {
{ 0 }, { 1 }, { 0 }, { 1 }
}, true);
var weights = new List <Tensor> ();
weights.Add(new Tensor(Matrix.Random(2, 3, r), true));
weights.Add(new Tensor(Matrix.Random(3, 1, r), true));
var sgd = new StochasticGradientDescent(weights, 0.1f);
for (var i = 0; i < 10; i++)
{
var pred = data.MatMul(weights[0]).Add(weights[1]);
var diff = pred.Sub(target);
var loss = diff.Mul(diff).Sum(AxisZero.vertical);
loss.Backward(new Tensor(Matrix.Ones(loss.Data.X, loss.Data.Y)));
sgd.Step();
Console.WriteLine($"Epoch: {i} Loss: {loss}");
}
}
private void Learn(double[][] input, double[] output)
{
//LinearDualCoordinateDescent
//StochasticGradientDescent
//ProbabilisticDualCoordinateDescent
var teacher = new StochasticGradientDescent()
{
//Loss = Loss.L1,
//Complexity = 1000,
//Tolerance = .1
};
_svm = teacher.Learn(input, output);
}
// Start is called before the first frame update
IEnumerator Start()
{
var r = new System.Random(2);
var x = (Matrix) new double[1000, 1];
Matrix.MatrixLoop((i, j) =>
{
x[i, 0] = i;
}, x.X, x.Y);
var y = (Matrix) new double[1000, 1];
Matrix.MatrixLoop((i, j) =>
{
y[i, 0] = i * 12 + 15 + r.Next(10);
}, x.X, x.Y);
// var x = new double[,] { { 0, 0 }, { 0, 1 }, { 1, 0 }, { 1, 1 } };
// var y = new double[,] { { 0 }, { 1 }, { 0 }, { 1 } };
var X = new Tensor(x, true);
var Y = new Tensor(y, true);
var seq = new Sequential();
seq.Layers.Add(new Linear(1, 1, r));
var sgd = new StochasticGradientDescent(seq.Parameters, 0.001);
var mse = new MeanSquaredError();
for (var i = 0; i < 10000; i++)
{
yield return(null);
var pred = seq.Forward(X);
print(pred.Data.Size);
var loss = mse.Forward(pred, Y);
loss.Backward();
sgd.Step();
print($"Epoch: {i} Loss: {loss.Data[0, 0]}");
print(Y);
print(pred);
}
print(seq.Forward(new Tensor(x)));
}
private static void Train(BasicNetwork network, BasicMLDataSet trainingSet)
{
var propagation = new StochasticGradientDescent(network, trainingSet);
var epoch = 1;
for (var i = 0; i < 200; i++)
{
propagation.Iteration(1000);
Console.WriteLine($"Epoch: {epoch} Error: {propagation.Error}");
epoch++;
}
propagation.FinishTraining();
}
private void Start()
{
r = new System.Random(seed);
seq = new Sequential();
seq.Layers.Add(new Linear(4, 100, r));
seq.Layers.Add(new ReLuLayer());
seq.Layers.Add(new Linear(100, 10, r));
seq.Layers.Add(new ReLuLayer());
seq.Layers.Add(new Linear(10, 1, r));
sgd = new StochasticGradientDescent(seq.Parameters, learningRate);
mse = new MeanSquaredError();
cartPole = GetComponent <DirectMovementCartPole>();
}
public void LearnTest()
{
double[][] inputs =
{
new double[] { -1, -1 },
new double[] { -1, 1 },
new double[] { 1, -1 },
new double[] { 1, 1 }
};
int[] xor =
{
-1,
1,
1,
-1
};
var kernel = new Polynomial(2, 0.0);
double[][] augmented = new double[inputs.Length][];
for (int i = 0; i < inputs.Length; i++)
{
augmented[i] = kernel.Transform(inputs[i]);
}
// Create the Least Squares Support Vector Machine teacher
var learn = new StochasticGradientDescent()
{
LearningRate = 1e-3
};
// Run the learning algorithm
var svm = learn.Learn(augmented, xor);
bool[] predicted = svm.Decide(augmented);
double error = new ZeroOneLoss(xor).Loss(predicted);
Assert.AreEqual(0, error);
int[] output = augmented.Apply(p => Math.Sign(svm.Compute(p)));
for (int i = 0; i < output.Length; i++)
{
Assert.AreEqual(System.Math.Sign(xor[i]), System.Math.Sign(output[i]));
}
}
public void ComputeGradient()
{
Random random = new Random();
StochasticGradientDescent <char> trainer = new StochasticGradientDescent <char>(this.net)
{
LearningRate = 0.0001,
Momentum = 0.0,
BatchSize = 1,
L2Decay = 0.0
};
// here we only test the gradient at data, but if this is
// right then that's comforting, because it is a function
// of all gradients above, for all layers.
Volume volume = new Volume(new float[] { random.NextDouble() * 2 - 1, random.NextDouble() * 2 - 1 });
int gti = (int)Math.Floor(random.NextDouble() * 3); // ground truth index
trainer.Learn(Enumerable.Repeat(Tuple.Create(volume, this.classes[gti]), 1)); // computes gradients at all layers, and at x
Volume gradient = this.net.Layers[0].InputGradient;
float delta = 0.000001;
for (int i = 0; i < volume.Length; i++)
{
float gradAnalytic = gradient[i];
float xold = volume[i];
volume[i] += delta;
float c0 = this.net.CostLoss(this.net.Compute(volume, false), this.classes[gti]);
volume[i] -= 2 * delta;
float c1 = this.net.CostLoss(this.net.Compute(volume, false), this.classes[gti]);
volume[i] = xold; // reset
float gradNumeric = (c0 - c1) / (2 * delta);
float relError = Math.Abs(gradAnalytic - gradNumeric) / Math.Abs(gradAnalytic + gradNumeric);
Console.WriteLine("step: {0}, numeric: {1}, analytic: {2}, => relError: {3}", i, gradNumeric, gradAnalytic, relError);
Assert.IsTrue(relError < 1e-2);
}
}
void ItWorks()
{
var network2 = new BasicNetwork();
var data = LoadData();
var label = LoadLabel();
var network = new BasicNetwork();
network.AddLayer(new BasicLayer(null, true, 4));
network.AddLayer(new BasicLayer(new ActivationReLU(), true, 8));
network.AddLayer(new BasicLayer(new ActivationSigmoid(), false, 1));
network.Structure.FinalizeStructure();
network.Reset();
var trainingSet = new BasicMLDataSet(data, label);
var propagation = new StochasticGradientDescent(network, trainingSet);
var epoch = 1;
for (var i = 0; i < 5000; i++)
{
propagation.Iteration();
Console.WriteLine($"Epoch: {epoch} Error: {propagation.Error}");
epoch++;
}
propagation.FinishTraining();
var weights = network.Flat.Weights;
var biases = network.Flat.BiasActivation;
foreach (var pair in trainingSet)
{
var output = network.Compute(pair.Input);
Debug.Log(pair.Input[0] + "," + pair.Input[1]
+ ", actual=" + output[0] + ",ideal=" + pair.Ideal[0]);
}
}
public void Run()
{
var layer = new ILayer[] { new FullyConnectedLayer(new Sigmoid(), 3, 2), new FullyConnectedLayer(new Sigmoid(), 1, 3) };
Randomiser.Randomise(layer, new Random(5));
Console.WriteLine("Evaluationg untrained neural network");
DisplayEvaluation(layer);
var stochasticGradientDescent = new StochasticGradientDescent(new CrossEntropy(), layer, 3000, 4, 5D, 0D);
var trainingData = new List <TrainingElement>
{
new TrainingElement
{
Inputs = new double[] { 0D, 0D },
ExpectedOutputs = new double[] { 0D }
},
new TrainingElement
{
Inputs = new double[] { 1D, 0D },
ExpectedOutputs = new double[] { 1D }
},
new TrainingElement
{
Inputs = new double[] { 0D, 1D },
ExpectedOutputs = new double[] { 1D }
},
new TrainingElement
{
Inputs = new double[] { 1D, 1d },
ExpectedOutputs = new double[] { 0D }
}
};
stochasticGradientDescent.Train(trainingData);
Console.WriteLine("Evaluationg trained neural network");
DisplayEvaluation(layer);
}
public void Init(StochasticGradientDescent theTraining)
{
_training = theTraining;
_cache = new double[theTraining.Flat.Weights.Length];
}
public SimplePerceptron(Vector <double> initialWeights, double learningRate = 0.01, int maxIters = 100)
{
_sgd = new StochasticGradientDescent(this, initialWeights, learningRate, maxIters);
}
static void Main(string[] args)
{
//Set the path of the file containing the data set
//string dataFilePath = @"C:\Users\kevin\Desktop\squaredtest.csv"; NutrioxDataset
string dataFilePath = @"C:\Users\Bruker\Desktop\NutrioxDataset.csv";
//string dataFilePath = @"C:\Users\Bruker\Desktop\-5to5-200Rows.csv";
//Create a new data set
DataSet.DataSet dataSet = new DataSet.DataSet(dataFilePath, true);
//Apply desired data preprocessing to the data set
dataSet.PreProcessDataSet(NormalizationType.MinMax, 2, EncodingType.None, null);
//Create a model hyperparameter layer structure
LayerStructure layerStructure = new LayerStructure()
{
numberOfInputNodes = 2, HiddenLayerList = new List <int> {
5, 5
}, numberOfOutputNodes = 1
};
//Create an instance of the desired optimalization strategy to use
var regularizationStrategyFactory = new RegularizationStrategyFactory();
StochasticGradientDescent SGD = new StochasticGradientDescent(new SigmoidFunction(), new IdentityFunction(), new MeanSquaredError(), RegularizationType.None, regularizationStrategyFactory);
//Create training hyperparameters
TrainingParameters trainingParams = new TrainingParameters()
{
epochs = 500, learningRate = 0.01, momentum = 0.01, RegularizationLambda = 0.00
};
//Create an instance of a neural network
//ArtificialNeuralNetwork ann = new ArtificialNeuralNetwork(layerStructure, trainingParams, dataSet, SGD, new GaussianDistribution());
//Or Load a Network from XML
XML xml = new XML();
ArtificialNeuralNetwork ann = xml.LoadNetwork(@"C:\Users\Bruker\Desktop\BestNet.xml", dataSet) as ArtificialNeuralNetwork;
//Apply the desired training/test data set split ratios.
ann.SplitDataSetIntoTrainAndTestSets(0.7);
//Initiate network training
//ann.TrainNetwork();
var crossValidationStrategyFactory = new CrossValidationStrategyFactory();
NetworkEvaluator evaluator = new NetworkEvaluator(ann);
CrossValidator crossValidator = new CrossValidator(ann, evaluator, crossValidationStrategyFactory);
//Cross-validate the fitted model
//crossValidator.KFold(10, 0.007);
//Evaluate the fitted model on the test set
evaluator.EvaluateNetwork(0.007);
//--Optional--//
//Serialize and save the fitted model
//XML xml = new XML();
//xml.SaveNetwork(dataFilePath, ann);
//Extract model information
//ann.SaveListOfErrors();
//ann.GetApproximatedFunction(ann.SavePath + "/Function.txt");
Console.ReadLine();
}
