// Start is called before the first frame update
IEnumerator Start()
{
var r = new System.Random(2);
var x = (Matrix) new double[1000, 1];
Matrix.MatrixLoop((i, j) =>
{
x[i, 0] = i;
}, x.X, x.Y);
var y = (Matrix) new double[1000, 1];
Matrix.MatrixLoop((i, j) =>
{
y[i, 0] = i * 12 + 15 + r.Next(10);
}, x.X, x.Y);
// var x = new double[,] { { 0, 0 }, { 0, 1 }, { 1, 0 }, { 1, 1 } };
// var y = new double[,] { { 0 }, { 1 }, { 0 }, { 1 } };
var X = new Tensor(x, true);
var Y = new Tensor(y, true);
var seq = new Sequential();
seq.Layers.Add(new Linear(1, 1, r));
var sgd = new StochasticGradientDescent(seq.Parameters, 0.001);
var mse = new MeanSquaredError();
for (var i = 0; i < 10000; i++)
{
yield return(null);
var pred = seq.Forward(X);
print(pred.Data.Size);
var loss = mse.Forward(pred, Y);
loss.Backward();
sgd.Step();
print($"Epoch: {i} Loss: {loss.Data[0, 0]}");
print(Y);
print(pred);
}
print(seq.Forward(new Tensor(x)));
}
static void ThirdNN()
{
var r = new Random();
var data = new Tensor((Matrix) new double[, ] {
{ 0, 0 }, { 0, 1 }, { 1, 0 }, { 1, 1 }
}, true);
var target = new Tensor((Matrix) new double[, ] {
{ 0 }, { 1 }, { 0 }, { 1 }
}, true);
var seq = new Sequential();
seq.Layers.Add(new Linear(2, 3, r));
seq.Layers.Add(new Linear(3, 1, r));
var sgd = new StochasticGradientDescent(seq.Parameters, 0.1f);
for (var i = 0; i < 10; i++)
{
var pred = seq.Forward(data);
var diff = pred.Sub(target);
var loss = diff.Mul(diff).Sum(AxisZero.vertical);
loss.Backward(new Tensor(Matrix.Ones(loss.Data.X, loss.Data.Y)));
sgd.Step();
Console.WriteLine($"Epoch: {i} Loss: {loss}");
}
}
static void FourthNN()
{
var r = new Random();
var data = new Tensor((Matrix) new double[, ] {
{ 0, 0 }, { 0, 1 }, { 1, 0 }, { 1, 1 }
}, true);
var target = new Tensor((Matrix) new double[, ] {
{ 0 }, { 1 }, { 0 }, { 1 }
}, true);
var seq = new Sequential();
seq.Layers.Add(new Linear(2, 3, r));
seq.Layers.Add(new Linear(3, 1, r));
var sgd = new StochasticGradientDescent(seq.Parameters, 0.1f);
var mse = new MeanSquaredError();
for (var i = 0; i < 10; i++)
{
var pred = seq.Forward(data);
var loss = mse.Forward(pred, target);
loss.Backward(new Tensor(Matrix.Ones(loss.Data.X, loss.Data.Y)));
sgd.Step();
Console.WriteLine($"Epoch: {i} Loss: {loss}");
}
}
public float[] Forward(float[] x)
{
flat_mem_stack = Utils.FlattenArray(mem_stack);
float[] inputPlusMemory = Utils.ConcatArray(x, flat_mem_stack);
float[] output = h_layers.Forward(inputPlusMemory);
// setting the mem Stack
float[,] mem2 = new float[n_memory, n_outputs + x.Length];
for (int i = 0; i < n_memory; i++)
{
for (int j = 0; j < n_outputs + x.Length; j++)
{
try {
mem2[i + 1, j] = mem_stack[i, j];
}
catch (System.Exception) {
break;
}
}
}
float[] outputPlusInput = Utils.ConcatArray(x, output);
for (int i = 0; i < outputPlusInput.Length; i++)
{
mem2[0, i] = outputPlusInput[i];
}
mem_stack = mem2;
return(output);
}
public override TorchTensor Forward(TorchTensor input)
{
using (var f = features.Forward(input))
using (var avg = avgPool.Forward(f))
using (var x = avg.View(new long[] { avg.Shape[0], 256 * 2 * 2 }))
return(classifier.Forward(x));
}
public unsafe static void XORExampleNew()
{
//Hyperparameters
Hyperparameters.LearningRate = 0.1f;
Hyperparameters.Optimizer = new SGD();
//Model Creation
//var l1 = new DenseModule(2, 16, "sigmoid");
//var model = new DenseModule(16, 1, "sigmoid", l1);
var model = new Sequential(
new DenseModule(2, 16, "sigmoid"),
new DenseModule(16, 1, "sigmoid")
);
//Data preparation
Tensor x_train = new Tensor((4, 2), DeviceConfig.Host_Float);
Tensor y_train = new Tensor((4, 1), DeviceConfig.Host_Float);
float *xt = (float *)x_train.Array;
float *yt = (float *)y_train.Array;
// 1,1 = 0
// 1,0 = 1
// 0,1 = 1
// 0,0 = 0
xt[0] = 1; xt[1] = 1;
xt[2] = 1; xt[3] = 0;
xt[4] = 0; xt[5] = 1;
xt[6] = 0; xt[7] = 0;
yt[0] = 0;
yt[1] = 1;
yt[2] = 1;
yt[3] = 0;
//Give data to the model
Terms.Term[] data = new Terms.Term[] { new Terms.Variable(x_train) };
Terms.Term[] label = new Terms.Term[] { new Terms.Variable(y_train) };
Terms.Term[] res = model.Forward(data);
WriteLine(res[0].GetResult());
}
// Evaluate the model on the test set. Prints the total number of training samples that are classified correctly
private static void EvaluateModel(Sequential model, DataSet testingSet, int numInputs)
{
float totalCorrect = 0;
for (int batchStart = 0; batchStart <= testingSet.inputs.Shape[0] - BatchSize; batchStart += BatchSize)
{
using (var mbInputs = testingSet.inputs.Narrow(0, batchStart, BatchSize))
using (var mbTargetValues = testingSet.targetValues.Narrow(0, batchStart, BatchSize))
{
var modelOutput = model.Forward(mbInputs, ModelMode.Evaluate);
totalCorrect += (modelOutput.TVar().Argmax(1) == mbTargetValues)
.SumAll()
.ToScalar()
.Evaluate();
}
}
Console.WriteLine("Test set total correct: " + totalCorrect + " / " + testingSet.inputs.Shape[0]);
}
// Runs a single epoch of training.
private static void TrainEpoch(Sequential model, ICriterion criterion, SgdOptimizer optim, DataSet trainingSet, int numInputs, bool useTargetClasses)
{
using (new SimpleTimer("Training epoch completed in {0}ms"))
{
for (int batchStart = 0; batchStart <= trainingSet.inputs.Shape[0] - BatchSize; batchStart += BatchSize)
{
Console.Write(".");
var grad = new GradFunc(parameters =>
{
using (var mbInputs = trainingSet.inputs.Narrow(0, batchStart, BatchSize))
using (var mbTargets = trainingSet.targets.Narrow(0, batchStart, BatchSize))
using (var mbTargetClasses = trainingSet.targetValues.Narrow(0, batchStart, BatchSize))
{
foreach (var gradNDArray in model.GetGradParameters())
{
Ops.Fill(gradNDArray, 0);
}
var modelOutput = model.Forward(mbInputs, ModelMode.Train);
var criterionOutput = criterion.UpdateOutput(modelOutput, useTargetClasses ? mbTargetClasses : mbTargets);
var criterionGradIn = criterion.UpdateGradInput(modelOutput, useTargetClasses ? mbTargetClasses : mbTargets);
model.Backward(mbInputs, criterionGradIn, ModelMode.Train);
return(new OutputAndGrads()
{
output = modelOutput, grads = model.GetGradParameters().ToArray()
});
}
});
optim.Update(grad, model.GetParameters().ToArray());
}
}
Console.WriteLine();
}
public void AdaDeltaTest()
{
var net = new Sequential(new QuadraticCost(), new AdaDelta(0.03));
net.CreateLayer(new Dense(3, new Tanh()));
net.CreateLayer(new Dense(3, new Tanh()));
net.CreateLayer(new Dense(3, new Tanh()));
net.CreateLayer(new Output(1, new Tanh()));
net.InitNetwork();
var inputs = new List <Matrix>();
var outputs = new List <Matrix>();
// 0 0 0 => 0
inputs.Add(new Matrix(new[, ] {
{ 0.0 }, { 0.0 }, { 0.0 }
}));
outputs.Add(new Matrix(new[, ] {
{ 0.0 }
}));
// 0 0 1 => 1
inputs.Add(new Matrix(new[, ] {
{ 0.0 }, { 0.0 }, { 1.0 }
}));
outputs.Add(new Matrix(new[, ] {
{ 1.0 }
}));
// 0 1 0 => 1
inputs.Add(new Matrix(new[, ] {
{ 0.0 }, { 1.0 }, { 0.0 }
}));
outputs.Add(new Matrix(new[, ] {
{ 1.0 }
}));
// 0 1 1 => 0
inputs.Add(new Matrix(new[, ] {
{ 0.0 }, { 1.0 }, { 1.0 }
}));
outputs.Add(new Matrix(new[, ] {
{ 1.0 }
}));
// 1 0 0 => 1
inputs.Add(new Matrix(new[, ] {
{ 1.0 }, { 0.0 }, { 0.0 }
}));
outputs.Add(new Matrix(new[, ] {
{ 1.0 }
}));
// 1 0 1 => 0
inputs.Add(new Matrix(new[, ] {
{ 1.0 }, { 0.0 }, { 1.0 }
}));
outputs.Add(new Matrix(new[, ] {
{ 0.0 }
}));
// 1 1 0 => 0
inputs.Add(new Matrix(new[, ] {
{ 1.0 }, { 1.0 }, { 0.0 }
}));
outputs.Add(new Matrix(new[, ] {
{ 0.0 }
}));
// 1 1 1 => 1
inputs.Add(new Matrix(new[, ] {
{ 1.0 }, { 1.0 }, { 1.0 }
}));
outputs.Add(new Matrix(new[, ] {
{ 1.0 }
}));
for (var i = 0; i < 8; i++)
{
net.Train(inputs[i % 8], outputs[i % 8]);
}
var correct = 0;
for (var i = 0; i < 10; i++)
{
correct += Math.Abs(net.Forward(inputs[0])[0, 0]) < 0.1 ? 1 : 0;
correct += Math.Abs(net.Forward(inputs[1])[0, 0]) - 1 < 0.1 ? 1 : 0;
correct += Math.Abs(net.Forward(inputs[2])[0, 0]) - 1 < 0.1 ? 1 : 0;
correct += Math.Abs(net.Forward(inputs[3])[0, 0]) < 0.1 ? 1 : 0;
correct += Math.Abs(net.Forward(inputs[4])[0, 0]) - 1 < 0.1 ? 1 : 0;
correct += Math.Abs(net.Forward(inputs[5])[0, 0]) < 0.1 ? 1 : 0;
correct += Math.Abs(net.Forward(inputs[6])[0, 0]) < 0.1 ? 1 : 0;
correct += Math.Abs(net.Forward(inputs[7])[0, 0]) - 1 < 0.1 ? 1 : 0;
}
var acc = correct / 80.0 * 100.0;
Trace.WriteLine(" Acc: " + acc);
Assert.IsTrue(acc > 80.0, "Network did not learn XOR");
}
private void Train()
{
var array = new int[totalInputList.Count];
for (int i = 0; i < totalInputList.Count; i++)
{
array[i] = i;
}
System.Array.Sort <int>(array, new System.Comparison <int>(
(i1, i2) => totalInputList[i2].Count.CompareTo(totalInputList[i1].Count)));
var newArray = new int[getFirst];
var regCount = 0;
for (int i = 0; i < getFirst; i++)
{
newArray[i] = i;
regCount += totalOutputList[i].Count;
}
var MatrixX = new double[regCount, 4];
var MatrixY = new double[regCount, 1];
var x = 0;
for (int i = 0; i < newArray.Length; i++)
{
for (int j = 0; j < totalInputList[i].Count; j++)
{
MatrixX[x, 0] = totalInputList[newArray[i]][j][0, 0];
MatrixX[x, 1] = totalInputList[newArray[i]][j][0, 1];
MatrixX[x, 2] = totalInputList[newArray[i]][j][0, 2];
MatrixX[x, 3] = totalInputList[newArray[i]][j][0, 3];
MatrixY[x, 0] = totalOutputList[newArray[i]][j][0, 0];
x++;
}
}
var X = new Tensor(MatrixX, true);
var Y = new Tensor(MatrixY, true);
for (var i = 0; i < fittingEpoch; i++)
{
var pred = seq.Forward(X);
var loss = mse.Forward(pred, Y);
loss.Backward();
sgd.Step();
if (double.IsNaN(loss.Data[0, 0]))
{
Debug.LogError("LOSS IS NAN");
Debug.Break();
}
print("Epoch: " + i + " Loss: " + loss.Data[0, 0]);
}
totalInputList.Clear();
totalOutputList.Clear();
}
public void DenseMnist()
{
// Load Train Data
var lines = File.ReadAllLines("..\\..\\..\\..\\..\\datasets\\mnist_train.csv").ToList();
var mnistLables = new List <Matrix>();
var mnistData = new List <Matrix>();
for (var j = 1; j < lines.Count; j++)
{
var t = lines[j];
var data = t.Split(',').ToList();
mnistLables.Add(new Matrix(10, 1).Fill(0));
mnistLables[j - 1][int.Parse(data[0]), 0] = 1.0;
var mnist = new Matrix(784, 1);
for (var i = 1; i < data.Count; i++)
{
mnist[i - 1, 0] = double.Parse(data[i]);
}
mnistData.Add(mnist);
}
// Load Test Data
var testlines = File.ReadAllLines("..\\..\\..\\..\\..\\datasets\\mnist_test.csv").ToList();
var mnistTestLables = new List <Matrix>();
var mnistTestData = new List <Matrix>();
for (var j = 1; j < testlines.Count; j++)
{
var t = testlines[j];
var data = t.Split(',').ToList();
mnistTestLables.Add(new Matrix(10, 1).Fill(0));
mnistTestLables[j - 1][int.Parse(data[0]), 0] = 1.0;
var mnist = new Matrix(784, 1);
for (var i = 1; i < data.Count; i++)
{
mnist[i - 1, 0] = double.Parse(data[i]);
}
mnistTestData.Add(mnist);
}
// Create Network
var net = new Sequential(new CategoricalCrossEntropy(), new Adam(0.3), null, 128);
net.CreateLayer(new Dense(784, new Relu()));
net.CreateLayer(new Dense(128, new Relu()));
net.CreateLayer(new Output(10, new Softmax()));
net.InitNetwork();
// Train Network
for (var i = 0; i < mnistData.Count / 100; i++)
{
net.Train(mnistData[i % mnistData.Count], mnistLables[i % mnistData.Count]);
}
// Test Network
for (var i = 0; i < mnistTestData.Count / 100; i++)
{
var mat = net.Forward(mnistTestData[i % mnistTestData.Count]);
var matx = mnistTestLables[i % mnistTestData.Count];
}
Trace.WriteLine(" Metrics Accuracy: ");
//Assert.IsTrue(acc > 80.0, "Network did not learn MNIST");
}