public void TestGrad()
{
var lin1 = Linear(1000, 100);
var lin2 = Linear(100, 10);
var seq = Sequential(
("lin1", lin1),
("relu1", Relu()),
("lin2", lin2));
var x = FloatTensor.RandomN(new long[] { 64, 1000 }, requiresGrad: true);
var y = FloatTensor.RandomN(new long[] { 64, 10 }, requiresGrad: true);
var eval = seq.Forward(x);
var loss = MSE(NN.Reduction.Sum);
var output = loss(eval, y);
seq.ZeroGrad();
output.Backward();
foreach (var parm in seq.GetParameters())
{
var grad = parm.Grad();
}
}
public void TestGrad2()
{
var y = FloatTensor.RandomN(new long[] { 32, 1 }, device: "cpu:0");
var input = new double[] { -2.75, 0.77, -0.61, 0.14, 1.39, 0.38, -0.53, -0.5, -2.13, -0.39, 0.46, -0.61, -0.37, -0.12, 0.55, -1, 0.84, -0.02, 1.3, -0.24, -0.5, -2.12, -0.85, -0.91, 1.81, 0.02, -0.78, -1.41, -1.09, -0.65, 0.9, -0.37, -0.22, 0.28, 1.05, -0.24, 0.3, -0.99, 0.19, 0.32, -0.95, -1.19, -0.63, 0.75, 0.16, 0.15, 0.3, -0.69, 0.2, -0.4, -0.67, 0.18, -1.43, -0.61, -0.78, -0.11, -1.07, -1.71, -0.45, -0.6, 0.05, -1.59, 1.24, 0.62, 0.01, 1.35, -0.9, -1.25, 1.62, -1.45, 0.92, 1.51, -0.19, -1.33, -0.01, -0.13, 0.1, -1.34, 1.23, 0.57, -0.24, 0.5, 0.71, -0.15, -1.37, -1.03, 1.8, 1.4, -0.63, 0.8, -0.97, -0.64, 0.51, 0.52, 0.95, 0.86, 0.43, 0.73, -1.38, -0.56, 0.44, 1.2, -1.45, -0.07, 1.88, 1.57, 0.38, -2.2, -0.56, -1.52, -0.17, 1.38, -1.02, -1.61, -0.13, -0.44, -0.37, 0.23, 1.75, 0.83, -0.02, -1.91, -0.23, -0.47, -1.41, -1.01, -0.91, -0.56, -1.72, 1.47, 0.31, 0.24, 0.48, 2.06, 0.07, -0.96, 1.03, -0.4, -0.64, -0.85, 0.42, -0.33, 0.85, -0.11, -1.24, -0.71, -1.04, -0.37, -0.37, 0.84, -0.9, -1.63, -2.91, -0.71, 0.09, 1.64, -1.1, -1.05, 0.51, 0.57, 0.19, 0.36, 1.36, 1.45, 0.35, -1.66, -0.65, 0.47, 1.95, -0.32, 0.19, -2.06, 0.5, 1.03, 0.94, -0.65, -2.94, 0.41, 1.13, 0.95, -0.02, 1.12, 0.19, 0.66, -0.77, -0.39, 0.59, -1.58, -0.67, 0.88, 0.26, -0.63, 0.49, 1.38, 1.48, -0.55, 0.4, 0.65, 0.19, 0.25, 0.03, -0.31, 0.75, 2.16, -1.36, 0.05, 0.22, 0.65, 1.28, 0.42, 1.35, -0.08, 1.1, 0.25, 0.44, 1.06, -1.78, 0.47, 1.38, 0.43, -1.56, 0.14, -0.22, 1.48, 0.04, 0.33, 0.1, 0.2, -0.99, 1.04, 0.61, -0.4, 0.96, 0.4, 0.5, 0.1, 0.02, 0.01, 0.22, 1.45, -0.77, 0.69, 0.95, 0.96, -0.09, -0.26, 0.22, -1.61, 1.86, -0.06, -0.34, -0.35, 0.55, -1.08, 1.29, 0.92, 0.16, 0.55, -0.01, 0.2, -0.61, -0.28, -2.17, -0.46, 1.63, 1.61, 0.64, 0.32, -0.75, 0.33, 0.3, -1.15, 0.42, -0.06, -1.14, 1.62, -0.9, -0.39, 0.4, 1.52, -0.43, 1.22, -0.32, -0.02, 1, -0.92, 0.11, 0.8, -0.99, -0.26, -2.85, -1.13, 0.49, -0.63, -0.54, -0.86, -0.97, -0.9, 0.23, 1.26, -1.78, -0.84, -0.48, 0.35, -1.13, -2.23, 0.1, 0.95, 1.27, 0.08, -2.21, 0.67, -0.2, 0.6, -1.14, 0.65, -0.73, -0.01, 0.9, -1.33, -1.16, 0.29, 1.16, 1.19, 0.84, 0.66, -1.55, -0.58, 1.85, -1.16, -0.95, 0.98, -0.1, -1.47, 0.78, -0.75, -1.32, 0.61, -0.5, -1, -0.42, 0.96, -1.39, 0.08, -1.82, 0.51, -0.71, -0.02, 2.32, -0.71, 0.08, -1.07 }.ToTorchTensor(new long[] { 32, 11 }).ToType(ATenScalarMapping.Float);
var inputs = new TorchTensor[] { input };
var scaler = new double[] { 0.2544529, 0.3184713, 0.2597403, 0.3246753, 0.3144654, 0.3322259, 0.3436426, 0.3215434, 0.308642, 0.3154574, 0.3448276 }.ToTorchTensor(new long[] { 1, 11 }).ToType(ATenScalarMapping.Float).RequiresGrad(true);
var linear = new NN.Linear(11, 1, true);
linear.Bias = new double[] { 373.8864 }.ToTorchTensor(new long[] { 1, 1 }).ToType(ATenScalarMapping.Float).RequiresGrad(true);
linear.Weight = new double[] { 300.2818, -0.5905267, 286.2787, 0.1970505, 0.9004903, 0.1373157, 55.85495, 11.43741, 1.525748, 0.4299785, 239.9356 }.ToTorchTensor(new long[] { 1, 11 }).ToType(ATenScalarMapping.Float).RequiresGrad(true);
var afterCat = inputs.Cat(1);
var afterScaler = afterCat * scaler;
var prediction = linear.Forward(afterScaler);
var loss = NN.LossFunction.MSE();
var output = loss(prediction, y);
linear.ZeroGrad();
output.Backward();
var scalerGrad = scaler.Grad();
var weightGrad = linear.Weight.Grad();
var biasGrad = linear.Bias.Value.Grad();
Assert.True(scalerGrad.Shape.Length == 2);
Assert.True(weightGrad.Shape.Length == 2);
Assert.True(biasGrad.Shape.Length == 2);
}
public void TestAutoGradMode()
{
var x = FloatTensor.RandomN(new long[] { 2, 3 }, requiresGrad: true);
using (var mode = new AutoGradMode(false))
{
Assert.False(AutoGradMode.IsAutogradEnabled());
var sum = x.Sum();
Assert.Throws <ExternalException>(() => sum.Backward());
//var grad = x.Grad();
//Assert.True(grad.Handle == IntPtr.Zero);
}
using (var mode = new AutoGradMode(true))
{
Assert.True(AutoGradMode.IsAutogradEnabled());
var sum = x.Sum();
sum.Backward();
var grad = x.Grad();
Assert.False(grad.Handle == IntPtr.Zero);
var data = grad.Data <float>();
for (int i = 0; i < 2 * 3; i++)
{
Assert.Equal(1.0, data[i]);
}
}
}
public void TestTrainingAdam()
{
var lin1 = Linear(1000, 100);
var lin2 = Linear(100, 10);
var seq = Sequential(("lin1", lin1), ("relu1", Relu()), ("lin2", lin2));
var x = FloatTensor.RandomN(new long[] { 64, 1000 });
var y = FloatTensor.RandomN(new long[] { 64, 10 });
double learning_rate = 0.00004f;
float prevLoss = float.MaxValue;
var optimizer = NN.Optimizer.Adam(seq.GetParameters(), learning_rate);
var loss = MSE(NN.Reduction.Sum);
for (int i = 0; i < 10; i++)
{
var eval = seq.Forward(x);
var output = loss(eval, y);
var lossVal = output.DataItem <float>();
Assert.True(lossVal < prevLoss);
prevLoss = lossVal;
optimizer.ZeroGrad();
output.Backward();
optimizer.Step();
}
}
public void TestTrainingAdam()
{
var lin1 = NN.Module.Linear(1000, 100);
var lin2 = NN.Module.Linear(100, 10);
var seq = NN.Module.Sequential(lin1, NN.Module.Relu(), lin2);
var x = FloatTensor.RandomN(new long[] { 64, 1000 }, device: "cpu:0");
var y = FloatTensor.RandomN(new long[] { 64, 10 }, device: "cpu:0");
double learning_rate = 0.00004f;
float prevLoss = float.MaxValue;
var optimizer = NN.Optimizer.Adam(seq.Parameters(), learning_rate);
var loss = NN.LossFunction.MSE(NN.Reduction.Sum);
for (int i = 0; i < 10; i++)
{
var eval = seq.Forward(x);
var output = loss(eval, y);
var lossVal = output.DataItem <float>();
Assert.True(lossVal < prevLoss);
prevLoss = lossVal;
optimizer.ZeroGrad();
output.Backward();
optimizer.Step();
}
}
public void TestCustomModule()
{
var module = new TestModule("test", FloatTensor.RandomN(new long[] { 2, 2 }), true);
var name = module.GetName();
Assert.NotNull(name);
Assert.True(module.HasParameter("test"));
}
public void EvalSequence()
{
var lin1 = NN.Module.Linear(1000, 100);
var lin2 = NN.Module.Linear(100, 10);
var seq = NN.Module.Sequential(lin1, NN.Module.Relu(), lin2);
var x = FloatTensor.RandomN(new long[] { 64, 1000 }, device: "cpu:0", requiresGrad: true);
var eval = seq.Forward(x);
}
public void TestGradConditional()
{
var modT = new CondModel("modT", true);
var modF = new CondModel("modF", false);
var psT = modT.GetParameters();
Assert.Equal(4, psT.Length);
var psF = modF.GetParameters();
Assert.Equal(4, psF.Length);
var x = FloatTensor.RandomN(new long[] { 64, 1000 }, requiresGrad: true);
var y = FloatTensor.RandomN(new long[] { 64, 10 }, requiresGrad: true);
modT.Train();
var eval = modT.Forward(x);
var loss = MSE(NN.Reduction.Sum);
var output = loss(eval, y);
modT.ZeroGrad();
output.Backward();
var gradCounts = 0;
foreach (var parm in modT.GetParameters())
{
var grad = parm.Grad();
gradCounts += grad.Handle == IntPtr.Zero ? 0 : 1;
}
Assert.Equal(2, gradCounts);
//{ "grad can be implicitly created only for scalar outputs (_make_grads at ..\\..\\torch\\csrc\\autograd\\autograd.cpp:47)\n(no backtrace available)"}
modF.Train();
eval = modF.Forward(x);
output = loss(eval, y);
modF.ZeroGrad();
output.Backward();
gradCounts = 0;
foreach (var parm in modF.GetParameters())
{
var grad = parm.Grad();
gradCounts += grad.Handle == IntPtr.Zero ? 0 : 1;
}
Assert.Equal(3, gradCounts);
}
public void EvalSequence()
{
var lin1 = Linear(1000, 100);
var lin2 = Linear(100, 10);
var seq = Sequential(
("lin1", lin1),
("relu1", Relu()),
("lin2", lin2));
var x = FloatTensor.RandomN(new long[] { 64, 1000 }, requiresGrad: true);
var eval = seq.Forward(x);
}
public void TestLinearEditBias()
{
var lin = NN.Module.Linear(1000, 100, true);
var bias = FloatTensor.RandomN(new long[] { 100 });
lin.Bias = bias;
for (int i = 0; i < 100; i++)
{
Assert.Equal(lin.Bias.Value.Data <float>()[i], bias.Data <float>()[i]);
}
}
public void EvalLossSequence()
{
var lin1 = NN.Module.Linear(1000, 100);
var lin2 = NN.Module.Linear(100, 10);
var seq = NN.Module.Sequential(lin1, NN.Module.Relu(), lin2);
var x = FloatTensor.RandomN(new long[] { 64, 1000 }, device: "cpu:0");
var y = FloatTensor.RandomN(new long[] { 64, 10 }, device: "cpu:0");
var eval = seq.Forward(x);
var loss = NN.LossFunction.MSE(NN.Reduction.Sum);
var output = loss(eval, y);
var result = output.DataItem <float>();
}
public void TestLinearEditWeightsAndBiasGetParameters()
{
var lin = NN.Module.Linear(0, 0, true);
var bias = FloatTensor.RandomN(new long[] { 100 });
var weights = FloatTensor.RandomN(new long[] { 100, 1000 });
lin.Bias = bias;
lin.Weight = weights;
var parameters = lin.Parameters().ToArray();
Assert.Equal(lin.Weight.Shape.Length, parameters[0].Shape.Length);
Assert.Equal(lin.Weight.Shape[0], parameters[0].Shape[0]);
Assert.Equal(lin.Weight.Shape[1], parameters[0].Shape[1]);
}
public void TestCustomModuleWithInPlaceModification()
{
var param = FloatTensor.RandomN(new long[] { 1000, 100 });
var module = new TestModule("test", param, true);
Assert.Equal(1000, module.GetParameter("test").Shape[0]);
Assert.Equal(100, module.GetParameter("test").Shape[1]);
using (var grad = new AutoGradMode(false))
{
param.TransposeInPlace(0, 1);
}
Assert.Equal(100, module.GetParameter("test").Shape[0]);
Assert.Equal(1000, module.GetParameter("test").Shape[1]);
Assert.Equal(100, param.Shape[0]);
Assert.Equal(1000, param.Shape[1]);
}
public void TestLinearNoBias()
{
var lin = NN.Module.Linear(1000, 100, false);
var weight = lin.Weight.Transpose(0, 1);
var input = FloatTensor.RandomN(new long[] { 1, 1000 });
var forward = lin.Forward(input);
var matmul = input.MatMul(weight);
Assert.Equal(forward.Shape.Length, matmul.Shape.Length);
Assert.Equal(forward.Shape[0], matmul.Shape[0]);
Assert.Equal(forward.Shape[1], matmul.Shape[1]);
for (int i = 0; i < 100; i++)
{
Assert.Equal(forward.Data <float>()[i], matmul.Data <float>()[i]);
}
}
public void TestBackward()
{
var lin1 = NN.Module.Linear(1000, 100);
var lin2 = NN.Module.Linear(100, 10);
var seq = NN.Module.Sequential(lin1, NN.Module.Relu(), lin2);
var x = FloatTensor.RandomN(new long[] { 64, 1000 }, device: "cpu:0");
var y = FloatTensor.RandomN(new long[] { 64, 10 }, device: "cpu:0");
var eval = seq.Forward(x);
var loss = NN.LossFunction.MSE(NN.Reduction.None);
var output = loss(eval, y);
seq.ZeroGrad();
output.Backward();
}
public void TestGradConditional()
{
var modT = new CondModel(true);
var modF = new CondModel(false);
var x = FloatTensor.RandomN(new long[] { 64, 1000 }, device: "cpu:0");
var y = FloatTensor.RandomN(new long[] { 64, 10 }, device: "cpu:0");
modT.Train();
var eval = modT.Forward(x);
var loss = NN.LossFunction.MSE(NN.Reduction.None);
var output = loss(eval, y);
modT.ZeroGrad();
output.Backward();
var gradCounts = 0;
foreach (var parm in modT.Parameters())
{
var grad = parm.Grad();
gradCounts += grad.Handle == IntPtr.Zero ? 0 : 1;
}
Assert.Equal(2, gradCounts);
modF.Train();
eval = modF.Forward(x);
output = loss(eval, y);
modF.ZeroGrad();
output.Backward();
gradCounts = 0;
foreach (var parm in modF.Parameters())
{
var grad = parm.Grad();
gradCounts += grad.Handle == IntPtr.Zero ? 0 : 1;
}
Assert.Equal(3, gradCounts);
}
public void TestLinearWithBias()
{
var lin = NN.Module.Linear(1000, 100, true);
var bias = lin.Bias;
var weight = lin.Weight.T();
var input = FloatTensor.RandomN(new long[] { 1, 1000 });
var forward = lin.Forward(input);
var matmul = input.MatMul(weight).Add(bias.Value);
Assert.Equal(forward.Shape.Length, matmul.Shape.Length);
Assert.Equal(forward.Shape[0], matmul.Shape[0]);
Assert.Equal(forward.Shape[1], matmul.Shape[1]);
for (int i = 0; i < 100; i++)
{
Assert.InRange(forward.Data <float>()[i], matmul.Data <float>()[i] - 10e5f, matmul.Data <float>()[i] + 10e5f);
}
}
public void TestLinearEditWeightsAndBias()
{
var lin = NN.Module.Linear(0, 0, true);
var bias = FloatTensor.RandomN(new long[] { 100 });
var weights = FloatTensor.RandomN(new long[] { 100, 1000 });
lin.Bias = bias;
lin.Weight = weights;
Assert.Equal(lin.Weight.Shape.Length, weights.Shape.Length);
Assert.Equal(lin.Weight.Shape[0], weights.Shape[0]);
Assert.Equal(lin.Weight.Shape[1], weights.Shape[1]);
for (int i = 0; i < 100; i++)
{
Assert.Equal(lin.Bias.Value.Data <float>()[i], bias.Data <float>()[i]);
}
}
public void EvalLossSequence()
{
var lin1 = Linear(1000, 100);
var lin2 = Linear(100, 10);
var seq = Sequential(
("lin1", lin1),
("relu1", Relu()),
("lin2", lin2));
var x = FloatTensor.RandomN(new long[] { 64, 1000 });
var y = FloatTensor.RandomN(new long[] { 64, 10 });
var eval = seq.Forward(x);
var loss = MSE(NN.Reduction.Sum);
var output = loss(eval, y);
var result = output.DataItem <float>();
}
public void TestTraining()
{
var lin1 = NN.Module.Linear(1000, 100);
var lin2 = NN.Module.Linear(100, 10);
var seq = NN.Module.Sequential(lin1, NN.Module.Relu(), lin2);
var x = FloatTensor.RandomN(new long[] { 64, 1000 }, device: "cpu:0");
var y = FloatTensor.RandomN(new long[] { 64, 10 }, device: "cpu:0");
float learning_rate = 0.00004f;
float prevLoss = float.MaxValue;
var loss = NN.LossFunction.MSE(NN.Reduction.Sum);
for (int i = 0; i < 10; i++)
{
var eval = seq.Forward(x);
var output = loss(eval, y);
var lossVal = output.DataItem <float>();
Assert.True(lossVal < prevLoss);
prevLoss = lossVal;
seq.ZeroGrad();
output.Backward();
using (var noGrad = new AutoGradMode(false))
{
foreach (var param in seq.Parameters())
{
var grad = param.Grad();
var update = grad.Mul(learning_rate.ToScalar());
param.SubInPlace(update);
}
}
}
}
