public void TestAutoGradMode()
{
var x = FloatTensor.RandomN(new long[] { 2, 3 }, device: "cpu:0", requiresGrad: true);
using (var mode = new AutoGradMode(false))
{
Assert.False(AutoGradMode.IsAutogradEnabled());
var sum = x.Sum();
sum.Backward();
var grad = x.Grad();
Assert.True(grad.Handle == IntPtr.Zero);
}
using (var mode = new AutoGradMode(true))
{
Assert.True(AutoGradMode.IsAutogradEnabled());
var sum = x.Sum();
sum.Backward();
var grad = x.Grad();
Assert.False(grad.Handle == IntPtr.Zero);
var data = grad.Data <float>();
for (int i = 0; i < 2 * 3; i++)
{
Assert.Equal(1.0, data[i]);
}
}
}
public void TestCustomModuleWithInPlaceModification()
{
var param = FloatTensor.RandomN(new long[] { 1000, 100 });
var module = new TestModule("test", param, true);
Assert.Equal(1000, module.GetParameter("test").Shape[0]);
Assert.Equal(100, module.GetParameter("test").Shape[1]);
using (var grad = new AutoGradMode(false))
{
param.TransposeInPlace(0, 1);
}
Assert.Equal(100, module.GetParameter("test").Shape[0]);
Assert.Equal(1000, module.GetParameter("test").Shape[1]);
Assert.Equal(100, param.Shape[0]);
Assert.Equal(1000, param.Shape[1]);
}
public void TestTraining()
{
var lin1 = NN.Module.Linear(1000, 100);
var lin2 = NN.Module.Linear(100, 10);
var seq = NN.Module.Sequential(lin1, NN.Module.Relu(), lin2);
var x = FloatTensor.RandomN(new long[] { 64, 1000 }, device: "cpu:0");
var y = FloatTensor.RandomN(new long[] { 64, 10 }, device: "cpu:0");
float learning_rate = 0.00004f;
float prevLoss = float.MaxValue;
var loss = NN.LossFunction.MSE(NN.Reduction.Sum);
for (int i = 0; i < 10; i++)
{
var eval = seq.Forward(x);
var output = loss(eval, y);
var lossVal = output.DataItem <float>();
Assert.True(lossVal < prevLoss);
prevLoss = lossVal;
seq.ZeroGrad();
output.Backward();
using (var noGrad = new AutoGradMode(false))
{
foreach (var param in seq.Parameters())
{
var grad = param.Grad();
var update = grad.Mul(learning_rate.ToScalar());
param.SubInPlace(update);
}
}
}
}
