public void TestGrad() { var lin1 = Linear(1000, 100); var lin2 = Linear(100, 10); var seq = Sequential( ("lin1", lin1), ("relu1", Relu()), ("lin2", lin2)); var x = FloatTensor.RandomN(new long[] { 64, 1000 }, requiresGrad: true); var y = FloatTensor.RandomN(new long[] { 64, 10 }, requiresGrad: true); var eval = seq.Forward(x); var loss = MSE(NN.Reduction.Sum); var output = loss(eval, y); seq.ZeroGrad(); output.Backward(); foreach (var parm in seq.GetParameters()) { var grad = parm.Grad(); } }
public void TestGrad2() { var y = FloatTensor.RandomN(new long[] { 32, 1 }, device: "cpu:0"); var input = new double[] { -2.75, 0.77, -0.61, 0.14, 1.39, 0.38, -0.53, -0.5, -2.13, -0.39, 0.46, -0.61, -0.37, -0.12, 0.55, -1, 0.84, -0.02, 1.3, -0.24, -0.5, -2.12, -0.85, -0.91, 1.81, 0.02, -0.78, -1.41, -1.09, -0.65, 0.9, -0.37, -0.22, 0.28, 1.05, -0.24, 0.3, -0.99, 0.19, 0.32, -0.95, -1.19, -0.63, 0.75, 0.16, 0.15, 0.3, -0.69, 0.2, -0.4, -0.67, 0.18, -1.43, -0.61, -0.78, -0.11, -1.07, -1.71, -0.45, -0.6, 0.05, -1.59, 1.24, 0.62, 0.01, 1.35, -0.9, -1.25, 1.62, -1.45, 0.92, 1.51, -0.19, -1.33, -0.01, -0.13, 0.1, -1.34, 1.23, 0.57, -0.24, 0.5, 0.71, -0.15, -1.37, -1.03, 1.8, 1.4, -0.63, 0.8, -0.97, -0.64, 0.51, 0.52, 0.95, 0.86, 0.43, 0.73, -1.38, -0.56, 0.44, 1.2, -1.45, -0.07, 1.88, 1.57, 0.38, -2.2, -0.56, -1.52, -0.17, 1.38, -1.02, -1.61, -0.13, -0.44, -0.37, 0.23, 1.75, 0.83, -0.02, -1.91, -0.23, -0.47, -1.41, -1.01, -0.91, -0.56, -1.72, 1.47, 0.31, 0.24, 0.48, 2.06, 0.07, -0.96, 1.03, -0.4, -0.64, -0.85, 0.42, -0.33, 0.85, -0.11, -1.24, -0.71, -1.04, -0.37, -0.37, 0.84, -0.9, -1.63, -2.91, -0.71, 0.09, 1.64, -1.1, -1.05, 0.51, 0.57, 0.19, 0.36, 1.36, 1.45, 0.35, -1.66, -0.65, 0.47, 1.95, -0.32, 0.19, -2.06, 0.5, 1.03, 0.94, -0.65, -2.94, 0.41, 1.13, 0.95, -0.02, 1.12, 0.19, 0.66, -0.77, -0.39, 0.59, -1.58, -0.67, 0.88, 0.26, -0.63, 0.49, 1.38, 1.48, -0.55, 0.4, 0.65, 0.19, 0.25, 0.03, -0.31, 0.75, 2.16, -1.36, 0.05, 0.22, 0.65, 1.28, 0.42, 1.35, -0.08, 1.1, 0.25, 0.44, 1.06, -1.78, 0.47, 1.38, 0.43, -1.56, 0.14, -0.22, 1.48, 0.04, 0.33, 0.1, 0.2, -0.99, 1.04, 0.61, -0.4, 0.96, 0.4, 0.5, 0.1, 0.02, 0.01, 0.22, 1.45, -0.77, 0.69, 0.95, 0.96, -0.09, -0.26, 0.22, -1.61, 1.86, -0.06, -0.34, -0.35, 0.55, -1.08, 1.29, 0.92, 0.16, 0.55, -0.01, 0.2, -0.61, -0.28, -2.17, -0.46, 1.63, 1.61, 0.64, 0.32, -0.75, 0.33, 0.3, -1.15, 0.42, -0.06, -1.14, 1.62, -0.9, -0.39, 0.4, 1.52, -0.43, 1.22, -0.32, -0.02, 1, -0.92, 0.11, 0.8, -0.99, -0.26, -2.85, -1.13, 0.49, -0.63, -0.54, -0.86, -0.97, -0.9, 0.23, 1.26, -1.78, -0.84, -0.48, 0.35, -1.13, -2.23, 0.1, 0.95, 1.27, 0.08, -2.21, 0.67, -0.2, 0.6, -1.14, 0.65, -0.73, -0.01, 0.9, -1.33, -1.16, 0.29, 1.16, 1.19, 0.84, 0.66, -1.55, -0.58, 1.85, -1.16, -0.95, 0.98, -0.1, -1.47, 0.78, -0.75, -1.32, 0.61, -0.5, -1, -0.42, 0.96, -1.39, 0.08, -1.82, 0.51, -0.71, -0.02, 2.32, -0.71, 0.08, -1.07 }.ToTorchTensor(new long[] { 32, 11 }).ToType(ATenScalarMapping.Float); var inputs = new TorchTensor[] { input }; var scaler = new double[] { 0.2544529, 0.3184713, 0.2597403, 0.3246753, 0.3144654, 0.3322259, 0.3436426, 0.3215434, 0.308642, 0.3154574, 0.3448276 }.ToTorchTensor(new long[] { 1, 11 }).ToType(ATenScalarMapping.Float).RequiresGrad(true); var linear = new NN.Linear(11, 1, true); linear.Bias = new double[] { 373.8864 }.ToTorchTensor(new long[] { 1, 1 }).ToType(ATenScalarMapping.Float).RequiresGrad(true); linear.Weight = new double[] { 300.2818, -0.5905267, 286.2787, 0.1970505, 0.9004903, 0.1373157, 55.85495, 11.43741, 1.525748, 0.4299785, 239.9356 }.ToTorchTensor(new long[] { 1, 11 }).ToType(ATenScalarMapping.Float).RequiresGrad(true); var afterCat = inputs.Cat(1); var afterScaler = afterCat * scaler; var prediction = linear.Forward(afterScaler); var loss = NN.LossFunction.MSE(); var output = loss(prediction, y); linear.ZeroGrad(); output.Backward(); var scalerGrad = scaler.Grad(); var weightGrad = linear.Weight.Grad(); var biasGrad = linear.Bias.Value.Grad(); Assert.True(scalerGrad.Shape.Length == 2); Assert.True(weightGrad.Shape.Length == 2); Assert.True(biasGrad.Shape.Length == 2); }
public void TestAutoGradMode() { var x = FloatTensor.RandomN(new long[] { 2, 3 }, requiresGrad: true); using (var mode = new AutoGradMode(false)) { Assert.False(AutoGradMode.IsAutogradEnabled()); var sum = x.Sum(); Assert.Throws <ExternalException>(() => sum.Backward()); //var grad = x.Grad(); //Assert.True(grad.Handle == IntPtr.Zero); } using (var mode = new AutoGradMode(true)) { Assert.True(AutoGradMode.IsAutogradEnabled()); var sum = x.Sum(); sum.Backward(); var grad = x.Grad(); Assert.False(grad.Handle == IntPtr.Zero); var data = grad.Data <float>(); for (int i = 0; i < 2 * 3; i++) { Assert.Equal(1.0, data[i]); } } }
public void TestTrainingAdam() { var lin1 = Linear(1000, 100); var lin2 = Linear(100, 10); var seq = Sequential(("lin1", lin1), ("relu1", Relu()), ("lin2", lin2)); var x = FloatTensor.RandomN(new long[] { 64, 1000 }); var y = FloatTensor.RandomN(new long[] { 64, 10 }); double learning_rate = 0.00004f; float prevLoss = float.MaxValue; var optimizer = NN.Optimizer.Adam(seq.GetParameters(), learning_rate); var loss = MSE(NN.Reduction.Sum); for (int i = 0; i < 10; i++) { var eval = seq.Forward(x); var output = loss(eval, y); var lossVal = output.DataItem <float>(); Assert.True(lossVal < prevLoss); prevLoss = lossVal; optimizer.ZeroGrad(); output.Backward(); optimizer.Step(); } }
public void TestTrainingAdam() { var lin1 = NN.Module.Linear(1000, 100); var lin2 = NN.Module.Linear(100, 10); var seq = NN.Module.Sequential(lin1, NN.Module.Relu(), lin2); var x = FloatTensor.RandomN(new long[] { 64, 1000 }, device: "cpu:0"); var y = FloatTensor.RandomN(new long[] { 64, 10 }, device: "cpu:0"); double learning_rate = 0.00004f; float prevLoss = float.MaxValue; var optimizer = NN.Optimizer.Adam(seq.Parameters(), learning_rate); var loss = NN.LossFunction.MSE(NN.Reduction.Sum); for (int i = 0; i < 10; i++) { var eval = seq.Forward(x); var output = loss(eval, y); var lossVal = output.DataItem <float>(); Assert.True(lossVal < prevLoss); prevLoss = lossVal; optimizer.ZeroGrad(); output.Backward(); optimizer.Step(); } }
public void TestCustomModule() { var module = new TestModule("test", FloatTensor.RandomN(new long[] { 2, 2 }), true); var name = module.GetName(); Assert.NotNull(name); Assert.True(module.HasParameter("test")); }
public void EvalSequence() { var lin1 = NN.Module.Linear(1000, 100); var lin2 = NN.Module.Linear(100, 10); var seq = NN.Module.Sequential(lin1, NN.Module.Relu(), lin2); var x = FloatTensor.RandomN(new long[] { 64, 1000 }, device: "cpu:0", requiresGrad: true); var eval = seq.Forward(x); }
public void TestGradConditional() { var modT = new CondModel("modT", true); var modF = new CondModel("modF", false); var psT = modT.GetParameters(); Assert.Equal(4, psT.Length); var psF = modF.GetParameters(); Assert.Equal(4, psF.Length); var x = FloatTensor.RandomN(new long[] { 64, 1000 }, requiresGrad: true); var y = FloatTensor.RandomN(new long[] { 64, 10 }, requiresGrad: true); modT.Train(); var eval = modT.Forward(x); var loss = MSE(NN.Reduction.Sum); var output = loss(eval, y); modT.ZeroGrad(); output.Backward(); var gradCounts = 0; foreach (var parm in modT.GetParameters()) { var grad = parm.Grad(); gradCounts += grad.Handle == IntPtr.Zero ? 0 : 1; } Assert.Equal(2, gradCounts); //{ "grad can be implicitly created only for scalar outputs (_make_grads at ..\\..\\torch\\csrc\\autograd\\autograd.cpp:47)\n(no backtrace available)"} modF.Train(); eval = modF.Forward(x); output = loss(eval, y); modF.ZeroGrad(); output.Backward(); gradCounts = 0; foreach (var parm in modF.GetParameters()) { var grad = parm.Grad(); gradCounts += grad.Handle == IntPtr.Zero ? 0 : 1; } Assert.Equal(3, gradCounts); }
public void EvalSequence() { var lin1 = Linear(1000, 100); var lin2 = Linear(100, 10); var seq = Sequential( ("lin1", lin1), ("relu1", Relu()), ("lin2", lin2)); var x = FloatTensor.RandomN(new long[] { 64, 1000 }, requiresGrad: true); var eval = seq.Forward(x); }
public void TestLinearEditBias() { var lin = NN.Module.Linear(1000, 100, true); var bias = FloatTensor.RandomN(new long[] { 100 }); lin.Bias = bias; for (int i = 0; i < 100; i++) { Assert.Equal(lin.Bias.Value.Data <float>()[i], bias.Data <float>()[i]); } }
public void EvalLossSequence() { var lin1 = NN.Module.Linear(1000, 100); var lin2 = NN.Module.Linear(100, 10); var seq = NN.Module.Sequential(lin1, NN.Module.Relu(), lin2); var x = FloatTensor.RandomN(new long[] { 64, 1000 }, device: "cpu:0"); var y = FloatTensor.RandomN(new long[] { 64, 10 }, device: "cpu:0"); var eval = seq.Forward(x); var loss = NN.LossFunction.MSE(NN.Reduction.Sum); var output = loss(eval, y); var result = output.DataItem <float>(); }
public void TestLinearEditWeightsAndBiasGetParameters() { var lin = NN.Module.Linear(0, 0, true); var bias = FloatTensor.RandomN(new long[] { 100 }); var weights = FloatTensor.RandomN(new long[] { 100, 1000 }); lin.Bias = bias; lin.Weight = weights; var parameters = lin.Parameters().ToArray(); Assert.Equal(lin.Weight.Shape.Length, parameters[0].Shape.Length); Assert.Equal(lin.Weight.Shape[0], parameters[0].Shape[0]); Assert.Equal(lin.Weight.Shape[1], parameters[0].Shape[1]); }
public void TestCustomModuleWithInPlaceModification() { var param = FloatTensor.RandomN(new long[] { 1000, 100 }); var module = new TestModule("test", param, true); Assert.Equal(1000, module.GetParameter("test").Shape[0]); Assert.Equal(100, module.GetParameter("test").Shape[1]); using (var grad = new AutoGradMode(false)) { param.TransposeInPlace(0, 1); } Assert.Equal(100, module.GetParameter("test").Shape[0]); Assert.Equal(1000, module.GetParameter("test").Shape[1]); Assert.Equal(100, param.Shape[0]); Assert.Equal(1000, param.Shape[1]); }
public void TestLinearNoBias() { var lin = NN.Module.Linear(1000, 100, false); var weight = lin.Weight.Transpose(0, 1); var input = FloatTensor.RandomN(new long[] { 1, 1000 }); var forward = lin.Forward(input); var matmul = input.MatMul(weight); Assert.Equal(forward.Shape.Length, matmul.Shape.Length); Assert.Equal(forward.Shape[0], matmul.Shape[0]); Assert.Equal(forward.Shape[1], matmul.Shape[1]); for (int i = 0; i < 100; i++) { Assert.Equal(forward.Data <float>()[i], matmul.Data <float>()[i]); } }
public void TestBackward() { var lin1 = NN.Module.Linear(1000, 100); var lin2 = NN.Module.Linear(100, 10); var seq = NN.Module.Sequential(lin1, NN.Module.Relu(), lin2); var x = FloatTensor.RandomN(new long[] { 64, 1000 }, device: "cpu:0"); var y = FloatTensor.RandomN(new long[] { 64, 10 }, device: "cpu:0"); var eval = seq.Forward(x); var loss = NN.LossFunction.MSE(NN.Reduction.None); var output = loss(eval, y); seq.ZeroGrad(); output.Backward(); }
public void TestGradConditional() { var modT = new CondModel(true); var modF = new CondModel(false); var x = FloatTensor.RandomN(new long[] { 64, 1000 }, device: "cpu:0"); var y = FloatTensor.RandomN(new long[] { 64, 10 }, device: "cpu:0"); modT.Train(); var eval = modT.Forward(x); var loss = NN.LossFunction.MSE(NN.Reduction.None); var output = loss(eval, y); modT.ZeroGrad(); output.Backward(); var gradCounts = 0; foreach (var parm in modT.Parameters()) { var grad = parm.Grad(); gradCounts += grad.Handle == IntPtr.Zero ? 0 : 1; } Assert.Equal(2, gradCounts); modF.Train(); eval = modF.Forward(x); output = loss(eval, y); modF.ZeroGrad(); output.Backward(); gradCounts = 0; foreach (var parm in modF.Parameters()) { var grad = parm.Grad(); gradCounts += grad.Handle == IntPtr.Zero ? 0 : 1; } Assert.Equal(3, gradCounts); }
public void TestLinearWithBias() { var lin = NN.Module.Linear(1000, 100, true); var bias = lin.Bias; var weight = lin.Weight.T(); var input = FloatTensor.RandomN(new long[] { 1, 1000 }); var forward = lin.Forward(input); var matmul = input.MatMul(weight).Add(bias.Value); Assert.Equal(forward.Shape.Length, matmul.Shape.Length); Assert.Equal(forward.Shape[0], matmul.Shape[0]); Assert.Equal(forward.Shape[1], matmul.Shape[1]); for (int i = 0; i < 100; i++) { Assert.InRange(forward.Data <float>()[i], matmul.Data <float>()[i] - 10e5f, matmul.Data <float>()[i] + 10e5f); } }
public void TestLinearEditWeightsAndBias() { var lin = NN.Module.Linear(0, 0, true); var bias = FloatTensor.RandomN(new long[] { 100 }); var weights = FloatTensor.RandomN(new long[] { 100, 1000 }); lin.Bias = bias; lin.Weight = weights; Assert.Equal(lin.Weight.Shape.Length, weights.Shape.Length); Assert.Equal(lin.Weight.Shape[0], weights.Shape[0]); Assert.Equal(lin.Weight.Shape[1], weights.Shape[1]); for (int i = 0; i < 100; i++) { Assert.Equal(lin.Bias.Value.Data <float>()[i], bias.Data <float>()[i]); } }
public void EvalLossSequence() { var lin1 = Linear(1000, 100); var lin2 = Linear(100, 10); var seq = Sequential( ("lin1", lin1), ("relu1", Relu()), ("lin2", lin2)); var x = FloatTensor.RandomN(new long[] { 64, 1000 }); var y = FloatTensor.RandomN(new long[] { 64, 10 }); var eval = seq.Forward(x); var loss = MSE(NN.Reduction.Sum); var output = loss(eval, y); var result = output.DataItem <float>(); }
public void TestTraining() { var lin1 = NN.Module.Linear(1000, 100); var lin2 = NN.Module.Linear(100, 10); var seq = NN.Module.Sequential(lin1, NN.Module.Relu(), lin2); var x = FloatTensor.RandomN(new long[] { 64, 1000 }, device: "cpu:0"); var y = FloatTensor.RandomN(new long[] { 64, 10 }, device: "cpu:0"); float learning_rate = 0.00004f; float prevLoss = float.MaxValue; var loss = NN.LossFunction.MSE(NN.Reduction.Sum); for (int i = 0; i < 10; i++) { var eval = seq.Forward(x); var output = loss(eval, y); var lossVal = output.DataItem <float>(); Assert.True(lossVal < prevLoss); prevLoss = lossVal; seq.ZeroGrad(); output.Backward(); using (var noGrad = new AutoGradMode(false)) { foreach (var param in seq.Parameters()) { var grad = param.Grad(); var update = grad.Mul(learning_rate.ToScalar()); param.SubInPlace(update); } } } }