public void TestScanOnTanhSumDot() { var W = T.Shared(0.2f * NN.Random.Uniform(-1.0f, 1.0f, 4, 5).As <float>(), "W"); Func <Tensor <float>, Tensor <float>, Tensor <float> > recurrence = (x, acc) => T.Tanh(acc + T.Dot(W, x)); var X = T.Matrix <float>(-1, 5, "X"); var acc0 = T.Shared(NN.Zeros <float>(4), "acc0"); var result = T.Scan(fn: recurrence, sequences: new[] { X }, outputsInfo: acc0); var norm2 = T.Norm2(result[-1]); var f = T.Function(X, norm2); var grad = T.Grad(norm2, W); var df = T.Function(input: X, output: (norm2, grad)); df(NN.Array(new[, ] { { 0f, 0f, 0f, 0f, 0f } })); AssertTensor.PassesGradientCheck(X, norm2, acc0); AssertTensor.PassesGradientCheck(X, norm2, W); }
public void Convolve2DPassesGradientCheck() { //int[] poolingShape = new int[] { 1, 1 }; int[] kernelShape = new int[] { 7, 7 }; int[] inputShape = new int[] { 100, 100 }; var iS = NN.Array(inputShape).As <float>(); var kS = NN.Array(kernelShape).As <float>(); // layers var W = T.Shared(NN.Random.Uniform(-0.01f, 0.01f, kernelShape).As <float>(), "W"); //var flatShape = ((inputShape[0] + kernelShape[0] - 1) / poolingShape[0] ) * ((inputShape[1] + kernelShape[1] - 1) / poolingShape[1] ); var flatShape = ((inputShape[0] + kernelShape[0] - 1)) * ((inputShape[1] + kernelShape[1] - 1)); var scaling = (((iS[0] + kS[0] - 1f)) + ((iS[1] + kS[1] - 1f))); var S = T.Shared(NN.Random.Uniform(-10f, 10f, 2, flatShape).As <float>() / scaling, "S"); var Sb = T.Shared(NN.Zeros <float>(2, 1), "Sb"); var x = T.Matrix <float>(inputShape[0], inputShape[1], "x"); // [inputLength] var h = T.Sigmoid(T.Convolve2d(x, W, mode: ConvMode.Full)); //h = T.MaxPooling2d(h, poolingShape[0], poolingShape[1], true); h = h.Reshape(flatShape, 1); var debug = (T.Dot(S, h) + Sb).Reshape(2); var pred = T.Softmax(debug); var nll = -T.Mean(T.Log(pred)[1]); AssertTensor.PassesGradientCheck(x, nll, W, relativeErr: 1e-3f, absErr: 1e-3f); }
public void ScanPassesGradientCheckOnSeq2Seq() { int embeddingSize = 10, vocabSize = 100; var L = T.Shared(NN.Random.Uniform(-0.01f, 0.01f, vocabSize, embeddingSize), "L"); var W = T.Shared(NN.Random.Uniform(-0.01f, 0.01f, embeddingSize, embeddingSize), "W"); var ids = T.Vector <int>(-1, "ids"); var xs = L[ids]; var scan = T.Scan((x, acc) => T.Tanh(T.Dot(acc + x, W)), sequence: xs, outputsInfo: T.Zeros <float>(embeddingSize)); var norm2 = T.Norm2(scan); var grad = T.Grad(norm2); var updates = new OrderedDictionary { [W] = W - 0.001f * grad[W], [L] = L - 0.001f * grad[L] }; var f = T.Function(input: ids, output: norm2, updates: updates); Func <Array <int> > init = () => NN.Random.Uniform(0, vocabSize - 1, 10).As <int>(); f(init()); AssertTensor.PassesGradientCheck(ids, norm2, W, init: init); AssertTensor.PassesGradientCheck(ids, norm2, L, init: init); }
public void ItemPassesGradientCheck() { var y = T.Vector <float>(10, "y"); var b = T.Shared(NN.Random.Uniform(-1f, 1f, 10).As <float>(), "b"); AssertTensor.PassesGradientCheck(y, (y + b).Item[5], b); AssertTensor.PassesGradientCheck(y, (y + b).Item[-3], b); }
public void TanhPerceptronPassesGradientCheck() { var x = T.Vector <float>("x"); int n = 20, m = 5; var W = T.Shared(NN.Random.Uniform(-1f, 1f, m, n).As <float>(), "W"); var W_op = T.Shared(NN.Random.Uniform(-1f, 1f, m, n).As <float>(), "W_op"); var loss = T.Norm2(T.Tanh(T.Dot(W, x)) - T.Dot(W_op, x)); AssertTensor.PassesGradientCheck(x, loss, W); }
public void DimShufflePassesGradientCheck() { var X = T.Matrix <float>(5, 3, "X"); var b = T.Shared(NN.Random.Uniform(-1f, 1f, 3), "b"); var b2 = b.DimShuffle('x', 0); var Xb = X * b2; var loss = T.Norm2(Xb); AssertTensor.PassesGradientCheck(X, loss, b); }
public void MinPassesGradientCheck() { var x = T.Shared(0f, "x"); var min = T.Min(x, 0f); x.Value = 1; AssertTensor.PassesGradientCheck(min, x); x.Value = -1; AssertTensor.PassesGradientCheck(min, x); }
public void ConcatPassesGradientCheck() { var x = T.Shared(NN.Random.Uniform(-1f, 1f, 4, 10), "x"); var y = T.Shared(NN.Random.Uniform(-1f, 1f, 6, 10), "y"); var z = T.Concat(0, x, y); var loss = T.Norm2(z[Range(2, 8)]); AssertTensor.PassesGradientCheck(loss, x); AssertTensor.PassesGradientCheck(loss, y); }
public void TensorDot3Dx1DPassesGradientCheck() { var x = T.Vector <float>("x"); int n = 6, m = 4, l = 2; var W = T.Shared(NN.Random.Uniform(-1f, 1f, l, m, n).As <float>(), "W"); var W_op = T.Shared(NN.Random.Uniform(-1f, 1f, l, m, n).As <float>(), "W_op"); var loss = T.Norm2(T.Dot(W, x) - T.Dot(W_op, x)); AssertTensor.PassesGradientCheck(x, loss, W, relativeErr: 1e-3f, absErr: 1e-4f); }
public void MaxPassesGradientCheck() { var x = T.Shared(0f, "x"); var max = T.Max(x, 0f); x.Value = 1; AssertTensor.PassesGradientCheck(max, x); x.Value = -1; AssertTensor.PassesGradientCheck(max, x); }
public void TensorDot3Dx2DAsEinsteinPassesGradientCheck() { int n = 6, m = 4, l = 2; var x = T.Matrix <float>("x"); var W = T.Shared(NN.Random.Uniform(-1f, 1f, l, m, n).As <float>(), "W"); var W_op = T.Shared(NN.Random.Uniform(-1f, 1f, l, m, n).As <float>(), "W_op"); var loss = T.Norm2(T.EinsteinSum(W, x, "lmn,nx->lmx") - T.Dot(W_op, x)); AssertTensor.PassesGradientCheck(x, loss, W); }
public void TensorDot2Dx3DPassesGradientCheck() { int n = 6, m = 4, l = 2, k = 3; var x = T.Tensor3 <float>(n, m, l, "x"); var xT = x.DimShuffle(1, 0, 2); var W = T.Shared(NN.Random.Uniform(-1f, 1f, l, k).As <float>(), "W"); var W_op = T.Shared(NN.Random.Uniform(-1f, 1f, l, k).As <float>(), "W_op"); var loss = T.Norm2(T.Softmax(T.Dot(xT, W)) - T.Dot(xT, W_op)); AssertTensor.PassesGradientCheck(x, loss, W); }
public void LogSumExpPassesGradientCheck() { var x = T.Matrix <float>(5, 4, "x"); var W = T.Shared(NN.Random.Uniform(-1f, 1f, 8, 5).As <float>(), "W"); AssertTensor.PassesGradientCheck(x, T.Sum(T.LogSumExp(T.Dot(W, x))), W); AssertTensor.PassesGradientCheck(x, T.LogSumExp(T.Dot(W, x)).Item[2], W); var y = T.Vector <float>(10, "y"); var b = T.Shared(NN.Random.Uniform(-1f, 1f, 10).As <float>(), "b"); AssertTensor.PassesGradientCheck(y, (Scalar <float>)T.LogSumExp(y + b), b); }
public void TensorDot3Dx3DPassesGradientCheck() { int n = 6, m = 4, l = 2, k = 5; var x = T.Tensor3 <float>(k, l, m, "x"); var W = T.Shared(NN.Random.Uniform(-1f, 1f, l, m, n).As <float>(), "W"); var W_op = T.Shared(NN.Random.Uniform(-1f, 1f, l, m, n).As <float>(), "W_op"); var axesW = new[] { 0, 1 }; var axesX = new[] { 1, 2 }; var loss = T.Norm2(T.TensorDot(W, axesW, x, axesX) - T.TensorDot(W_op, axesW, x, axesX)); AssertTensor.PassesGradientCheck(x, loss, W); }
// TODO: this test doesn't work due to bugs in Elementwise public void PushCoherentGradientOnComplexAbstraction() { var x = T.Shared(NN.Range <float>(4), "x"); var b = T.Scalar <float>("b"); var y = T.Apply(x, x_ => (x_ > 0f) * b + x_ + b); var loss = T.Sum(y); AssertArray.WithMessage("Can't compile the gradient.", () => T.Function(input: b, output: T.Grad(loss, b)) ); AssertTensor.PassesGradientCheck(loss, b); }
public void MaxPoolingPassesGradientCheck() { var x_ = NN.Array(new float[, ] { { 0, 1, 0 }, { 1, 0, 0 }, { 0, 0, 1 }, }); var x = T.Shared(x_, "x"); var x_pooled = T.Max(x, axis: 1); var loss = T.Sum(x_pooled); AssertTensor.PassesGradientCheck(loss, x); }
public void TanhNNTransposedAsEinsteinSumPassesGradientCheck() { var x0 = T.Vector <float>("x"); int n0 = 20, n1 = 5, n2 = 20; var W1 = T.Shared(NN.Random.Uniform(-1f, 1f, n0, n1).As <float>(), "W1"); var W2 = T.Shared(NN.Random.Uniform(-1f, 1f, n1, n2).As <float>(), "W2"); var b1 = T.Shared(NN.Random.Uniform(-1f, 1f, n1).As <float>(), "b1"); var b2 = T.Shared(NN.Random.Uniform(-0.1f, 0.1f, n2).As <float>(), "b2"); var W_op = T.Shared(NN.Random.Uniform(-1f, 1f, n0, n2).As <float>(), "W_op"); var x1 = T.Tanh(T.EinsteinSum(x0, W1, "i,ij->j") + b1); var x2 = T.Tanh(T.EinsteinSum(x1, W2, "i,ij->j") + b2); var loss = T.Norm2(x2 - T.Dot(W_op, x0)); AssertTensor.PassesGradientCheck(x0, loss, b1); AssertTensor.PassesGradientCheck(x0, loss, W1); AssertTensor.PassesGradientCheck(x0, loss, W2); AssertTensor.PassesGradientCheck(x0, loss, b2, absErr: 0.002f); }
public void ReluNNPassesGradientCheck() { var x0 = T.Vector <float>("x"); int n0 = 20, n1 = 5, n2 = 20; var W1 = T.Shared(NN.Random.Uniform(-1f, 1f, n1, n0).As <float>(), "W1"); var W2 = T.Shared(NN.Random.Uniform(-1f, 1f, n2, n1).As <float>(), "W2"); var b1 = T.Shared(NN.Random.Uniform(-1f, 1f, n1).As <float>(), "b1"); var b2 = T.Shared(NN.Random.Uniform(-0.1f, 0.1f, n2).As <float>(), "b2"); var W_op = T.Shared(NN.Random.Uniform(-1f, 1f, n2, n0).As <float>(), "W_op"); var x1 = T.ReLu(T.Dot(W1, x0) + b1); var x2 = T.ReLu(T.Dot(W2, x1) + b2); var loss = T.Norm2(x2 - T.Dot(W_op, x0)); AssertTensor.PassesGradientCheck(x0, loss, b1); AssertTensor.PassesGradientCheck(x0, loss, W1); AssertTensor.PassesGradientCheck(x0, loss, W2); AssertTensor.PassesGradientCheck(x0, loss, b2); }
public void IndexingPassesGradientCheck() { var maxLength = 20; var n = 10; var ids = T.Vector <int>("X"); var W = T.Shared(NN.Random.Uniform(-1f, 1f, n, n), "W"); var loss = T.Sum(W[ids]); AssertTensor.PassesGradientCheck( ids, loss, W, init: () => NN.Random.Uniform <int>(0, n - 1, (int)(NN.Random.NextDouble() * maxLength) + 1) ); var loss2 = T.Sum(W[ids, ids]); AssertTensor.PassesGradientCheck( ids, loss2, W, init: () => NN.Random.Uniform <int>(0, n - 1, (int)(NN.Random.NextDouble() * maxLength) + 1) ); }