public void TestCombineWithBias3D_1D() { var t = NN.Ones <float>(6, 5, 4); t[2, _, _] *= 2; t[_, 1, _] *= -1; t[_, _, 2] *= 3; var x = NN.Array <float>(1, -1, -2); var y = NN.Array <float>(1, -1, 3, 1); Array <float> txy2 = null; txy2 = t[_, Upto(-1), Upto(-1)].Combine(x, y, result: txy2); var txy = t.CombineWithBias(x, y); var xb = NN.Ones <float>(4); xb[Upto(-1)] = x; var yb = NN.Ones <float>(5); yb[Upto(-1)] = y; var txbyb = t.Combine(xb, yb); AssertArray.AreAlmostEqual(txbyb, txy); }
public void TestCombine() { var t = NN.Ones <float>(5, 4, 3); t[2, _, _] *= 2; t[_, 1, _] *= -1; t[_, _, 2] *= 3; var x = NN.Array <float>(1, -1, 3); var y = NN.Array <float>(1, -1, 3, 2); var txy = t.Combine(x, y); var z = NN.Zeros <float>(5); for (int k = 0; k < z.Shape[0]; ++k) { for (int j = 0; j < y.Shape[0]; ++j) { for (int i = 0; i < x.Shape[0]; ++i) { z.Item[k] += t.Item[k, j, i] * y.Item[j] * x.Item[i]; } } } var expected = new float[] { 63, 63, 126, 63, 63 }; AssertArray.AreAlmostEqual(expected, z); AssertArray.AreAlmostEqual(expected, t.Dot(x).Dot(y)); AssertArray.AreAlmostEqual(expected, txy); }
public void TestLoop1() { // Computing tanh(x(t).dot(W) + b) elementwise //http://deeplearning.net/software/theano/tutorial/loop.html // defining the tensor variables var X = T.Matrix <float>("x"); var W = T.Matrix <float>("W"); var b_sym = T.Vector <float>("b_sym"); var results = T.Scan(v => T.Tanh(T.Dot(v, W) + b_sym), sequence: X); var compute_elementwise = T.Function(inputs: new[] { X, W, b_sym }, output: results); // test values var x = NN.Eye <float>(2); var w = NN.Ones <float>(2, 2); var b = NN.Ones <float>(2); b.Item[1] = 2; var result = compute_elementwise(new[] { x, w, b }); var expected = NN.Tanh(x.Dot(w) + b); AssertArray.AreAlmostEqual(expected[0], result[0]); }
public void SumAsEinsteinSumMatchesSum() { var x = NN.Random.Uniform(-1f, 1f, 10); var y = NN.Ones(1); AssertArray.AreEqual(x.Sum(axis: 0), NN.EinsteinSum(x, y, "i,->")); }
public void TestCombine2() { var a = NN.Ones <float>(4, 5, 6); var x = NN.Ones <float>(6); var y = NN.Ones <float>(5); var z = a.Combine(x, y); var expected = NN.Array <float>(30, 30, 30, 30); AssertArray.AreAlmostEqual(expected, z); }
public void TestDotWithIdentity() { var a = NN.Ones <float>(4, 5); a[_, Upto(-1)] = NN.Eye <float>(4); var b = NN.Random.Uniform(-1, 1, 4).As <float>(); var c = NN.Ones <float>(5); c[Upto(4)] = b; var ac = a.Dot(c); var ab = a.DotWithBias(b); AssertArray.AreEqual(ac, ab); }
public void TestDotWithBias1D_1D() { var a = NN.Array <float>(1, 2, 1, 3); var b = NN.Array <float>(1, -1, 1); AssertArray.AreAlmostEqual(3, (float)a.DotWithBias(b)); a = NN.Random.Uniform(-1, 1, 5).As <float>(); b = NN.Random.Uniform(-1, 1, 4).As <float>(); var c = NN.Ones <float>(5); c[Upto(4)] = b; AssertArray.AreAlmostEqual(a.Dot(c), a.DotWithBias(b)); }
public void TestDotWithBias2D_1D() { var a = NN.Array <float>(new float[, ] { { 1, 1, 3, 1 }, { 2, 2, 6, 2 }, { 1, 1, 3, 1 } }); var b = NN.Array <float>(-1, 1, 4); var c = NN.Ones <float>(4); c[Upto(-1)] = b; var ac = a.Dot(c); var ab = a.DotWithBias(b); AssertArray.AreEqual(ac, ab); }
public void SumProductWithSharedCanTrain() { var n = 2; // sequence of input var xs = T.Matrix <float>("xs"); // accumulator var z = T.Vector <float>("z"); var b = T.Shared(NN.Ones(n), "b"); // sum xs in the accumulator Func <Tensor <float>, Tensor <float>, IList <Tensor <float> > > rec = (x, a) => new List <Tensor <float> >() { x + a, x *a + b }; var loop = T.Scan(rec, xs, new[] { z, null }); // get the last value var prod = loop[1][-1]; // compute the cost and the gradient for the shared b. var cost = T.Sum(prod); var db = T.Grad(cost, b); var costFunction = T.Function(input: (xs, z), output: cost); var xs_ = NN.Array(new float[, ] { { 1, -1 }, { 0, -2 } }); var z_ = NN.Zeros(n); var cost_xs_z = costFunction(xs_, z_); Assert.AreEqual(4, cost_xs_z); var updates = new OrderedDictionary { { b, b - 0.05f * db } }; var train = T.Function(input: (xs, z), output: cost, updates: updates); var cost_xs_z2 = train(xs_, z_); AssertArray.AreAlmostEqual(NN.Array(new[] { 0.95f, 0.95f }), b.Value); }
public void CanSwapTwoShared() { var a = make_shared("a", 5, 4, 3); Assert.IsTrue(a.Value.Shape.Zip(new int[] { 5, 4, 3 }, (x, y) => x == y).All(x => x)); var b = make_shared("b", 5, 4, 3); Assert.IsTrue(b.Value.Shape.Zip(new int[] { 5, 4, 3 }, (x, y) => x == y).All(x => x)); a.Value = NN.Zeros <float>(5, 4, 3); b.Value = NN.Ones <float>(5, 4, 3); exchange_shared(a, b); Assert.IsTrue(a.Value.All(x => x == 1.0f)); Assert.IsTrue(b.Value.All(x => x == 0.0f)); var f = make_exchange_func(a, b); f(); Assert.IsTrue(a.Value.All(x => x == 0.0f)); Assert.IsTrue(b.Value.All(x => x == 1.0f)); }
public void TestRecursive() { // http://deeplearning.net/software/theano/tutorial/loop.html // define tensor variables var X = T.Vector <float>("X"); var W = T.Matrix <float>("W"); var b_sym = T.Matrix <float>("b_sym"); var U = T.Matrix <float>("U"); var Y = T.Matrix <float>("Y"); var V = T.Matrix <float>("V"); var P = T.Matrix <float>("P"); var results = T.Scan((yy, pp, xx_tm1) => T.Tanh(T.Dot(xx_tm1, W) + T.Dot(yy, U) + T.Dot(pp, V)), sequences: new[] { Y, P[XSlicer.Step(-1)] }, outputsInfo: X); var compute_seq = T.Function(inputs: new[] { X, W, Y, U, P, V }, output: results); // test values var x = NN.Zeros <float>(2); x.Item[1] = 1; var w = NN.Ones <float>(2, 2); var y = NN.Ones <float>(5, 2); y.Item[0] = -3; var u = NN.Ones <float>(2, 2); var p = NN.Ones <float>(5, 2); p.Item[0] = 3; var v = NN.Ones <float>(2, 2); var result = compute_seq(new[] { x, w, y, u, p, v }); // Array<float>[5] => theano returns Array<float>[5][1] // comparison with numpy var x_res = NN.Zeros <float>(5, 2); x_res[0] = NN.Tanh(x.Dot(w) + y[0].Dot(u) + p[4].Dot(v)); for (int i = 1; i < 5; i++) { x_res[i] = NN.Tanh(x_res[i - 1].Dot(w) + y[i].Dot(u) + p[4 - i].Dot(v)); } AssertArray.AreAlmostEqual(x_res, result); }
public Tsne(Array <float> X_, int dims, float perplexity) { X_.AssertOfDim(2); int n = X_.Shape[0]; X = T.Shared(X_, "X"); Y = T.Shared(NN.Random.Uniform(-1f, 1f, n, dims), "Y"); YMomentum = T.Shared(NN.Zeros(n, dims), "YMomentum"); dYLast = T.Shared(NN.Zeros(n, dims), "dYLast"); // ones everywhere, zero on the diag mask = T.Shared(NN.Ones(n, n) - NN.Eye(n), "mask"); // Compute pairwise affinities var sum_Y = T.Sum(Y * Y, 1, keepDims: true); var num = 1 / (1 - T.DimShuffle((2 * T.Dot(Y, Y, transposeY: true) + sum_Y), 1, 0) + sum_Y); // set the diag to zero num *= mask; var Q = num / T.Sum(num); //Q = T.Max(Q, 1e-12f); var P_ = x2p(X_, 1e-5f, perplexity); P_ = P_ * 4f; // early exaggeration P_ = NN.Apply(P_, x => Math.Max(x, 1e-12f)); P = T.Shared(P_, "P"); KL_Loss = T.Sum(P * T.Log(P / Q)); dY = T.Function(output: T.Grad(KL_Loss, Y)); Loss = T.Function(output: KL_Loss); var updates = MomentumUpdate(Y, YMomentum, dYLast, T.Grad(KL_Loss, Y), 500); Train = T.Function(updates); }
public static void TestLook1() { // defining the tensor variables var X = T.Matrix <float>("x"); var W = T.Matrix <float>("W"); var b_sym = T.Matrix <float>("b_sym"); var results = T.Scan(v => T.Tanh(T.Dot(v, W) + b_sym), sequence: X); var compute_elementwise = T.Function(inputs: new[] { X, W, b_sym }, output: results); // test values var x = NN.Eye <float>(2); var w = NN.Ones <float>(2, 2); var b = NN.Ones <float>(2); b.Item[1] = 2; Console.WriteLine(compute_elementwise(new[] { x, w, b }).Item[0]); // comparison with tensors Console.WriteLine(NN.Tanh(x.Dot(w) + b)); }
public void TestGradientThroughImplicitBroadcast() { var M = T.Matrix <float>("M"); var W_ = NN.Array(new float[, ] { { 0, 0, 1 }, { 1, 1, 0 } }); var W = T.Shared(W_, "W"); var X = T.Matrix <float>(-1, 1, "X"); var loss = T.Sum((M + X) * W); var M_ = NN.Array(new float[, ] { { 0, 3, 7 }, { 5, 2, 0 } }); var X_ = NN.Ones(2).Reshape(-1, 1); var dL = T.Function(input: (X, M), output: T.Grad(loss, X)); var dX = NN.Array(new float[, ] { { 1 }, { 2 } }); AssertArray.AreAlmostEqual(dX, dL(X_, M_)); }
public void NumNetInitOnes(int n, int m) { var a = NN.Ones(n, m); }
public Array <float> x2p(Array <float> X, float tol = 1e-5f, float perplexity = 30f, bool sym = true, bool normalize = true) { //"""Performs a binary search to get P-values in such a way that each conditional Gaussian has the same perplexity.""" // Initialize some variables Console.WriteLine("Computing pairwise distances..."); int n = X.Shape[0], d = X.Shape[1]; var sum_X = NN.Sum(X * X, axis: 1); var D = (-2 * X.Dot(X.T) + sum_X).T + sum_X; var P = NN.Zeros(n, n); var beta = NN.Ones(n); var logU = (float)Math.Log(perplexity); var Di = NN.Zeros(n, n - 1); // Loop over all datapoints for (int i = 0; i < n; ++i) { // Print progress if (i % 500 == 0) { Console.WriteLine("Computing P-values for point {0} of {1} ...", i, n); } // Compute the Gaussian kernel and entropy for the current precision var betamin = float.NegativeInfinity; var betamax = float.PositiveInfinity; Di[i, Until(i)] = D[Until(i)]; if (i + 1 < n) { Di[i, From(i + 1)] = D[From(i)]; } var H_thisP = Hbeta(Di, beta.Item[i]); var H = H_thisP.Item1; var thisP = H_thisP.Item2; // Evaluate whether the perplexity is within tolerance var Hdiff = H - logU; var tries = 0; while (Math.Abs(Hdiff) > tol && tries < 50) { // If not, increase or decrease precision if (Hdiff > 0) { betamin = beta.Item[i]; if (float.IsInfinity(betamax)) { beta.Item[i] = beta.Item[i] * 2; } else { beta.Item[i] = (beta.Item[i] + betamax) / 2; } } else { betamax = beta.Item[i]; if (float.IsInfinity(betamin)) { beta.Item[i] = beta.Item[i] / 2; } else { beta.Item[i] = (beta.Item[i] + betamin) / 2; } } // Recompute the values H_thisP = Hbeta(Di, beta.Item[i]); H = H_thisP.Item1; thisP = H_thisP.Item2; Hdiff = H - logU; tries = tries + 1; } // Set the final row of P P[i, Until(i)] = thisP[Until(i)]; if (i + 1 < n) { P[i, From(i + 1)] = thisP[From(i)]; } } var sigma = NN.Mean(NN.Sqrt(1 / beta)); Console.WriteLine("Mean value of sigma: {0}", sigma); // Return final P-matrix if (sym) { P += P.T; } if (normalize) { P /= NN.Sum(P); } return(P); }