Esempio n. 1
0
        public FloatTensor Pow(FloatTensor x, bool inline = false, FloatTensor result = null)
        {
            if (!IsContiguous() || !x.IsContiguous())
            {
                throw new InvalidOperationException("All tensors must be contiguous, call Contiguous() to convert");
            }

            // Check if both tensors are compatible for sum
            SameSizeDimensionsShapeAndLocation(ref x);

            result = HookAutograd(ref result, ref x, "pow_elem", inline);

            if (dataOnGpu)
            {
                result.Gpu(shader);
                if (!inline)
                {
                    return(PowElemGPU(x, result));
                }
                result.PowElemGPU_(x);
                return(this);
            }

            result.Data = data.AsParallel().Zip(x.Data.AsParallel(), (a, b) => (float)Math.Pow((double)a, b))
                          .ToArray();

            return(result);
        }
Esempio n. 2
0
        public FloatTensor Sub(FloatTensor x, bool inline = false, FloatTensor result = null)
        {
            if (!IsContiguous() || !x.IsContiguous())
            {
                throw new InvalidOperationException("All tensors must be contiguous, call Contiguous() to convert");
            }

            // Check if both tensors are compatible for sum
            SameSizeDimensionsShapeAndLocation(ref x);

            result = HookAutograd(ref result, ref x, "sub_elem", inline);

            if (dataOnGpu & x.dataOnGpu)
            {
                if (inline)
                {
                    if (autograd)
                    {
                        throw new InvalidOperationException(
                                  "Cannot call inline functions if you intend to run backprop.");
                    }
                    SubElemGPU_(x);
                    return(this);
                }
                result = SubElemGPU(x, result);
            }
            else
            {
                result.Data = data.AsParallel().Zip(x.Data.AsParallel(), (a, b) => a - b).ToArray();
            }

            return(result);
        }
Esempio n. 3
0
        public FloatTensor AddMatrixVectorProduct(FloatTensor matrix, FloatTensor vector)
        {
            if (!IsContiguous() || !matrix.IsContiguous() || !vector.IsContiguous())
            {
                throw new InvalidOperationException("Tensor must be contiguous, call Contiguous() to convert");
            }

            var gpu = dataOnGpu & matrix.DataOnGpu & vector.DataOnGpu;
            var cpu = !(dataOnGpu | matrix.DataOnGpu | vector.DataOnGpu);

            var ref_shape    = this.Shape;
            var matrix_shape = matrix.Shape;
            var vector_shape = vector.Shape;

            if (ref_shape.Length != 1)
            {
                throw new InvalidOperationException(
                          "Cannot perform this operation on a tensor with more than one dimension");
            }
            if (ref_shape[0] != vector_shape[0])
            {
                throw new InvalidOperationException(String.Format(
                                                        "Cannot add matrix-vector product to tensor: {0} & {1}.", ref_shape[0], vector_shape[0]));
            }
            if (matrix_shape[1] != vector_shape[0])
            {
                throw new InvalidOperationException(String.Format("Last dimension of matrix doesn't match: {0} vs {1}.",
                                                                  matrix_shape[1], vector_shape[0]));
            }

            if (gpu)
            {
                AddMatrixVectorProductGPU(matrix, vector);
            }
            else if (cpu)
            {
                var nCpu = SystemInfo.processorCount;
                Parallel.For(0, nCpu, workerId =>
                {
                    var max = size * (workerId + 1) / nCpu;
                    for (var idx = size * workerId / nCpu; idx < max; idx++)
                    {
                        for (var j = 0; j < ref_shape[0]; j++)
                        {
                            this[idx] += vector[j] * matrix[j + (idx * ref_shape[0])];
                        }
                    }
                });
            }
            else
            {
                Debug.Log("Data for all Tensors needs to be colocated on the same device. - CPU != GPU");
            }

            return(this);
        }
Esempio n. 4
0
        public FloatTensor AddMatrixMultiply(FloatTensor tensor1, FloatTensor tensor2)
        {
            if (!IsContiguous() || !tensor1.IsContiguous() || !tensor2.IsContiguous())
            {
                throw new InvalidOperationException("All tensors must be contiguous, call Contiguous() to convert");
            }

            bool gpu = dataOnGpu & tensor1.DataOnGpu & tensor2.DataOnGpu;
            bool cpu = !(dataOnGpu | tensor1.DataOnGpu | tensor2.DataOnGpu);

            int[] res_shape = this.Shape;
            int[] shape1    = tensor1.Shape;
            int[] shape2    = tensor2.Shape;

            if (shape1[1] != shape2[0])
            {
                throw new InvalidOperationException(String.Format("Matrix multiply not possible: {0} & {1}.", shape1[1], shape2[0]));
            }
            if (res_shape[0] != shape1[0])
            {
                throw new InvalidOperationException(String.Format("First dimension doesn't match: {0} vs {1}.", res_shape[0], shape1[0]));
            }
            if (res_shape[1] != shape2[1])
            {
                throw new InvalidOperationException(String.Format("Last dimension doesn't match: {0} vs {1}.", res_shape[res_shape.Length - 1], shape2[shape2.Length - 1]));
            }

            if (gpu)
            {
                AddMatrixMultiplyGPU(tensor1, tensor2);
            }
            else if (cpu)
            {
                var nCpu = SystemInfo.processorCount;
                Parallel.For(0, nCpu, workerId =>
                {
                    var max = size * (workerId + 1) / nCpu;
                    for (var idx = size * workerId / nCpu; idx < max; idx++)
                    {
                        int col        = idx % res_shape[1];
                        int row        = (idx - col) / res_shape[1];
                        int row_offset = row * shape1[1];
                        for (var j = 0; j < shape1[1]; j++)
                        {
                            Data[idx] += tensor1.Data[j + row_offset] * tensor2.Data[j * shape2[1] + col];
                        }
                    }
                });
                return(this);
            }
            else
            {
                Debug.Log("Data for all Tensors needs to be colocated on the same device. - CPU != GPU");
            }
            return(this);
        }
Esempio n. 5
0
        public FloatTensor Remainder(FloatTensor divisor, bool inline = false)
        {
            if (!IsContiguous() || !divisor.IsContiguous())
            {
                throw new InvalidOperationException("All tensor must be contiguous, call Contiguous() to convert");
            }

            SameSizeDimensionsShapeAndLocation(ref divisor);
            if (inline & autograd)
            {
                throw new InvalidOperationException("Cannot call inline functions if you intend to run backprop.");
            }
            if (autograd)
            {
                throw new InvalidOperationException("Autograd not available for Remainder.");
            }

            var result = inline ? this : this.emptyTensorCopy();

            if (dataOnGpu)
            {
                result.Gpu(shader);
                if (inline)
                {
                    RemainderElemGPU_(divisor);
                    return(this);
                }
                else
                {
                    result = RemainderElemGPU(divisor, result);
                }
            }
            else
            {
                var nCpu = SystemInfo.processorCount;
                Parallel.For(0, nCpu, workerId => {
                    var max = size * (workerId + 1) / nCpu;
                    for (var i = size * workerId / nCpu; i < max; i++)
                    {
                        result[i] = this[i] % divisor[i];
                    }
                    ;
                });
            }

            return(result);
        }
Esempio n. 6
0
        public FloatTensor Add(FloatTensor x, bool inline = false, FloatTensor result = null)
        {
            if (!IsContiguous() || !x.IsContiguous())
            {
                throw new InvalidOperationException("All tensors must be contiguous, call Contiguous() to convert");
            }

            // Check if both tensors are compatible for sum
            SameSizeDimensionsShapeAndLocation(ref x);


            result = HookAutograd(ref result, ref x, "add_elem", inline);


            if (dataOnGpu)
            {
                if (inline)
                {
                    if (autograd)
                    {
                        throw new InvalidOperationException("Cannot call inline functions if you intend to run backprop.");
                    }


                    AddElemGPU_(x);
                    return(this);
                }
                else
                {
                    return(AddElemGPU(x, result));
                }
            }

            var nCpu = SystemInfo.processorCount;

            Parallel.For(0, nCpu, workerId => {
                var max = size * (workerId + 1) / nCpu;
                for (var i = size * workerId / nCpu; i < max; i++)
                {
                    result.Data [i] = x.Data [i] + Data [i];
                }
            });


            return(result);
        }
Esempio n. 7
0
        public FloatTensor MM(FloatTensor x, FloatTensor result = null)
        {
            if (!IsContiguous() || !x.IsContiguous())
            {
                throw new InvalidOperationException("All tensors must be contiguous, call Contiguous() to convert");
            }

            if (this.shape.Length != 2 || x.shape.Length != 2)
            {
                throw new InvalidOperationException(
                          "Cannot do MM on tensors that aren't 2 dimentional. Try calling view() to reshape");
            }

            result = HookAutograd(ref result, ref x, "mm", false, new int[] { shape[0], x.shape[1] });

            result.AddMatrixMultiply(this, x);

            return(result);
        }
Esempio n. 8
0
        public void Backward(FloatTensor grad = null, FloatTensor grad_origin = null)
        {
            //Debug.Log("Backward:" + this.id + " creation_op:" + creation_op);

            if (autograd)
            {
                if (grad == null)
                {
                    Debug.Log("Grad not Found... Creating Gradient of 1s");
                    grad          = this.createOnesTensorLike();
                    grad.Autograd = false;
                }

                if (grad_origin != null)
                {
                    int child_index = children_indices.IndexOf(grad_origin.Id);
                    if (children_counts[child_index] > 0)
                    {
                        throw new InvalidOperationException("Can't backprop more than once.");
                    }
                    else
                    {
                        children_counts[child_index] += 1;
                    }
                }

                if (this.Grad == null)
                {
                    this.Grad = grad;
                    //Debug.Log("Setting Grad Tensor Id:" + this.id);
                }
                else
                {
                    if (this.Grad.id == grad.id)
                    {
                        // do nothing
                        //Debug.Log("Not Updating For Tensor Id:" + this.id);
                    }
                    else
                    {
                        //Debug.Log("Updating For Tensor Id:" + this.id);
                        //this.Grad.Zero_();
                        this.Grad.Add(grad, inline: true);
                    }
                }

                // grads must not have grads of their own
                if (this.Grad.autograd == true)
                {
                    throw new InvalidOperationException("Sorry, grads cannot have grads");
                }

                // RULES FOR AUTOGRAD:
                // 1) if you need to use "this" for calculating a gradient, copy it first and set autograd to false (see sigmoid)
                // 2) if you use a method in your backprop logic that doesn't hook into the dynamic graph yet, backprop
                // will not work!!! Make sure there's a "hookautograd" function in every method you use for backprop.
                // 3) whenever backpropping into a method where the forward prop involved a scalar (such as scalar
                // multiplication), current implementations assume you will NOT backprop into the scalar itself.
                // 4) Because of rule (2), do NOT use "emptyTensorCopy" at all in backprop unless you know what you're
                // doing.
                // 5) I will be especially strict about Unit tests for all backprop logic as this is the most complex
                // piece of functionality we have. Furthermore, most errors go completely undetected (not discovered
                // by runtime errors). Autograd bugs just make convergence go slowly and sub-optimally.
                // 6) If you use a forward propagation tensor to backprop, you MUST remember to turn off autograd
                // when backpropagating (see "mm" below for example). Otherwise, it will cause autograd to break because
                // whatever child you select will think it needs to wait for another gradient before backpropagating.
                // 7) In the "view" backprop method, you'll notice that we set parent.grad = null. This keeps grads from
                // accumulating when forward and backprop is called multiple times. However, it doesn't cause any new
                // memory allocation.

                // only continue backpropping if there's something to backprop into
                // only continue backpropping if all gradients (from children) are accounted for
                // override waiting for children if "backprop" was called on this variable directly
                if (this.creators != null && this.creators.Count > 0 && (grad_origin == null || AllAutogradChildrenAccountedFor()))
                {
                    if (creation_op == "abs")
                    {
                        FloatTensor c = this.Copy(autograd: false);

                        var parent = factory.Get(creators[0]);

                        parent.Backward(parent.Div(c).Mul(grad));
                    }
                    else if (creation_op == "add_elem")
                    {
                        factory.Get(creators[0]).Backward(grad, this);
                        factory.Get(creators[1]).Backward(grad, this);
                    }
                    else if (creation_op == "add_scalar")
                    {
                        factory.Get(creators[0]).Backward(grad, this);
                    }
                    else if (creation_op.Contains("concatenate_"))
                    {
                        int dim = int.Parse(creation_op.Split('_')[1]);

                        for (int i = 0; i < creators.Count; i++)
                        {
                            FloatTensor slice = grad.IndexSelect(factory.ctrl.intTensorFactory.Get(int_creators[i]), dim);

                            factory.Get(creators[i]).Backward(slice);
                        }
                    }
                    else if (creation_op == "contiguous")
                    {
                        //Debug.Log("Contiguous Backpropping Grad:" + grad.Id);
                        //Debug.Log("Contiguous Storing Grad:" + this.Grad.Id);
                        factory.Get(creators[0]).Backward(this.Grad.Copy(autograd: this.Grad.Autograd), this);
                    }
                    else if (creation_op == "copy")
                    {
                        factory.Get(creators[0]).Backward(grad, this);
                    }
                    else if (creation_op == "div_elem")
                    {
                        FloatTensor x = factory.Get(creators[0]);
                        FloatTensor y = factory.Get(creators[1]);

                        x.Backward(grad.Div(y));

                        FloatTensor y2       = y.Pow(2);
                        FloatTensor xn       = x.Neg();
                        FloatTensor xny2     = xn.Div(y2);
                        FloatTensor gradxny2 = grad.Mul(xny2);
                        y.Backward(gradxny2);
                    }
                    else if (creation_op == "div_scalar")
                    {
                        factory.Get(creators[0]).Backward(grad.Div(factory.Get(creators[1]).data[0]), this);
                    }
                    else if (creation_op == "emptyTensorCopy_Hooked")
                    {
                        factory.Get(creators[0]).Backward(grad, this);
                    }
                    else if (creation_op == "expand")
                    {
                        var parent = factory.Get(creators[0]);
                        parent.Grad = null;

                        FloatTensor local_grad = grad.Copy(autograd: grad.Autograd);

                        var grad_shape = new int[shape.Length];

                        for (int i = 0; i < grad.shape.Length; i++)
                        {
                            grad_shape[i] = grad.shape[i];
                        }

                        for (int i = 0; i < shape.Length; i++)
                        {
                            grad_shape[i] = parent.shape[i];
                            if (parent.shape[i] == 1 && shape[i] > 1)
                            {
                                local_grad = local_grad.Sum(i).View(grad_shape);
                            }
                        }

                        parent.Backward(local_grad, this);
                    }
                    else if (creation_op.Contains("shaped_index_select"))
                    {
                        FloatTensor parent    = factory.Get(creators[0]);
                        IntTensor   indices   = factory.ctrl.intTensorFactory.Get(int_creators[0]);
                        FloatTensor back_grad = parent.emptyTensorCopy(hook_graph: true);
                        back_grad.autograd = false;
                        back_grad.Zero_();

                        FloatTensor out_grad = back_grad.IndexAdd(indices, -1, grad);
                        parent.Backward(out_grad);
                    }
                    else if (creation_op.Contains("index_select"))
                    {
                        FloatTensor parent = factory.Get(creators[0]);

                        IntTensor indices = factory.ctrl.intTensorFactory.Get(int_creators[0]);

                        int dim = int.Parse(creation_op.Split('_')[2]);

                        FloatTensor back_grad = parent.emptyTensorCopy(hook_graph: true);
                        back_grad.autograd = false;

                        FloatTensor out_grad = back_grad.IndexAdd(indices, dim, grad);
                        parent.Backward(out_grad);
                    }
                    else if (creation_op == "log")
                    {
                        // TOOD: sum backprop logic
                        FloatTensor x = factory.Get(creators[0]).Copy(autograd: false);

                        factory.Get(creators[0]).Backward(grad.Mul(x.Pow(-1)), this);
                    }
                    else if (creation_op == "mul_elem")
                    {
                        factory.Get(creators[0]).Backward(grad.Mul(factory.Get(creators[1])), this);
                        factory.Get(creators[1]).Backward(grad.Mul(factory.Get(creators[0])), this);
                    }
                    else if (creation_op == "mul_scalar")
                    {
                        factory.Get(creators[0]).Backward(grad.Mul(factory.Get(creators[1]).data[0]), this);
                    }
                    else if (creation_op == "mm")
                    {
                        FloatTensor x = factory.Get(creators[1]).Transpose();
                        x.autograd = false;

                        FloatTensor y = factory.Get(creators[0]).Transpose();
                        y.autograd = false;

                        factory.Get(creators[0]).Backward(grad.MM(x), this);
                        factory.Get(creators[1]).Backward(y.MM(grad), this);
                    }
                    else if (creation_op == "neg")
                    {
                        factory.Get(creators[0]).Backward(grad.Neg(), this);
                    }
                    else if (creation_op == "pow_scalar")
                    {
                        FloatTensor x = factory.Get(creators[0]).Copy(autograd: false);

                        factory.Get(creators[0]).Backward(x.Mul(grad).Mul(factory.Get(creators[1]).Data[0]), this);
                    }
                    else if (creation_op == "relu")
                    {
                        // TOOD: replace with simple comparison and mulitplication (should be 2 liner)
                        FloatTensor c = this.Copy(autograd: false);

                        FloatTensor output = c;

                        var dimSize = 1;

                        for (var i = 0; i < output.Shape.Length; ++i)
                        {
                            dimSize *= output.Shape[i];
                        }

                        var gradInput = output.Copy(autograd: false);
                        gradInput.Zero_();

                        var nCpu = SystemInfo.processorCount;
                        Parallel.For(0, nCpu, workerId =>
                        {
                            var max = dimSize * (workerId + 1) / nCpu;
                            for (var i = dimSize * workerId / nCpu; i < max; i++)
                            {
                                if (output.Data[i] > 0)
                                {
                                    gradInput.Data[i] = 1;
                                }
                                else
                                {
                                    gradInput.Data[i] = 0;
                                }
                            }
                        });

                        factory.Get(creators[0]).Backward((gradInput).Mul(grad), this);
                    }
                    else if (creation_op == "sub_elem")
                    {
                        factory.Get(creators[0]).Backward(grad, this);
                        factory.Get(creators[1]).Backward(grad.Neg(), this);
                    }
                    else if (creation_op == "sub_scalar")
                    {
                        factory.Get(creators[0]).Backward(grad, this);
                    }
                    else if (creation_op == "sigmoid")
                    {
                        FloatTensor self_nograd = this.Copy(autograd: false);

                        factory.Get(creators[0]).Backward(self_nograd.Neg().Add(1f).Mul(self_nograd).Mul(grad), this);
                    }
                    else if (creation_op.Contains("softmax-"))
                    {
                        FloatTensor c   = this.Copy(autograd: false);
                        var         dim = int.Parse(creation_op.Split('-')[1]);

                        FloatTensor output     = this;
                        FloatTensor gradOutput = grad;

                        if (!output.IsContiguous() || !gradOutput.IsContiguous())
                        {
                            throw new NotImplementedException(
                                      "Softmax Gradient does not support non-contiguous tensors at the moment!");
                        }
                        var outerSize = 1;
                        var innerSize = 1;
                        var dimSize   = output.Shape[dim];

                        for (var i = 0; i < dim; ++i)
                        {
                            outerSize *= output.Shape[i];
                        }

                        for (var i = dim + 1; i < output.Shape.Length; ++i)
                        {
                            innerSize *= output.Shape[i];
                        }

                        var dimStride   = innerSize;
                        var outerStride = dimSize * dimStride;

                        var gradInput = output.Copy(autograd: false);

                        var nCpu = SystemInfo.processorCount;
                        Parallel.For(0, nCpu, workerId =>
                        {
                            var max = (outerSize * innerSize) * (workerId + 1) / nCpu;
                            for (var i = (outerSize * innerSize) * workerId / nCpu; i < max; i++)
                            {
                                int outerIdx = i / innerSize;
                                int innerIdx = i % innerSize;

                                // works for contiguous!!
                                var index = outerIdx * outerStride + innerIdx;

                                float sum = 0;
                                for (var d = 0; d < dimSize; d++)
                                {
                                    sum += output.Data[index + d * dimStride] * gradOutput.Data[index + d * dimStride];
                                }

                                for (var d = 0; d < dimSize; d++)
                                {
                                    gradInput.Data[index + d * dimStride] =
                                        output.Data[index + d * dimStride] * (gradOutput.Data[index + d * dimStride] - sum);
                                }
                            }
                        });

                        gradInput.Autograd = false;

                        factory.Get(creators[0]).Backward(gradInput, this);
                    }
                    else if (creation_op.Contains("sum"))
                    {
                        // TOOD: sum backprop logic
                        FloatTensor parent = factory.Get(creators[0]);
                        parent.Grad = null;

                        int dim = int.Parse(creation_op.Split('_')[1]);

                        if (dim >= 0)
                        {
                            int[] view_shape = (int[])parent.shape.Clone();
                            view_shape[dim] = 1;
                            parent.Backward(grad.View(view_shape).Expand(parent.shape).Contiguous());
                        }
                        else
                        {
                            int[] view_shape = (int[])parent.shape.Clone();
                            for (int i = 0; i < parent.shape.Length; i++)
                            {
                                view_shape[i] = 1;
                            }
                            parent.Backward(grad.View(view_shape).Expand(parent.shape).Contiguous());
                        }
                    }
                    else if (creation_op == "transpose")
                    {
                        factory.Get(creators[0]).Backward(grad.Transpose());
                    }
                    else if (creation_op == "tanh")
                    {
                        FloatTensor c = this.Copy(autograd: false);

                        factory.Get(creators[0]).Backward(c.Pow(2).Neg().Add(1f).Mul(grad), this);
                    }
                    else if (creation_op.Contains("view_"))
                    {
                        FloatTensor parent = factory.Get(creators[0]);

                        parent.Grad = null;                     // prevents gradient from simply being added to the previous gradient
                        // instead the backpropagated gradient is set to a new value.

                        parent.Backward(this.Grad.View(parent.shape));
                    }
                    else
                    {
                        Debug.Log("Autograd couldn't find matching operation for:" + creation_op);
                    }
                }
            }
            else
            {
                Debug.Log("Autograd off - skipping backprop at tensor:" + id + " with creation_op:" + creation_op);
            }
        }