public void Backward(FloatTensor grad = null, FloatTensor grad_origin = null)
        {
            //Debug.Log("Backward:" + this.id + " creation_op:" + creation_op);

            if (autograd)
            {
                if (grad == null)
                {
                    Debug.Log("Grad not Found... Creating Gradient of 1s");
                    grad          = this.createOnesTensorLike();
                    grad.Autograd = false;
                }

                if (grad_origin != null)
                {
                    int child_index = children_indices.IndexOf(grad_origin.Id);
                    if (children_counts[child_index] > 0)
                    {
                        throw new InvalidOperationException("Can't backprop more than once.");
                    }
                    else
                    {
                        children_counts[child_index] += 1;
                    }
                }

                if (this.Grad == null)
                {
                    this.Grad = grad;
                    //Debug.Log("Setting Grad Tensor Id:" + this.id);
                }
                else
                {
                    if (this.Grad.id == grad.id)
                    {
                        // do nothing
                        //Debug.Log("Not Updating For Tensor Id:" + this.id);
                    }
                    else
                    {
                        //Debug.Log("Updating For Tensor Id:" + this.id);
                        //this.Grad.Zero_();
                        this.Grad.Add(grad, inline: true);
                    }
                }

                // grads must not have grads of their own
                if (this.Grad.autograd == true)
                {
                    throw new InvalidOperationException("Sorry, grads cannot have grads");
                }

                // RULES FOR AUTOGRAD:
                // 1) if you need to use "this" for calculating a gradient, copy it first and set autograd to false (see sigmoid)
                // 2) if you use a method in your backprop logic that doesn't hook into the dynamic graph yet, backprop
                // will not work!!! Make sure there's a "hookautograd" function in every method you use for backprop.
                // 3) whenever backpropping into a method where the forward prop involved a scalar (such as scalar
                // multiplication), current implementations assume you will NOT backprop into the scalar itself.
                // 4) Because of rule (2), do NOT use "emptyTensorCopy" at all in backprop unless you know what you're
                // doing.
                // 5) I will be especially strict about Unit tests for all backprop logic as this is the most complex
                // piece of functionality we have. Furthermore, most errors go completely undetected (not discovered
                // by runtime errors). Autograd bugs just make convergence go slowly and sub-optimally.
                // 6) If you use a forward propagation tensor to backprop, you MUST remember to turn off autograd
                // when backpropagating (see "mm" below for example). Otherwise, it will cause autograd to break because
                // whatever child you select will think it needs to wait for another gradient before backpropagating.

                // only continue backpropping if there's something to backprop into
                // only continue backpropping if all gradients (from children) are accounted for
                // override waiting for children if "backprop" was called on this variable directly
                if (this.creators != null && this.creators.Count > 0 && (grad_origin == null || AllAutogradChildrenAccountedFor()))
                {
                    if (creation_op == "add_elem")
                    {
                        factory.Get(creators[0]).Backward(grad, this);
                        factory.Get(creators[1]).Backward(grad, this);
                    }
                    else if (creation_op == "add_scalar")
                    {
                        factory.Get(creators[0]).Backward(grad, this);
                    }
                    else if (creation_op == "copy")
                    {
                        factory.Get(creators[0]).Backward(grad, this);
                    }
                    else if (creation_op == "div_elem")
                    {
                        FloatTensor x = factory.Get(creators[0]);
                        FloatTensor y = factory.Get(creators[1]);

                        x.Backward(grad.Div(y));

                        FloatTensor y2       = y.Pow(2);
                        FloatTensor xn       = x.Neg();
                        FloatTensor xny2     = xn.Div(y2);
                        FloatTensor gradxny2 = grad.Mul(xny2);
                        y.Backward(gradxny2);
                    }
                    else if (creation_op == "div_scalar")
                    {
                        factory.Get(creators[0]).Backward(grad.Div(factory.Get(creators[1]).data[0]), this);
                    }
                    else if (creation_op == "mul_elem")
                    {
                        factory.Get(creators[0]).Backward(grad.Mul(factory.Get(creators[1])), this);
                        factory.Get(creators[1]).Backward(grad.Mul(factory.Get(creators[0])), this);
                    }
                    else if (creation_op == "mul_scalar")
                    {
                        factory.Get(creators[0]).Backward(grad.Mul(factory.Get(creators[1]).data[0]), this);
                    }
                    else if (creation_op == "mm")
                    {
                        FloatTensor x = factory.Get(creators[1]).Transpose();
                        x.autograd = false;

                        FloatTensor y = factory.Get(creators[0]).Transpose();
                        y.autograd = false;

                        factory.Get(creators[0]).Backward(grad.MM(x), this);
                        factory.Get(creators[1]).Backward(y.MM(grad), this);
                    }
                    else if (creation_op == "neg")
                    {
                        factory.Get(creators[0]).Backward(grad.Neg(), this);
                    }
                    else if (creation_op == "pow_scalar")
                    {
                        FloatTensor x = factory.Get(creators[0]).Copy();
                        x.autograd = false;

                        factory.Get(creators[0]).Backward(x.Mul(grad).Mul(factory.Get(creators[1]).Data[0]), this);
                    }
                    else if (creation_op == "sub_elem")
                    {
                        factory.Get(creators[0]).Backward(grad, this);
                        factory.Get(creators[1]).Backward(grad.Neg(), this);
                    }
                    else if (creation_op == "sub_scalar")
                    {
                        factory.Get(creators[0]).Backward(grad, this);
                    }
                    else if (creation_op == "sigmoid")
                    {
                        FloatTensor self_nograd = this.Copy();
                        self_nograd.autograd = false;

                        factory.Get(creators[0]).Backward(self_nograd.Neg().Add((float)1).Mul(self_nograd).Mul(grad), this);
                    }
                    else if (creation_op == "transpose")
                    {
                        factory.Get(creators[0]).Backward(grad.Transpose());
                    }
                    else if (creation_op == "tanh")
                    {
                        FloatTensor c = this.Copy();
                        c.autograd = false;

                        factory.Get(creators[0]).Backward(c.Pow(2).Neg().Add(1f).Mul(grad), this);
                    }
                    else if (creation_op.Contains("softmax-"))
                    {
                        FloatTensor c = this.Copy();
                        c.autograd = false;
                        var dim = int.Parse(creation_op.Split('-')[1]);
                        factory.Get(creators[0]).Backward(Functional.SoftmaxGradient(this, grad, dim), this);
                    }
                    else
                    {
                        Debug.Log("Autograd couldn't find matching operation for:" + creation_op);
                    }
                }
            }
            else
            {
                Debug.Log("Autograd off - skipping backprop...");
            }
        }
Exemple #2
0
        public void Backward(FloatTensor grad = null, FloatTensor grad_origin = null)
        {
            //Debug.Log("Backward:" + this.id + " creation_op:" + creation_op);

            if (autograd)
            {
                if (grad == null)
                {
                    Debug.Log("Grad not Found... Creating Gradient of 1s");
                    grad          = this.createOnesTensorLike();
                    grad.Autograd = false;
                }

                if (grad_origin != null)
                {
                    int child_index = children_indices.IndexOf(grad_origin.Id);
                    if (children_counts[child_index] > 0)
                    {
                        throw new InvalidOperationException("Can't backprop more than once.");
                    }
                    else
                    {
                        children_counts[child_index] += 1;
                    }
                }

                if (this.Grad == null)
                {
                    this.Grad = grad;
                    //Debug.Log("Setting Grad Tensor Id:" + this.id);
                }
                else
                {
                    if (this.Grad.id == grad.id)
                    {
                        // do nothing
                        //Debug.Log("Not Updating For Tensor Id:" + this.id);
                    }
                    else
                    {
                        //Debug.Log("Updating For Tensor Id:" + this.id);
                        //this.Grad.Zero_();
                        this.Grad.Add(grad, inline: true);
                    }
                }

                // grads must not have grads of their own
                if (this.Grad.autograd == true)
                {
                    throw new InvalidOperationException("Sorry, grads cannot have grads");
                }

                // RULES FOR AUTOGRAD:
                // 1) if you need to use "this" for calculating a gradient, copy it first and set autograd to false (see sigmoid)
                // 2) if you use a method in your backprop logic that doesn't hook into the dynamic graph yet, backprop
                // will not work!!! Make sure there's a "hookautograd" function in every method you use for backprop.
                // 3) whenever backpropping into a method where the forward prop involved a scalar (such as scalar
                // multiplication), current implementations assume you will NOT backprop into the scalar itself.
                // 4) Because of rule (2), do NOT use "emptyTensorCopy" at all in backprop unless you know what you're
                // doing.
                // 5) I will be especially strict about Unit tests for all backprop logic as this is the most complex
                // piece of functionality we have. Furthermore, most errors go completely undetected (not discovered
                // by runtime errors). Autograd bugs just make convergence go slowly and sub-optimally.
                // 6) If you use a forward propagation tensor to backprop, you MUST remember to turn off autograd
                // when backpropagating (see "mm" below for example). Otherwise, it will cause autograd to break because
                // whatever child you select will think it needs to wait for another gradient before backpropagating.
                // 7) In the "view" backprop method, you'll notice that we set parent.grad = null. This keeps grads from
                // accumulating when forward and backprop is called multiple times. However, it doesn't cause any new
                // memory allocation.

                // only continue backpropping if there's something to backprop into
                // only continue backpropping if all gradients (from children) are accounted for
                // override waiting for children if "backprop" was called on this variable directly
                if (this.creators != null && this.creators.Count > 0 && (grad_origin == null || AllAutogradChildrenAccountedFor()))
                {
                    if (creation_op == "abs")
                    {
                        FloatTensor c = this.Copy(autograd: false);

                        var parent = factory.Get(creators[0]);

                        parent.Backward(parent.Div(c).Mul(grad));
                    }
                    else if (creation_op == "add_elem")
                    {
                        factory.Get(creators[0]).Backward(grad, this);
                        factory.Get(creators[1]).Backward(grad, this);
                    }
                    else if (creation_op == "add_scalar")
                    {
                        factory.Get(creators[0]).Backward(grad, this);
                    }
                    else if (creation_op.Contains("concatenate_"))
                    {
                        int dim = int.Parse(creation_op.Split('_')[1]);

                        for (int i = 0; i < creators.Count; i++)
                        {
                            FloatTensor slice = grad.IndexSelect(factory.ctrl.intTensorFactory.Get(int_creators[i]), dim);

                            factory.Get(creators[i]).Backward(slice);
                        }
                    }
                    else if (creation_op == "contiguous")
                    {
                        //Debug.Log("Contiguous Backpropping Grad:" + grad.Id);
                        //Debug.Log("Contiguous Storing Grad:" + this.Grad.Id);
                        factory.Get(creators[0]).Backward(this.Grad.Copy(autograd: this.Grad.Autograd), this);
                    }
                    else if (creation_op == "copy")
                    {
                        factory.Get(creators[0]).Backward(grad, this);
                    }
                    else if (creation_op == "div_elem")
                    {
                        FloatTensor x = factory.Get(creators[0]);
                        FloatTensor y = factory.Get(creators[1]);

                        x.Backward(grad.Div(y));

                        FloatTensor y2       = y.Pow(2);
                        FloatTensor xn       = x.Neg();
                        FloatTensor xny2     = xn.Div(y2);
                        FloatTensor gradxny2 = grad.Mul(xny2);
                        y.Backward(gradxny2);
                    }
                    else if (creation_op == "div_scalar")
                    {
                        factory.Get(creators[0]).Backward(grad.Div(factory.Get(creators[1]).data[0]), this);
                    }
                    else if (creation_op == "emptyTensorCopy_Hooked")
                    {
                        factory.Get(creators[0]).Backward(grad, this);
                    }
                    else if (creation_op == "expand")
                    {
                        var parent = factory.Get(creators[0]);
                        parent.Grad = null;

                        FloatTensor local_grad = grad.Copy(autograd: grad.Autograd);

                        var grad_shape = new int[shape.Length];

                        for (int i = 0; i < grad.shape.Length; i++)
                        {
                            grad_shape[i] = grad.shape[i];
                        }

                        for (int i = 0; i < shape.Length; i++)
                        {
                            grad_shape[i] = parent.shape[i];
                            if (parent.shape[i] == 1 && shape[i] > 1)
                            {
                                local_grad = local_grad.Sum(i).View(grad_shape);
                            }
                        }

                        parent.Backward(local_grad, this);
                    }
                    else if (creation_op.Contains("shaped_index_select"))
                    {
                        FloatTensor parent    = factory.Get(creators[0]);
                        IntTensor   indices   = factory.ctrl.intTensorFactory.Get(int_creators[0]);
                        FloatTensor back_grad = parent.emptyTensorCopy(hook_graph: true);
                        back_grad.autograd = false;
                        back_grad.Zero_();

                        FloatTensor out_grad = back_grad.IndexAdd(indices, -1, grad);
                        parent.Backward(out_grad);
                    }
                    else if (creation_op.Contains("index_select"))
                    {
                        FloatTensor parent = factory.Get(creators[0]);

                        IntTensor indices = factory.ctrl.intTensorFactory.Get(int_creators[0]);

                        int dim = int.Parse(creation_op.Split('_')[2]);

                        FloatTensor back_grad = parent.emptyTensorCopy(hook_graph: true);
                        back_grad.autograd = false;

                        FloatTensor out_grad = back_grad.IndexAdd(indices, dim, grad);
                        parent.Backward(out_grad);
                    }
                    else if (creation_op == "log")
                    {
                        // TOOD: sum backprop logic
                        FloatTensor x = factory.Get(creators[0]).Copy(autograd: false);

                        factory.Get(creators[0]).Backward(grad.Mul(x.Pow(-1)), this);
                    }
                    else if (creation_op == "mul_elem")
                    {
                        factory.Get(creators[0]).Backward(grad.Mul(factory.Get(creators[1])), this);
                        factory.Get(creators[1]).Backward(grad.Mul(factory.Get(creators[0])), this);
                    }
                    else if (creation_op == "mul_scalar")
                    {
                        factory.Get(creators[0]).Backward(grad.Mul(factory.Get(creators[1]).data[0]), this);
                    }
                    else if (creation_op == "mm")
                    {
                        FloatTensor x = factory.Get(creators[1]).Transpose();
                        x.autograd = false;

                        FloatTensor y = factory.Get(creators[0]).Transpose();
                        y.autograd = false;

                        factory.Get(creators[0]).Backward(grad.MM(x), this);
                        factory.Get(creators[1]).Backward(y.MM(grad), this);
                    }
                    else if (creation_op == "neg")
                    {
                        factory.Get(creators[0]).Backward(grad.Neg(), this);
                    }
                    else if (creation_op == "pow_scalar")
                    {
                        FloatTensor x = factory.Get(creators[0]).Copy(autograd: false);

                        factory.Get(creators[0]).Backward(x.Mul(grad).Mul(factory.Get(creators[1]).Data[0]), this);
                    }
                    else if (creation_op == "relu")
                    {
                        // TOOD: replace with simple comparison and mulitplication (should be 2 liner)
                        FloatTensor c = this.Copy(autograd: false);

                        FloatTensor output = c;

                        var dimSize = 1;

                        for (var i = 0; i < output.Shape.Length; ++i)
                        {
                            dimSize *= output.Shape[i];
                        }

                        var gradInput = output.Copy(autograd: false);
                        gradInput.Zero_();

                        var nCpu = SystemInfo.processorCount;
                        Parallel.For(0, nCpu, workerId =>
                        {
                            var max = dimSize * (workerId + 1) / nCpu;
                            for (var i = dimSize * workerId / nCpu; i < max; i++)
                            {
                                if (output.Data[i] > 0)
                                {
                                    gradInput.Data[i] = 1;
                                }
                                else
                                {
                                    gradInput.Data[i] = 0;
                                }
                            }
                        });

                        factory.Get(creators[0]).Backward((gradInput).Mul(grad), this);
                    }
                    else if (creation_op == "sub_elem")
                    {
                        factory.Get(creators[0]).Backward(grad, this);
                        factory.Get(creators[1]).Backward(grad.Neg(), this);
                    }
                    else if (creation_op == "sub_scalar")
                    {
                        factory.Get(creators[0]).Backward(grad, this);
                    }
                    else if (creation_op == "sigmoid")
                    {
                        FloatTensor self_nograd = this.Copy(autograd: false);

                        factory.Get(creators[0]).Backward(self_nograd.Neg().Add(1f).Mul(self_nograd).Mul(grad), this);
                    }
                    else if (creation_op.Contains("softmax-"))
                    {
                        FloatTensor c   = this.Copy(autograd: false);
                        var         dim = int.Parse(creation_op.Split('-')[1]);

                        FloatTensor output     = this;
                        FloatTensor gradOutput = grad;

                        if (!output.IsContiguous() || !gradOutput.IsContiguous())
                        {
                            throw new NotImplementedException(
                                      "Softmax Gradient does not support non-contiguous tensors at the moment!");
                        }
                        var outerSize = 1;
                        var innerSize = 1;
                        var dimSize   = output.Shape[dim];

                        for (var i = 0; i < dim; ++i)
                        {
                            outerSize *= output.Shape[i];
                        }

                        for (var i = dim + 1; i < output.Shape.Length; ++i)
                        {
                            innerSize *= output.Shape[i];
                        }

                        var dimStride   = innerSize;
                        var outerStride = dimSize * dimStride;

                        var gradInput = output.Copy(autograd: false);

                        var nCpu = SystemInfo.processorCount;
                        Parallel.For(0, nCpu, workerId =>
                        {
                            var max = (outerSize * innerSize) * (workerId + 1) / nCpu;
                            for (var i = (outerSize * innerSize) * workerId / nCpu; i < max; i++)
                            {
                                int outerIdx = i / innerSize;
                                int innerIdx = i % innerSize;

                                // works for contiguous!!
                                var index = outerIdx * outerStride + innerIdx;

                                float sum = 0;
                                for (var d = 0; d < dimSize; d++)
                                {
                                    sum += output.Data[index + d * dimStride] * gradOutput.Data[index + d * dimStride];
                                }

                                for (var d = 0; d < dimSize; d++)
                                {
                                    gradInput.Data[index + d * dimStride] =
                                        output.Data[index + d * dimStride] * (gradOutput.Data[index + d * dimStride] - sum);
                                }
                            }
                        });

                        gradInput.Autograd = false;

                        factory.Get(creators[0]).Backward(gradInput, this);
                    }
                    else if (creation_op.Contains("sum"))
                    {
                        // TOOD: sum backprop logic
                        FloatTensor parent = factory.Get(creators[0]);
                        parent.Grad = null;

                        int dim = int.Parse(creation_op.Split('_')[1]);

                        if (dim >= 0)
                        {
                            int[] view_shape = (int[])parent.shape.Clone();
                            view_shape[dim] = 1;
                            parent.Backward(grad.View(view_shape).Expand(parent.shape).Contiguous());
                        }
                        else
                        {
                            int[] view_shape = (int[])parent.shape.Clone();
                            for (int i = 0; i < parent.shape.Length; i++)
                            {
                                view_shape[i] = 1;
                            }
                            parent.Backward(grad.View(view_shape).Expand(parent.shape).Contiguous());
                        }
                    }
                    else if (creation_op == "transpose")
                    {
                        factory.Get(creators[0]).Backward(grad.Transpose());
                    }
                    else if (creation_op == "tanh")
                    {
                        FloatTensor c = this.Copy(autograd: false);

                        factory.Get(creators[0]).Backward(c.Pow(2).Neg().Add(1f).Mul(grad), this);
                    }
                    else if (creation_op.Contains("view_"))
                    {
                        FloatTensor parent = factory.Get(creators[0]);

                        parent.Grad = null;                     // prevents gradient from simply being added to the previous gradient
                        // instead the backpropagated gradient is set to a new value.

                        parent.Backward(this.Grad.View(parent.shape));
                    }
                    else
                    {
                        Debug.Log("Autograd couldn't find matching operation for:" + creation_op);
                    }
                }
            }
            else
            {
                Debug.Log("Autograd off - skipping backprop at tensor:" + id + " with creation_op:" + creation_op);
            }
        }