public FloatTensor Pow(FloatTensor x, bool inline = false, FloatTensor result = null)
{
if (!IsContiguous() || !x.IsContiguous())
{
throw new InvalidOperationException("All tensors must be contiguous, call Contiguous() to convert");
}
// Check if both tensors are compatible for sum
SameSizeDimensionsShapeAndLocation(ref x);
result = HookAutograd(ref result, ref x, "pow_elem", inline);
if (dataOnGpu)
{
result.Gpu(shader);
if (!inline)
{
return(PowElemGPU(x, result));
}
result.PowElemGPU_(x);
return(this);
}
result.Data = data.AsParallel().Zip(x.Data.AsParallel(), (a, b) => (float)Math.Pow((double)a, b))
.ToArray();
return(result);
}
public FloatTensor Sub(FloatTensor x, bool inline = false, FloatTensor result = null)
{
if (!IsContiguous() || !x.IsContiguous())
{
throw new InvalidOperationException("All tensors must be contiguous, call Contiguous() to convert");
}
// Check if both tensors are compatible for sum
SameSizeDimensionsShapeAndLocation(ref x);
result = HookAutograd(ref result, ref x, "sub_elem", inline);
if (dataOnGpu & x.dataOnGpu)
{
if (inline)
{
if (autograd)
{
throw new InvalidOperationException(
"Cannot call inline functions if you intend to run backprop.");
}
SubElemGPU_(x);
return(this);
}
result = SubElemGPU(x, result);
}
else
{
result.Data = data.AsParallel().Zip(x.Data.AsParallel(), (a, b) => a - b).ToArray();
}
return(result);
}
public FloatTensor AddMatrixVectorProduct(FloatTensor matrix, FloatTensor vector)
{
if (!IsContiguous() || !matrix.IsContiguous() || !vector.IsContiguous())
{
throw new InvalidOperationException("Tensor must be contiguous, call Contiguous() to convert");
}
var gpu = dataOnGpu & matrix.DataOnGpu & vector.DataOnGpu;
var cpu = !(dataOnGpu | matrix.DataOnGpu | vector.DataOnGpu);
var ref_shape = this.Shape;
var matrix_shape = matrix.Shape;
var vector_shape = vector.Shape;
if (ref_shape.Length != 1)
{
throw new InvalidOperationException(
"Cannot perform this operation on a tensor with more than one dimension");
}
if (ref_shape[0] != vector_shape[0])
{
throw new InvalidOperationException(String.Format(
"Cannot add matrix-vector product to tensor: {0} & {1}.", ref_shape[0], vector_shape[0]));
}
if (matrix_shape[1] != vector_shape[0])
{
throw new InvalidOperationException(String.Format("Last dimension of matrix doesn't match: {0} vs {1}.",
matrix_shape[1], vector_shape[0]));
}
if (gpu)
{
AddMatrixVectorProductGPU(matrix, vector);
}
else if (cpu)
{
var nCpu = SystemInfo.processorCount;
Parallel.For(0, nCpu, workerId =>
{
var max = size * (workerId + 1) / nCpu;
for (var idx = size * workerId / nCpu; idx < max; idx++)
{
for (var j = 0; j < ref_shape[0]; j++)
{
this[idx] += vector[j] * matrix[j + (idx * ref_shape[0])];
}
}
});
}
else
{
Debug.Log("Data for all Tensors needs to be colocated on the same device. - CPU != GPU");
}
return(this);
}
public FloatTensor AddMatrixMultiply(FloatTensor tensor1, FloatTensor tensor2)
{
if (!IsContiguous() || !tensor1.IsContiguous() || !tensor2.IsContiguous())
{
throw new InvalidOperationException("All tensors must be contiguous, call Contiguous() to convert");
}
bool gpu = dataOnGpu & tensor1.DataOnGpu & tensor2.DataOnGpu;
bool cpu = !(dataOnGpu | tensor1.DataOnGpu | tensor2.DataOnGpu);
int[] res_shape = this.Shape;
int[] shape1 = tensor1.Shape;
int[] shape2 = tensor2.Shape;
if (shape1[1] != shape2[0])
{
throw new InvalidOperationException(String.Format("Matrix multiply not possible: {0} & {1}.", shape1[1], shape2[0]));
}
if (res_shape[0] != shape1[0])
{
throw new InvalidOperationException(String.Format("First dimension doesn't match: {0} vs {1}.", res_shape[0], shape1[0]));
}
if (res_shape[1] != shape2[1])
{
throw new InvalidOperationException(String.Format("Last dimension doesn't match: {0} vs {1}.", res_shape[res_shape.Length - 1], shape2[shape2.Length - 1]));
}
if (gpu)
{
AddMatrixMultiplyGPU(tensor1, tensor2);
}
else if (cpu)
{
var nCpu = SystemInfo.processorCount;
Parallel.For(0, nCpu, workerId =>
{
var max = size * (workerId + 1) / nCpu;
for (var idx = size * workerId / nCpu; idx < max; idx++)
{
int col = idx % res_shape[1];
int row = (idx - col) / res_shape[1];
int row_offset = row * shape1[1];
for (var j = 0; j < shape1[1]; j++)
{
Data[idx] += tensor1.Data[j + row_offset] * tensor2.Data[j * shape2[1] + col];
}
}
});
return(this);
}
else
{
Debug.Log("Data for all Tensors needs to be colocated on the same device. - CPU != GPU");
}
return(this);
}
public FloatTensor Remainder(FloatTensor divisor, bool inline = false)
{
if (!IsContiguous() || !divisor.IsContiguous())
{
throw new InvalidOperationException("All tensor must be contiguous, call Contiguous() to convert");
}
SameSizeDimensionsShapeAndLocation(ref divisor);
if (inline & autograd)
{
throw new InvalidOperationException("Cannot call inline functions if you intend to run backprop.");
}
if (autograd)
{
throw new InvalidOperationException("Autograd not available for Remainder.");
}
var result = inline ? this : this.emptyTensorCopy();
if (dataOnGpu)
{
result.Gpu(shader);
if (inline)
{
RemainderElemGPU_(divisor);
return(this);
}
else
{
result = RemainderElemGPU(divisor, result);
}
}
else
{
var nCpu = SystemInfo.processorCount;
Parallel.For(0, nCpu, workerId => {
var max = size * (workerId + 1) / nCpu;
for (var i = size * workerId / nCpu; i < max; i++)
{
result[i] = this[i] % divisor[i];
}
;
});
}
return(result);
}
public FloatTensor Add(FloatTensor x, bool inline = false, FloatTensor result = null)
{
if (!IsContiguous() || !x.IsContiguous())
{
throw new InvalidOperationException("All tensors must be contiguous, call Contiguous() to convert");
}
// Check if both tensors are compatible for sum
SameSizeDimensionsShapeAndLocation(ref x);
result = HookAutograd(ref result, ref x, "add_elem", inline);
if (dataOnGpu)
{
if (inline)
{
if (autograd)
{
throw new InvalidOperationException("Cannot call inline functions if you intend to run backprop.");
}
AddElemGPU_(x);
return(this);
}
else
{
return(AddElemGPU(x, result));
}
}
var nCpu = SystemInfo.processorCount;
Parallel.For(0, nCpu, workerId => {
var max = size * (workerId + 1) / nCpu;
for (var i = size * workerId / nCpu; i < max; i++)
{
result.Data [i] = x.Data [i] + Data [i];
}
});
return(result);
}
public FloatTensor MM(FloatTensor x, FloatTensor result = null)
{
if (!IsContiguous() || !x.IsContiguous())
{
throw new InvalidOperationException("All tensors must be contiguous, call Contiguous() to convert");
}
if (this.shape.Length != 2 || x.shape.Length != 2)
{
throw new InvalidOperationException(
"Cannot do MM on tensors that aren't 2 dimentional. Try calling view() to reshape");
}
result = HookAutograd(ref result, ref x, "mm", false, new int[] { shape[0], x.shape[1] });
result.AddMatrixMultiply(this, x);
return(result);
}
public void Backward(FloatTensor grad = null, FloatTensor grad_origin = null)
{
//Debug.Log("Backward:" + this.id + " creation_op:" + creation_op);
if (autograd)
{
if (grad == null)
{
Debug.Log("Grad not Found... Creating Gradient of 1s");
grad = this.createOnesTensorLike();
grad.Autograd = false;
}
if (grad_origin != null)
{
int child_index = children_indices.IndexOf(grad_origin.Id);
if (children_counts[child_index] > 0)
{
throw new InvalidOperationException("Can't backprop more than once.");
}
else
{
children_counts[child_index] += 1;
}
}
if (this.Grad == null)
{
this.Grad = grad;
//Debug.Log("Setting Grad Tensor Id:" + this.id);
}
else
{
if (this.Grad.id == grad.id)
{
// do nothing
//Debug.Log("Not Updating For Tensor Id:" + this.id);
}
else
{
//Debug.Log("Updating For Tensor Id:" + this.id);
//this.Grad.Zero_();
this.Grad.Add(grad, inline: true);
}
}
// grads must not have grads of their own
if (this.Grad.autograd == true)
{
throw new InvalidOperationException("Sorry, grads cannot have grads");
}
// RULES FOR AUTOGRAD:
// 1) if you need to use "this" for calculating a gradient, copy it first and set autograd to false (see sigmoid)
// 2) if you use a method in your backprop logic that doesn't hook into the dynamic graph yet, backprop
// will not work!!! Make sure there's a "hookautograd" function in every method you use for backprop.
// 3) whenever backpropping into a method where the forward prop involved a scalar (such as scalar
// multiplication), current implementations assume you will NOT backprop into the scalar itself.
// 4) Because of rule (2), do NOT use "emptyTensorCopy" at all in backprop unless you know what you're
// doing.
// 5) I will be especially strict about Unit tests for all backprop logic as this is the most complex
// piece of functionality we have. Furthermore, most errors go completely undetected (not discovered
// by runtime errors). Autograd bugs just make convergence go slowly and sub-optimally.
// 6) If you use a forward propagation tensor to backprop, you MUST remember to turn off autograd
// when backpropagating (see "mm" below for example). Otherwise, it will cause autograd to break because
// whatever child you select will think it needs to wait for another gradient before backpropagating.
// 7) In the "view" backprop method, you'll notice that we set parent.grad = null. This keeps grads from
// accumulating when forward and backprop is called multiple times. However, it doesn't cause any new
// memory allocation.
// only continue backpropping if there's something to backprop into
// only continue backpropping if all gradients (from children) are accounted for
// override waiting for children if "backprop" was called on this variable directly
if (this.creators != null && this.creators.Count > 0 && (grad_origin == null || AllAutogradChildrenAccountedFor()))
{
if (creation_op == "abs")
{
FloatTensor c = this.Copy(autograd: false);
var parent = factory.Get(creators[0]);
parent.Backward(parent.Div(c).Mul(grad));
}
else if (creation_op == "add_elem")
{
factory.Get(creators[0]).Backward(grad, this);
factory.Get(creators[1]).Backward(grad, this);
}
else if (creation_op == "add_scalar")
{
factory.Get(creators[0]).Backward(grad, this);
}
else if (creation_op.Contains("concatenate_"))
{
int dim = int.Parse(creation_op.Split('_')[1]);
for (int i = 0; i < creators.Count; i++)
{
FloatTensor slice = grad.IndexSelect(factory.ctrl.intTensorFactory.Get(int_creators[i]), dim);
factory.Get(creators[i]).Backward(slice);
}
}
else if (creation_op == "contiguous")
{
//Debug.Log("Contiguous Backpropping Grad:" + grad.Id);
//Debug.Log("Contiguous Storing Grad:" + this.Grad.Id);
factory.Get(creators[0]).Backward(this.Grad.Copy(autograd: this.Grad.Autograd), this);
}
else if (creation_op == "copy")
{
factory.Get(creators[0]).Backward(grad, this);
}
else if (creation_op == "div_elem")
{
FloatTensor x = factory.Get(creators[0]);
FloatTensor y = factory.Get(creators[1]);
x.Backward(grad.Div(y));
FloatTensor y2 = y.Pow(2);
FloatTensor xn = x.Neg();
FloatTensor xny2 = xn.Div(y2);
FloatTensor gradxny2 = grad.Mul(xny2);
y.Backward(gradxny2);
}
else if (creation_op == "div_scalar")
{
factory.Get(creators[0]).Backward(grad.Div(factory.Get(creators[1]).data[0]), this);
}
else if (creation_op == "emptyTensorCopy_Hooked")
{
factory.Get(creators[0]).Backward(grad, this);
}
else if (creation_op == "expand")
{
var parent = factory.Get(creators[0]);
parent.Grad = null;
FloatTensor local_grad = grad.Copy(autograd: grad.Autograd);
var grad_shape = new int[shape.Length];
for (int i = 0; i < grad.shape.Length; i++)
{
grad_shape[i] = grad.shape[i];
}
for (int i = 0; i < shape.Length; i++)
{
grad_shape[i] = parent.shape[i];
if (parent.shape[i] == 1 && shape[i] > 1)
{
local_grad = local_grad.Sum(i).View(grad_shape);
}
}
parent.Backward(local_grad, this);
}
else if (creation_op.Contains("shaped_index_select"))
{
FloatTensor parent = factory.Get(creators[0]);
IntTensor indices = factory.ctrl.intTensorFactory.Get(int_creators[0]);
FloatTensor back_grad = parent.emptyTensorCopy(hook_graph: true);
back_grad.autograd = false;
back_grad.Zero_();
FloatTensor out_grad = back_grad.IndexAdd(indices, -1, grad);
parent.Backward(out_grad);
}
else if (creation_op.Contains("index_select"))
{
FloatTensor parent = factory.Get(creators[0]);
IntTensor indices = factory.ctrl.intTensorFactory.Get(int_creators[0]);
int dim = int.Parse(creation_op.Split('_')[2]);
FloatTensor back_grad = parent.emptyTensorCopy(hook_graph: true);
back_grad.autograd = false;
FloatTensor out_grad = back_grad.IndexAdd(indices, dim, grad);
parent.Backward(out_grad);
}
else if (creation_op == "log")
{
// TOOD: sum backprop logic
FloatTensor x = factory.Get(creators[0]).Copy(autograd: false);
factory.Get(creators[0]).Backward(grad.Mul(x.Pow(-1)), this);
}
else if (creation_op == "mul_elem")
{
factory.Get(creators[0]).Backward(grad.Mul(factory.Get(creators[1])), this);
factory.Get(creators[1]).Backward(grad.Mul(factory.Get(creators[0])), this);
}
else if (creation_op == "mul_scalar")
{
factory.Get(creators[0]).Backward(grad.Mul(factory.Get(creators[1]).data[0]), this);
}
else if (creation_op == "mm")
{
FloatTensor x = factory.Get(creators[1]).Transpose();
x.autograd = false;
FloatTensor y = factory.Get(creators[0]).Transpose();
y.autograd = false;
factory.Get(creators[0]).Backward(grad.MM(x), this);
factory.Get(creators[1]).Backward(y.MM(grad), this);
}
else if (creation_op == "neg")
{
factory.Get(creators[0]).Backward(grad.Neg(), this);
}
else if (creation_op == "pow_scalar")
{
FloatTensor x = factory.Get(creators[0]).Copy(autograd: false);
factory.Get(creators[0]).Backward(x.Mul(grad).Mul(factory.Get(creators[1]).Data[0]), this);
}
else if (creation_op == "relu")
{
// TOOD: replace with simple comparison and mulitplication (should be 2 liner)
FloatTensor c = this.Copy(autograd: false);
FloatTensor output = c;
var dimSize = 1;
for (var i = 0; i < output.Shape.Length; ++i)
{
dimSize *= output.Shape[i];
}
var gradInput = output.Copy(autograd: false);
gradInput.Zero_();
var nCpu = SystemInfo.processorCount;
Parallel.For(0, nCpu, workerId =>
{
var max = dimSize * (workerId + 1) / nCpu;
for (var i = dimSize * workerId / nCpu; i < max; i++)
{
if (output.Data[i] > 0)
{
gradInput.Data[i] = 1;
}
else
{
gradInput.Data[i] = 0;
}
}
});
factory.Get(creators[0]).Backward((gradInput).Mul(grad), this);
}
else if (creation_op == "sub_elem")
{
factory.Get(creators[0]).Backward(grad, this);
factory.Get(creators[1]).Backward(grad.Neg(), this);
}
else if (creation_op == "sub_scalar")
{
factory.Get(creators[0]).Backward(grad, this);
}
else if (creation_op == "sigmoid")
{
FloatTensor self_nograd = this.Copy(autograd: false);
factory.Get(creators[0]).Backward(self_nograd.Neg().Add(1f).Mul(self_nograd).Mul(grad), this);
}
else if (creation_op.Contains("softmax-"))
{
FloatTensor c = this.Copy(autograd: false);
var dim = int.Parse(creation_op.Split('-')[1]);
FloatTensor output = this;
FloatTensor gradOutput = grad;
if (!output.IsContiguous() || !gradOutput.IsContiguous())
{
throw new NotImplementedException(
"Softmax Gradient does not support non-contiguous tensors at the moment!");
}
var outerSize = 1;
var innerSize = 1;
var dimSize = output.Shape[dim];
for (var i = 0; i < dim; ++i)
{
outerSize *= output.Shape[i];
}
for (var i = dim + 1; i < output.Shape.Length; ++i)
{
innerSize *= output.Shape[i];
}
var dimStride = innerSize;
var outerStride = dimSize * dimStride;
var gradInput = output.Copy(autograd: false);
var nCpu = SystemInfo.processorCount;
Parallel.For(0, nCpu, workerId =>
{
var max = (outerSize * innerSize) * (workerId + 1) / nCpu;
for (var i = (outerSize * innerSize) * workerId / nCpu; i < max; i++)
{
int outerIdx = i / innerSize;
int innerIdx = i % innerSize;
// works for contiguous!!
var index = outerIdx * outerStride + innerIdx;
float sum = 0;
for (var d = 0; d < dimSize; d++)
{
sum += output.Data[index + d * dimStride] * gradOutput.Data[index + d * dimStride];
}
for (var d = 0; d < dimSize; d++)
{
gradInput.Data[index + d * dimStride] =
output.Data[index + d * dimStride] * (gradOutput.Data[index + d * dimStride] - sum);
}
}
});
gradInput.Autograd = false;
factory.Get(creators[0]).Backward(gradInput, this);
}
else if (creation_op.Contains("sum"))
{
// TOOD: sum backprop logic
FloatTensor parent = factory.Get(creators[0]);
parent.Grad = null;
int dim = int.Parse(creation_op.Split('_')[1]);
if (dim >= 0)
{
int[] view_shape = (int[])parent.shape.Clone();
view_shape[dim] = 1;
parent.Backward(grad.View(view_shape).Expand(parent.shape).Contiguous());
}
else
{
int[] view_shape = (int[])parent.shape.Clone();
for (int i = 0; i < parent.shape.Length; i++)
{
view_shape[i] = 1;
}
parent.Backward(grad.View(view_shape).Expand(parent.shape).Contiguous());
}
}
else if (creation_op == "transpose")
{
factory.Get(creators[0]).Backward(grad.Transpose());
}
else if (creation_op == "tanh")
{
FloatTensor c = this.Copy(autograd: false);
factory.Get(creators[0]).Backward(c.Pow(2).Neg().Add(1f).Mul(grad), this);
}
else if (creation_op.Contains("view_"))
{
FloatTensor parent = factory.Get(creators[0]);
parent.Grad = null; // prevents gradient from simply being added to the previous gradient
// instead the backpropagated gradient is set to a new value.
parent.Backward(this.Grad.View(parent.shape));
}
else
{
Debug.Log("Autograd couldn't find matching operation for:" + creation_op);
}
}
}
else
{
Debug.Log("Autograd off - skipping backprop at tensor:" + id + " with creation_op:" + creation_op);
}
}