public FloatTensor Abs(bool inline = false)
// Returns a new Tensor with the smallest integer greater than or equal to each element
{
FloatTensor result = inline ? this : this.Copy();
if (dataOnGpu)
{
result.Gpu(this.shader);
if (inline)
{
AbsGPU_(); return(this);
}
else
{
return(AbsGPU(result));
}
}
else
{
var nCpu = SystemInfo.processorCount;
Parallel.For(0, nCpu, workerId => {
var max = size * (workerId + 1) / nCpu;
for (var i = size * workerId / nCpu; i < max; i++)
{
result.Data [i] = (float)(Math.Abs(Data [i]));
}
});
}
return(result);
}
public FloatTensor Pow(FloatTensor x, bool inline = false, FloatTensor result = null)
{
if (!IsContiguous() || !x.IsContiguous())
{
throw new InvalidOperationException("All tensors must be contiguous, call Contiguous() to convert");
}
// Check if both tensors are compatible for sum
SameSizeDimensionsShapeAndLocation(ref x);
result = HookAutograd(ref result, ref x, "pow_elem", inline);
if (dataOnGpu)
{
result.Gpu(shader);
if (!inline)
{
return(PowElemGPU(x, result));
}
result.PowElemGPU_(x);
return(this);
}
result.Data = data.AsParallel().Zip(x.Data.AsParallel(), (a, b) => (float)Math.Pow((double)a, b))
.ToArray();
return(result);
}
public FloatTensor MM(FloatTensor x)
{
if (this.shape.Length != 2 || x.shape.Length != 2)
{
throw new InvalidOperationException(
"Cannot do MM on tensors that aren't 2 dimentional. Try calling view() to reshape");
}
var resultShape = new int[2];
resultShape[0] = shape[0];
resultShape[1] = x.shape[1];
var result = new FloatTensor(_controller: controller, _shape: resultShape);
if (this.dataOnGpu)
{
result.Gpu(shader);
}
result.AddMatrixMultiply(this, x);
if (autograd)
{
HookAutograd(ref result, ref x, "mm");
}
return(result);
}
public FloatTensor Div(FloatTensor x, bool inline = false, FloatTensor result = null)
{
if (!IsContiguous() || !x.IsContiguous())
{
throw new InvalidOperationException("Tensor must be contiguous, call Contiguous() to convert");
}
// Check if both tensors are compatible for sum
SameSizeDimensionsShapeAndLocation(ref x);
result = HookAutograd(ref result, ref x, "div_elem", inline);
if (dataOnGpu & x.dataOnGpu)
{
result.Gpu(shader);
if (inline)
{
if (autograd)
{
throw new InvalidOperationException(
"Cannot call inline functions if you intend to run backprop.");
}
DivElemGPU_(x);
return(this);
}
result = DivElemGPU(x, result);
}
else
{
result.Data = data.AsParallel().Zip(x.Data.AsParallel(), (a, b) => a / b).ToArray();
}
return(result);
}
public FloatTensor Sub(FloatTensor x)
{
// Check if both tensors are compatible for sum
SameSizeDimensionsShapeAndLocation(ref x);
FloatTensor result = new FloatTensor(shape, this.shader);
if (dataOnGpu & x.dataOnGpu)
{
result.Gpu();
SubElemGPU(x, result);
}
else
{
var nCpu = SystemInfo.processorCount;
Parallel.For(0, nCpu, workerId => {
var max = size * (workerId + 1) / nCpu;
for (var i = size * workerId / nCpu; i < max; i++)
{
result.Data [i] = Data [i] - x.Data [i];
}
});
}
return(result);
}
public FloatTensor Add(float value, bool inline = false, FloatTensor result = null)
{
result = HookAutograd(ref result, value, "add_scalar", inline);
if (dataOnGpu)
{
result.Gpu(shader);
if (inline)
{
AddScalarGPU_(value); return(this);
}
else
{
return(AddScalarGPU(value, result));
}
}
else
{
var nCpu = SystemInfo.processorCount;
Parallel.For(0, nCpu, workerId => {
var max = size * (workerId + 1) / nCpu;
for (var i = size * workerId / nCpu; i < max; i++)
{
result.Data [i] = value + Data [i];
}
});
}
return(result);
}
public FloatTensor Pow(FloatTensor x, bool inline = true)
{
// Check if both tensors are compatible for sum
SameSizeDimensionsShapeAndLocation(ref x);
FloatTensor result = inline ? this : this.emptyTensorCopy();
if (dataOnGpu)
{
result.Gpu(this.shader);
if (inline)
{
result.PowElemGPU_(x); return(this);
}
else
{
return(PowElemGPU(x, result));
}
}
else
{
var nCpu = SystemInfo.processorCount;
Parallel.For(0, nCpu, workerId => {
var max = size * (workerId + 1) / nCpu;
for (var i = size * workerId / nCpu; i < max; i++)
{
result.Data [i] = (float)Math.Pow((double)Data [i], x.Data [i]);
}
});
}
return(result);
}
public FloatTensor Floor(bool inline = false)
{
FloatTensor result = inline ? this : this.emptyTensorCopy();
if (dataOnGpu)
{
result.Gpu(this.shader);
if (inline)
{
FloorGPU_(); return(this);
}
else
{
return(FloorGPU(result));
}
}
var nCpu = SystemInfo.processorCount;
Parallel.For(0, nCpu, workerId =>
{
var max = size * (workerId + 1) / nCpu;
for (var i = size * workerId / nCpu; i < max; i++)
{
result.Data[i] = (float)(Math.Floor(this.Data[i]));
}
});
return(result);
}
public FloatTensor Ceil(bool inline = false)
// Returns a new Tensor with the smallest integer greater than or equal to each element
{
FloatTensor result = inline ? this : this.emptyTensorCopy();
if (dataOnGpu)
{
//TODO: Fix GPU operations. https://github.com/OpenMined/OpenMined/issues/126
result.Gpu(this.shader);
if (inline)
{
CeilGPU_(); return(this);
}
else
{
return(CeilGPU(result));
}
}
var nCpu = SystemInfo.processorCount;
Parallel.For(0, nCpu, workerId =>
{
var max = size * (workerId + 1) / nCpu;
for (var i = size * workerId / nCpu; i < max; i++)
{
result.Data[i] = (float)(Math.Ceiling(Data[i]));
}
});
return(result);
}
public FloatTensor Add(float value, bool inline = false)
{
FloatTensor result = inline ? this : this.emptyTensorCopy();
if (dataOnGpu)
{
result.Gpu(shader);
if (inline)
{
AddScalarGPU_(value); return(this);
}
else
{
return(AddScalarGPU(value, result));
}
}
else
{
var nCpu = SystemInfo.processorCount;
Parallel.For(0, nCpu, workerId => {
var max = size * (workerId + 1) / nCpu;
for (var i = size * workerId / nCpu; i < max; i++)
{
result.Data [i] = value + Data [i];
}
});
}
return(result);
}
public FloatTensor Acos(bool inline = false)
{
FloatTensor result = factory.ctrl.floatTensorFactory.Create(this.shape);
if (dataOnGpu)
{
result.Gpu(shader);
return(AcosGPU(result));
}
result.Data = data.AsParallel().Select(x => (float)Math.Acos((double)x)).ToArray();
return(result);
}
public FloatTensor Div(float value, bool inline = false, FloatTensor result = null)
{
result = HookAutograd(ref result, value, "div_scalar", inline);
if (dataOnGpu)
{
result.Gpu(shader);
if (!inline)
{
return(DivScalarGPU(value, result));
}
DivScalarGPU_(value);
return(this);
}
result.Data = data.AsParallel().Select(x => x / value).ToArray();
return(result);
}
public FloatTensor Neg(bool inline = false, FloatTensor result = null)
{
result = HookAutograd(ref result, "neg", inline);
if (dataOnGpu)
{
result.Gpu(shader);
if (!inline)
{
return(NegateGPU());
}
NegateGPU_();
return(this);
}
result.Data = data.AsParallel().Select(x => - x).ToArray();
return(result);
}
public FloatTensor Div(FloatTensor x, bool inline = false)
{
// Check if both tensors are compatible for sum
SameSizeDimensionsShapeAndLocation(ref x);
FloatTensor result = inline ? this : this.emptyTensorCopy();
if (dataOnGpu & x.dataOnGpu)
{
result.Gpu(shader);
if (inline)
{
if (autograd)
{
throw new InvalidOperationException("Cannot call inline functions if you intend to run backprop.");
}
DivElemGPU_(x);
return(this);
}
else
{
result = DivElemGPU(x, result);
}
}
else
{
var nCpu = SystemInfo.processorCount;
Parallel.For(0, nCpu, workerId => {
var max = size * (workerId + 1) / nCpu;
for (var i = size * workerId / nCpu; i < max; i++)
{
result.Data [i] = Data [i] / x.Data [i];
}
});
}
if (autograd)
{
HookAutograd(ref result, ref x, "div_elem");
}
return(result);
}
public FloatTensor Sin(bool inline = false)
{
FloatTensor result = factory.ctrl.floatTensorFactory.Create(shape);
if (dataOnGpu)
{
result.Gpu(shader);
if (inline)
{
throw new NotImplementedException();
}
else
{
return(SinGPU(result));
}
}
result.Data = data.AsParallel().Select(x => (float)Math.Sin((double)x)).ToArray();
return(result);
}
public FloatTensor View(int[] new_shape, bool inline = false)
{
var newSize = 1;
for (var i = 0; i < new_shape.Length; i++)
{
newSize *= new_shape[i];
}
var result = this;
if (newSize != size)
{
return(result);
}
if (dataOnGpu)
{
if (inline)
{
shape = new_shape;
shapeBuffer.Release();
shapeBuffer = new ComputeBuffer(shape.Length, sizeof(int));
shapeBuffer.SetData(shape);
}
else
{
result = new FloatTensor(_controller: controller, _shape: new_shape, _shader: this.shader);
result.Gpu(shader);
CopyBuffer(dataBuffer, result.DataBuffer);
}
}
else if (inline)
{
shape = new_shape;
}
else
{
result = new FloatTensor(_controller: controller, _data: data, _shape: new_shape, _shader: shader);
}
return(result);
}
public FloatTensor Sin(bool inline = false)
{
FloatTensor result = factory.ctrl.floatTensorFactory.Create(shape);
if (dataOnGpu)
{
result.Gpu(shader);
int kernel_id = shader.FindKernel("SinInt");
shader.SetBuffer(kernel_id, "SinIntData", this.DataBuffer);
shader.SetBuffer(kernel_id, "SinIntDataResult", result.DataBuffer);
shader.Dispatch(kernel_id, this.size, 1, 1);
return(result);
}
if (inline)
{
throw new NotImplementedException();
}
result.Data = data.AsParallel().Select(x => (float)Math.Sin((double)x)).ToArray();
return(result);
}
public FloatTensor Add(float value)
{
var result = new FloatTensor(shape, this.shader, false);
if (dataOnGpu)
{
result.Gpu();
return(AddScalarGPU(value, result));
}
else
{
var nCpu = SystemInfo.processorCount;
Parallel.For(0, nCpu, workerId => {
var max = size * (workerId + 1) / nCpu;
for (var i = size * workerId / nCpu; i < max; i++)
{
result.Data [i] = value + Data [i];
}
});
}
return(result);
}
public FloatTensor Abs()
// Returns a new Tensor with the smallest integer greater than or equal to each element
{
var result = new FloatTensor(shape, this.shader, dataOnGpu);
if (dataOnGpu)
{
result.Gpu();
return(AbsGPU(result));
}
else
{
var nCpu = SystemInfo.processorCount;
Parallel.For(0, nCpu, workerId => {
var max = size * (workerId + 1) / nCpu;
for (var i = size * workerId / nCpu; i < max; i++)
{
result.Data [i] = (float)(Math.Abs(Data [i]));
}
});
}
return(result);
}
public FloatTensor Pow(float value, bool inline = false, FloatTensor result = null)
{
if (inline & autograd)
{
throw new InvalidOperationException("Cannot call inline functions if you intend to run backprop.");
}
result = HookAutograd(ref result, value, "pow_scalar", inline);
if (dataOnGpu)
{
result.Gpu(shader);
if (!inline)
{
return(PowScalarGPU(value, result));
}
PowScalarGPU_(value);
return(this);
}
result.Data = data.AsParallel().Select(x => (float)Math.Pow((double)x, value)).ToArray();
return(result);
}
// hook autograd two parents
public FloatTensor HookAutograd(ref FloatTensor result, ref FloatTensor x, string creation_op, bool inline = false, int[] resultShape = null)
{
if (inline)
{
return(this);
}
// checks to see if the input has been seen previously. If so, then it assumes
// that we should just use the previous computation graph instead of initializing
// a new result. The assumption here is that if the same tensors are used to perform
// the same operation, then they should output to the same memory instead of allocating
// new memory.
bool autograd_pre_initialized = false;
if (result == null)
{
bool child_pre_initialized = false;
int child_index = 0;
if (this.children_indices.Count > 0)
{
// iterate through children
for (int i = 0; i < this.children_indices.Count; i++)
{
FloatTensor temp = factory.Get(children_indices[i]);
// if a child was created using the same op as the one currently being called
// and the child was also created using the same tensor as x
// then it's exactly the same operation and we can re-use variables.
if (temp.creation_op == creation_op && temp.creators.Contains(x.id))
{
child_pre_initialized = true;
child_index = children_indices[i];
}
}
}
if (child_pre_initialized)
{
//Debug.Log("Id:" + this.id + " Children:" + this.children_indices.Count);
autograd_pre_initialized = true;
result = factory.Get(child_index);
result.Zero_();
//Debug.Log("Graph:148:Fetching Tensor:" + result.id + " with creation_op:" + result.creation_op + " called under creation op:" + creation_op);
}
else
{
if (resultShape != null)
{
// initializes an empty tensor with new shape
result = factory.Create(
_shape: resultShape,
_dataOnGpu: dataOnGpu,
_autograd: x.autograd && autograd,
_keepgrads: keepgrads,
_creation_op: creation_op);
//Debug.Log("Graph:187:Creating Tensor:" + result.id + " with creation_op:" + result.creation_op);
}
else
{
// initializes an empty tensor with identical shape
result = factory.Create(
_shape: this.shape,
_data: data,
_dataBuffer: dataBuffer,
_shapeBuffer: shapeBuffer,
_shader: shader,
_copyData: true,
_dataOnGpu: dataOnGpu,
_autograd: x.autograd && autograd, // if either tensor doesn't have gradients
_keepgrads: keepgrads, // neither does the result. This might not end up being
_creation_op: creation_op); // a good decision in the long run. We'll see.
//Debug.Log("Graph:202:Creating Tensor:" + result.id + " with creation_op:" + result.creation_op);
}
// this is sortof a backup check. In theory, the result tensor should have been
// initialized correctly.
if (this.dataOnGpu)
{
result.Gpu(shader);
}
}
}
if (autograd_pre_initialized)
{
this.ResetAutogradCounts();
result.ResetAutogradCounts();
x.ResetAutogradCounts();
}
else
{
result.InitGraph();
result.creators.Add(this.id);
result.creators.Add(x.id);
result.creation_op = creation_op;
children_indices.Add(result.Id);
children_counts.Add(0);
x.children_indices.Add(result.Id);
x.children_counts.Add(0);
this.sibling = x.id;
}
return(result);
}
public FloatTensor HookGraph(ref FloatTensor result,
string creation_op,
bool inline,
float scalar_input = -1,
FloatTensor[] tensor_inputs = null,
int[] resultShape = null,
float[] resultData = null,
IntTensor[] indices = null)
{
// no dynamic graph for inline operations
if (inline)
{
return(this);
}
bool autograd_pre_initialized = false;
// if we don't override with a result tensor being passed in, let's first look to see if we can reuse one
// from a previous operation - if not - we'll create our own.
if (result == null)
{
bool child_pre_initialized = false;
int child_index = 0;
// iterate through all children to see if any were created using the same parameters and creation_op
// as is currently being requested
for (int i = 0; i < this.children_indices.Count; i++)
{
FloatTensor child = factory.Get(children_indices[i]);
if (child.creation_op == creation_op)
{
// if this creation_op requires no parameters - then we only have to match
// on the creation_op itself - which we have already done.
if (scalar_input == -1 && (tensor_inputs == null || tensor_inputs.Length == 0))
{
child_pre_initialized = true;
child_index = children_indices[i];
break;
}
// since there are paremeters - now this child must match all parameters exactly
bool keep_looking = false;
if (scalar_input != -1)
{
if (child.creators.Count > 1)
{
if (factory.Get(child.creators[1]).data[0] != scalar_input)
{
keep_looking = true;
}
}
}
if (tensor_inputs != null && tensor_inputs.Length >= 1)
{
foreach (FloatTensor tensor in tensor_inputs)
{
if (!child.creators.Contains(tensor.id))
{
keep_looking = true;
}
}
}
if (keep_looking)
{
continue;
}
// found a child that matches all parameters
child_pre_initialized = true;
child_index = children_indices[i];
break;
}
}
if (child_pre_initialized)
{
autograd_pre_initialized = true;
result = factory.Get(child_index);
result.Zero_();
}
else
{
bool resultAutograd = autograd;
if (tensor_inputs != null)
{
foreach (FloatTensor tensor in tensor_inputs)
{
resultAutograd = tensor.autograd && resultAutograd;
}
}
if (resultShape == null)
{
resultShape = this.shape;
if (resultData == null)
{
resultData = this.data;
}
}
else
{
// if shape is passed in - initialize a new dataset with that shape
resultData = null;
}
result = factory.Create(
_shape: resultShape,
_data: resultData,
_dataBuffer: dataBuffer,
_shapeBuffer: shapeBuffer,
_shader: shader,
_copyData: true,
_dataOnGpu: dataOnGpu,
_autograd: resultAutograd, // if either tensor doesn't have gradients
_keepgrads: keepgrads, // neither does the result. This might not end up being
_creation_op: creation_op); // a good decision in the long run. We'll see.
if (this.dataOnGpu)
{
result.Gpu(shader);
}
}
}
if (autograd_pre_initialized)
{
this.ResetAutogradCounts();
result.ResetAutogradCounts();
if (tensor_inputs != null)
{
foreach (FloatTensor tensor in tensor_inputs)
{
tensor.ResetAutogradCounts();
}
}
}
else
{
result.InitGraph();
result.creators.Add(this.id);
result.creation_op = creation_op;
children_indices.Add(result.Id);
children_counts.Add(0);
// hook autograd one parents - one scalar
if (scalar_input != -1)
{
result.creators.Add(factory.Create(
_shape: new int[] { 1 },
_data: new float[] { scalar_input },
_dataBuffer: dataBuffer,
_shapeBuffer: shapeBuffer,
_shader: shader,
_copyData: true,
_dataOnGpu: dataOnGpu,
_autograd: autograd,
_keepgrads: keepgrads,
_creation_op: creation_op).id);
}
// hook autograd - two parents
if (tensor_inputs != null)
{
foreach (FloatTensor tensor in tensor_inputs)
{
result.creators.Add(tensor.id);
tensor.children_indices.Add(result.Id);
tensor.children_counts.Add(0);
}
}
// special storage for the graph so that we can know which indices of the parent to
// backprop into. note that int_creators are expected to be non-differentiable and so we do
// not backprop into them directly
if (indices != null && indices.Length > 0)
{
if (result.int_creators.Count == 0)
{
foreach (IntTensor ind in indices)
{
result.int_creators.Add(ind.Id);
}
}
else if (result.int_creators.Count == indices.Length)
{
// TODO: after dynamic graph works for IntTensor you should be able to simply check to see if
// the ids are the same - but at the time of writing we always creating new IntTensors so that
// wouldn't work yet.
}
else
{
throw new Exception("Something is wrong... int_creators already existed but had the wrong length");
}
}
// TODO: this is just used so that eventually if any inline operation was run on "indices" to change it
// (before backpropagating), we could trigger a warning that backprop will be broken.
//indices.children_indices.Add(result.id);
}
return(result);
}