public FloatTensor HookGraph(ref FloatTensor result,
string creation_op,
bool inline,
float scalar_input = -1,
FloatTensor[] tensor_inputs = null,
int[] resultShape = null,
float[] resultData = null,
IntTensor[] indices = null)
{
// no dynamic graph for inline operations
if (inline)
{
return(this);
}
bool autograd_pre_initialized = false;
// if we don't override with a result tensor being passed in, let's first look to see if we can reuse one
// from a previous operation - if not - we'll create our own.
if (result == null)
{
bool child_pre_initialized = false;
int child_index = 0;
// iterate through all children to see if any were created using the same parameters and creation_op
// as is currently being requested
for (int i = 0; i < this.children_indices.Count; i++)
{
FloatTensor child = factory.Get(children_indices[i]);
if (child.creation_op == creation_op)
{
// if this creation_op requires no parameters - then we only have to match
// on the creation_op itself - which we have already done.
if (scalar_input == -1 && (tensor_inputs == null || tensor_inputs.Length == 0))
{
child_pre_initialized = true;
child_index = children_indices[i];
break;
}
// since there are paremeters - now this child must match all parameters exactly
bool keep_looking = false;
if (scalar_input != -1)
{
if (child.creators.Count > 1)
{
if (factory.Get(child.creators[1]).data[0] != scalar_input)
{
keep_looking = true;
}
}
}
if (tensor_inputs != null && tensor_inputs.Length >= 1)
{
foreach (FloatTensor tensor in tensor_inputs)
{
if (!child.creators.Contains(tensor.id))
{
keep_looking = true;
}
}
}
if (keep_looking)
{
continue;
}
// found a child that matches all parameters
child_pre_initialized = true;
child_index = children_indices[i];
break;
}
}
if (child_pre_initialized)
{
autograd_pre_initialized = true;
result = factory.Get(child_index);
result.Zero_();
}
else
{
bool resultAutograd = autograd;
if (tensor_inputs != null)
{
foreach (FloatTensor tensor in tensor_inputs)
{
resultAutograd = tensor.autograd && resultAutograd;
}
}
if (resultShape == null)
{
resultShape = this.shape;
if (resultData == null)
{
resultData = this.data;
}
}
else
{
// if shape is passed in - initialize a new dataset with that shape
resultData = null;
}
result = factory.Create(
_shape: resultShape,
_data: resultData,
_dataBuffer: dataBuffer,
_shapeBuffer: shapeBuffer,
_shader: shader,
_copyData: true,
_dataOnGpu: dataOnGpu,
_autograd: resultAutograd, // if either tensor doesn't have gradients
_keepgrads: keepgrads, // neither does the result. This might not end up being
_creation_op: creation_op); // a good decision in the long run. We'll see.
if (this.dataOnGpu)
{
result.Gpu(shader);
}
}
}
if (autograd_pre_initialized)
{
this.ResetAutogradCounts();
result.ResetAutogradCounts();
if (tensor_inputs != null)
{
foreach (FloatTensor tensor in tensor_inputs)
{
tensor.ResetAutogradCounts();
}
}
}
else
{
result.InitGraph();
result.creators.Add(this.id);
result.creation_op = creation_op;
children_indices.Add(result.Id);
children_counts.Add(0);
// hook autograd one parents - one scalar
if (scalar_input != -1)
{
result.creators.Add(factory.Create(
_shape: new int[] { 1 },
_data: new float[] { scalar_input },
_dataBuffer: dataBuffer,
_shapeBuffer: shapeBuffer,
_shader: shader,
_copyData: true,
_dataOnGpu: dataOnGpu,
_autograd: autograd,
_keepgrads: keepgrads,
_creation_op: creation_op).id);
}
// hook autograd - two parents
if (tensor_inputs != null)
{
foreach (FloatTensor tensor in tensor_inputs)
{
result.creators.Add(tensor.id);
tensor.children_indices.Add(result.Id);
tensor.children_counts.Add(0);
}
}
// special storage for the graph so that we can know which indices of the parent to
// backprop into. note that int_creators are expected to be non-differentiable and so we do
// not backprop into them directly
if (indices != null && indices.Length > 0)
{
if (result.int_creators.Count == 0)
{
foreach (IntTensor ind in indices)
{
result.int_creators.Add(ind.Id);
}
}
else if (result.int_creators.Count == indices.Length)
{
// TODO: after dynamic graph works for IntTensor you should be able to simply check to see if
// the ids are the same - but at the time of writing we always creating new IntTensors so that
// wouldn't work yet.
}
else
{
throw new Exception("Something is wrong... int_creators already existed but had the wrong length");
}
}
// TODO: this is just used so that eventually if any inline operation was run on "indices" to change it
// (before backpropagating), we could trigger a warning that backprop will be broken.
//indices.children_indices.Add(result.id);
}
return(result);
}
// hook autograd single parent
public FloatTensor HookAutograd(ref FloatTensor result, string creation_op, bool inline = false, int[] resultShape = null)
{
if (inline)
{
return(this);
}
bool autograd_pre_initialized = false;
//Debug.Log("Id:" + this.id + " Children:" + this.children.Count);
if (result == null)
{
bool child_pre_initialized = false;
int child_index = 0;
if (this.children_indices.Count > 0)
{
for (int i = 0; i < this.children_indices.Count; i++)
{
if (factory.Get(children_indices[i]).creation_op == creation_op)
{
child_pre_initialized = true;
child_index = children_indices[i];
}
}
}
if (child_pre_initialized)
{
autograd_pre_initialized = true;
result = factory.Get(child_index);
result.Zero_();
//Debug.Log("Graph:237:Fetching Tensor:" + result.id + " with creation_op:" + result.creation_op + " called under creation op:" + creation_op);
}
else
{
if (resultShape != null)
{
result = factory.Create(
_shape: resultShape,
_dataOnGpu: dataOnGpu,
_autograd: autograd,
_keepgrads: keepgrads,
_creation_op: creation_op);
//Debug.Log("Graph:187:Creating Tensor:" + result.id + " with creation_op:" + result.creation_op);
}
else
{
result = factory.Create(
_shape: this.shape,
_data: data,
_dataBuffer: dataBuffer,
_shapeBuffer: shapeBuffer,
_shader: shader,
_copyData: true,
_dataOnGpu: dataOnGpu,
_autograd: autograd,
_keepgrads: keepgrads,
_creation_op: creation_op);
//Debug.Log("Graph:254:Creating Tensor:" + result.id + " with creation_op:" + result.creation_op);
}
}
}
if (autograd_pre_initialized)
{
this.ResetAutogradCounts();
result.ResetAutogradCounts();
}
else
{
result.InitGraph();
result.creators.Add(this.id);
result.creation_op = creation_op;
children_indices.Add(result.Id);
children_counts.Add(0);
}
return(result);
}
// hook autograd one parents - one scalar
public FloatTensor HookAutograd(ref FloatTensor result, float x, string creation_op, bool inline)
{
if (inline)
{
return(this);
}
bool autograd_pre_initialized = false;
if (result == null)
{
bool child_pre_initialized = false;
int child_index = 0;
if (this.children_indices.Count > 0)
{
for (int i = 0; i < this.children_indices.Count; i++)
{
FloatTensor temp = factory.Get(children_indices[i]);
if (temp.creation_op == creation_op)
{
if (temp.creators.Count > 1)
{
FloatTensor temp2 = factory.Get(temp.creators[1]);
if (temp2.data[0] == x)
{
//if (temp2.autograd == temp.autograd)
//{
child_pre_initialized = true;
child_index = children_indices[i];
//}
}
}
}
}
}
if (child_pre_initialized)
{
autograd_pre_initialized = true;
result = factory.Get(child_index);
result.Zero_();
//Debug.Log("Graph:93:Fetching Tensor:" + result.id + " with creation_op:" + result.creation_op + " called under creation op:" + creation_op);
}
else
{
result = factory.Create(_shape: this.shape,
_data: data,
_dataBuffer: dataBuffer,
_shapeBuffer: shapeBuffer,
_shader: shader,
_copyData: true,
_dataOnGpu: dataOnGpu,
_autograd: autograd,
_keepgrads: keepgrads,
_creation_op: creation_op);
//Debug.Log("Graph:109:Creating Tensor:" + result.id + " with creation_op:" + result.creation_op);
}
}
if (autograd_pre_initialized)
{
this.ResetAutogradCounts();
result.ResetAutogradCounts();
}
else
{
/*
* FloatTensor new_child =
* new FloatTensor(_controller: controller, _shape: , _data: new float[] {x});
*/
FloatTensor new_child = factory.Create(
_shape: new int[] { 1 },
_data: new float[] { x },
_dataBuffer: dataBuffer,
_shapeBuffer: shapeBuffer,
_shader: shader,
_copyData: true,
_dataOnGpu: dataOnGpu,
_autograd: autograd,
_keepgrads: keepgrads,
_creation_op: creation_op);
result.InitGraph();
result.creators.Add(this.id);
result.creators.Add(new_child.id);
result.creation_op = creation_op;
children_indices.Add(result.Id);
children_counts.Add(0);
}
return(result);
}
// hook autograd two parents
public FloatTensor HookAutograd(ref FloatTensor result, ref FloatTensor x, string creation_op, bool inline = false, int[] resultShape = null)
{
if (inline)
{
return(this);
}
// checks to see if the input has been seen previously. If so, then it assumes
// that we should just use the previous computation graph instead of initializing
// a new result. The assumption here is that if the same tensors are used to perform
// the same operation, then they should output to the same memory instead of allocating
// new memory.
bool autograd_pre_initialized = false;
if (result == null)
{
bool child_pre_initialized = false;
int child_index = 0;
if (this.children_indices.Count > 0)
{
// iterate through children
for (int i = 0; i < this.children_indices.Count; i++)
{
FloatTensor temp = factory.Get(children_indices[i]);
// if a child was created using the same op as the one currently being called
// and the child was also created using the same tensor as x
// then it's exactly the same operation and we can re-use variables.
if (temp.creation_op == creation_op && temp.creators.Contains(x.id))
{
child_pre_initialized = true;
child_index = children_indices[i];
}
}
}
if (child_pre_initialized)
{
//Debug.Log("Id:" + this.id + " Children:" + this.children_indices.Count);
autograd_pre_initialized = true;
result = factory.Get(child_index);
result.Zero_();
//Debug.Log("Graph:148:Fetching Tensor:" + result.id + " with creation_op:" + result.creation_op + " called under creation op:" + creation_op);
}
else
{
if (resultShape != null)
{
// initializes an empty tensor with new shape
result = factory.Create(
_shape: resultShape,
_dataOnGpu: dataOnGpu,
_autograd: x.autograd && autograd,
_keepgrads: keepgrads,
_creation_op: creation_op);
//Debug.Log("Graph:187:Creating Tensor:" + result.id + " with creation_op:" + result.creation_op);
}
else
{
// initializes an empty tensor with identical shape
result = factory.Create(
_shape: this.shape,
_data: data,
_dataBuffer: dataBuffer,
_shapeBuffer: shapeBuffer,
_shader: shader,
_copyData: true,
_dataOnGpu: dataOnGpu,
_autograd: x.autograd && autograd, // if either tensor doesn't have gradients
_keepgrads: keepgrads, // neither does the result. This might not end up being
_creation_op: creation_op); // a good decision in the long run. We'll see.
//Debug.Log("Graph:202:Creating Tensor:" + result.id + " with creation_op:" + result.creation_op);
}
// this is sortof a backup check. In theory, the result tensor should have been
// initialized correctly.
if (this.dataOnGpu)
{
result.Gpu(shader);
}
}
}
if (autograd_pre_initialized)
{
this.ResetAutogradCounts();
result.ResetAutogradCounts();
x.ResetAutogradCounts();
}
else
{
result.InitGraph();
result.creators.Add(this.id);
result.creators.Add(x.id);
result.creation_op = creation_op;
children_indices.Add(result.Id);
children_counts.Add(0);
x.children_indices.Add(result.Id);
x.children_counts.Add(0);
this.sibling = x.id;
}
return(result);
}