public FloatTensor HookGraph(ref FloatTensor result, string creation_op, bool inline, float scalar_input = -1, FloatTensor[] tensor_inputs = null, int[] resultShape = null, float[] resultData = null, IntTensor[] indices = null) { // no dynamic graph for inline operations if (inline) { return(this); } bool autograd_pre_initialized = false; // if we don't override with a result tensor being passed in, let's first look to see if we can reuse one // from a previous operation - if not - we'll create our own. if (result == null) { bool child_pre_initialized = false; int child_index = 0; // iterate through all children to see if any were created using the same parameters and creation_op // as is currently being requested for (int i = 0; i < this.children_indices.Count; i++) { FloatTensor child = factory.Get(children_indices[i]); if (child.creation_op == creation_op) { // if this creation_op requires no parameters - then we only have to match // on the creation_op itself - which we have already done. if (scalar_input == -1 && (tensor_inputs == null || tensor_inputs.Length == 0)) { child_pre_initialized = true; child_index = children_indices[i]; break; } // since there are paremeters - now this child must match all parameters exactly bool keep_looking = false; if (scalar_input != -1) { if (child.creators.Count > 1) { if (factory.Get(child.creators[1]).data[0] != scalar_input) { keep_looking = true; } } } if (tensor_inputs != null && tensor_inputs.Length >= 1) { foreach (FloatTensor tensor in tensor_inputs) { if (!child.creators.Contains(tensor.id)) { keep_looking = true; } } } if (keep_looking) { continue; } // found a child that matches all parameters child_pre_initialized = true; child_index = children_indices[i]; break; } } if (child_pre_initialized) { autograd_pre_initialized = true; result = factory.Get(child_index); result.Zero_(); } else { bool resultAutograd = autograd; if (tensor_inputs != null) { foreach (FloatTensor tensor in tensor_inputs) { resultAutograd = tensor.autograd && resultAutograd; } } if (resultShape == null) { resultShape = this.shape; if (resultData == null) { resultData = this.data; } } else { // if shape is passed in - initialize a new dataset with that shape resultData = null; } result = factory.Create( _shape: resultShape, _data: resultData, _dataBuffer: dataBuffer, _shapeBuffer: shapeBuffer, _shader: shader, _copyData: true, _dataOnGpu: dataOnGpu, _autograd: resultAutograd, // if either tensor doesn't have gradients _keepgrads: keepgrads, // neither does the result. This might not end up being _creation_op: creation_op); // a good decision in the long run. We'll see. if (this.dataOnGpu) { result.Gpu(shader); } } } if (autograd_pre_initialized) { this.ResetAutogradCounts(); result.ResetAutogradCounts(); if (tensor_inputs != null) { foreach (FloatTensor tensor in tensor_inputs) { tensor.ResetAutogradCounts(); } } } else { result.InitGraph(); result.creators.Add(this.id); result.creation_op = creation_op; children_indices.Add(result.Id); children_counts.Add(0); // hook autograd one parents - one scalar if (scalar_input != -1) { result.creators.Add(factory.Create( _shape: new int[] { 1 }, _data: new float[] { scalar_input }, _dataBuffer: dataBuffer, _shapeBuffer: shapeBuffer, _shader: shader, _copyData: true, _dataOnGpu: dataOnGpu, _autograd: autograd, _keepgrads: keepgrads, _creation_op: creation_op).id); } // hook autograd - two parents if (tensor_inputs != null) { foreach (FloatTensor tensor in tensor_inputs) { result.creators.Add(tensor.id); tensor.children_indices.Add(result.Id); tensor.children_counts.Add(0); } } // special storage for the graph so that we can know which indices of the parent to // backprop into. note that int_creators are expected to be non-differentiable and so we do // not backprop into them directly if (indices != null && indices.Length > 0) { if (result.int_creators.Count == 0) { foreach (IntTensor ind in indices) { result.int_creators.Add(ind.Id); } } else if (result.int_creators.Count == indices.Length) { // TODO: after dynamic graph works for IntTensor you should be able to simply check to see if // the ids are the same - but at the time of writing we always creating new IntTensors so that // wouldn't work yet. } else { throw new Exception("Something is wrong... int_creators already existed but had the wrong length"); } } // TODO: this is just used so that eventually if any inline operation was run on "indices" to change it // (before backpropagating), we could trigger a warning that backprop will be broken. //indices.children_indices.Add(result.id); } return(result); }
// hook autograd single parent public FloatTensor HookAutograd(ref FloatTensor result, string creation_op, bool inline = false, int[] resultShape = null) { if (inline) { return(this); } bool autograd_pre_initialized = false; //Debug.Log("Id:" + this.id + " Children:" + this.children.Count); if (result == null) { bool child_pre_initialized = false; int child_index = 0; if (this.children_indices.Count > 0) { for (int i = 0; i < this.children_indices.Count; i++) { if (factory.Get(children_indices[i]).creation_op == creation_op) { child_pre_initialized = true; child_index = children_indices[i]; } } } if (child_pre_initialized) { autograd_pre_initialized = true; result = factory.Get(child_index); result.Zero_(); //Debug.Log("Graph:237:Fetching Tensor:" + result.id + " with creation_op:" + result.creation_op + " called under creation op:" + creation_op); } else { if (resultShape != null) { result = factory.Create( _shape: resultShape, _dataOnGpu: dataOnGpu, _autograd: autograd, _keepgrads: keepgrads, _creation_op: creation_op); //Debug.Log("Graph:187:Creating Tensor:" + result.id + " with creation_op:" + result.creation_op); } else { result = factory.Create( _shape: this.shape, _data: data, _dataBuffer: dataBuffer, _shapeBuffer: shapeBuffer, _shader: shader, _copyData: true, _dataOnGpu: dataOnGpu, _autograd: autograd, _keepgrads: keepgrads, _creation_op: creation_op); //Debug.Log("Graph:254:Creating Tensor:" + result.id + " with creation_op:" + result.creation_op); } } } if (autograd_pre_initialized) { this.ResetAutogradCounts(); result.ResetAutogradCounts(); } else { result.InitGraph(); result.creators.Add(this.id); result.creation_op = creation_op; children_indices.Add(result.Id); children_counts.Add(0); } return(result); }
// hook autograd one parents - one scalar public FloatTensor HookAutograd(ref FloatTensor result, float x, string creation_op, bool inline) { if (inline) { return(this); } bool autograd_pre_initialized = false; if (result == null) { bool child_pre_initialized = false; int child_index = 0; if (this.children_indices.Count > 0) { for (int i = 0; i < this.children_indices.Count; i++) { FloatTensor temp = factory.Get(children_indices[i]); if (temp.creation_op == creation_op) { if (temp.creators.Count > 1) { FloatTensor temp2 = factory.Get(temp.creators[1]); if (temp2.data[0] == x) { //if (temp2.autograd == temp.autograd) //{ child_pre_initialized = true; child_index = children_indices[i]; //} } } } } } if (child_pre_initialized) { autograd_pre_initialized = true; result = factory.Get(child_index); result.Zero_(); //Debug.Log("Graph:93:Fetching Tensor:" + result.id + " with creation_op:" + result.creation_op + " called under creation op:" + creation_op); } else { result = factory.Create(_shape: this.shape, _data: data, _dataBuffer: dataBuffer, _shapeBuffer: shapeBuffer, _shader: shader, _copyData: true, _dataOnGpu: dataOnGpu, _autograd: autograd, _keepgrads: keepgrads, _creation_op: creation_op); //Debug.Log("Graph:109:Creating Tensor:" + result.id + " with creation_op:" + result.creation_op); } } if (autograd_pre_initialized) { this.ResetAutogradCounts(); result.ResetAutogradCounts(); } else { /* * FloatTensor new_child = * new FloatTensor(_controller: controller, _shape: , _data: new float[] {x}); */ FloatTensor new_child = factory.Create( _shape: new int[] { 1 }, _data: new float[] { x }, _dataBuffer: dataBuffer, _shapeBuffer: shapeBuffer, _shader: shader, _copyData: true, _dataOnGpu: dataOnGpu, _autograd: autograd, _keepgrads: keepgrads, _creation_op: creation_op); result.InitGraph(); result.creators.Add(this.id); result.creators.Add(new_child.id); result.creation_op = creation_op; children_indices.Add(result.Id); children_counts.Add(0); } return(result); }
// hook autograd two parents public FloatTensor HookAutograd(ref FloatTensor result, ref FloatTensor x, string creation_op, bool inline = false, int[] resultShape = null) { if (inline) { return(this); } // checks to see if the input has been seen previously. If so, then it assumes // that we should just use the previous computation graph instead of initializing // a new result. The assumption here is that if the same tensors are used to perform // the same operation, then they should output to the same memory instead of allocating // new memory. bool autograd_pre_initialized = false; if (result == null) { bool child_pre_initialized = false; int child_index = 0; if (this.children_indices.Count > 0) { // iterate through children for (int i = 0; i < this.children_indices.Count; i++) { FloatTensor temp = factory.Get(children_indices[i]); // if a child was created using the same op as the one currently being called // and the child was also created using the same tensor as x // then it's exactly the same operation and we can re-use variables. if (temp.creation_op == creation_op && temp.creators.Contains(x.id)) { child_pre_initialized = true; child_index = children_indices[i]; } } } if (child_pre_initialized) { //Debug.Log("Id:" + this.id + " Children:" + this.children_indices.Count); autograd_pre_initialized = true; result = factory.Get(child_index); result.Zero_(); //Debug.Log("Graph:148:Fetching Tensor:" + result.id + " with creation_op:" + result.creation_op + " called under creation op:" + creation_op); } else { if (resultShape != null) { // initializes an empty tensor with new shape result = factory.Create( _shape: resultShape, _dataOnGpu: dataOnGpu, _autograd: x.autograd && autograd, _keepgrads: keepgrads, _creation_op: creation_op); //Debug.Log("Graph:187:Creating Tensor:" + result.id + " with creation_op:" + result.creation_op); } else { // initializes an empty tensor with identical shape result = factory.Create( _shape: this.shape, _data: data, _dataBuffer: dataBuffer, _shapeBuffer: shapeBuffer, _shader: shader, _copyData: true, _dataOnGpu: dataOnGpu, _autograd: x.autograd && autograd, // if either tensor doesn't have gradients _keepgrads: keepgrads, // neither does the result. This might not end up being _creation_op: creation_op); // a good decision in the long run. We'll see. //Debug.Log("Graph:202:Creating Tensor:" + result.id + " with creation_op:" + result.creation_op); } // this is sortof a backup check. In theory, the result tensor should have been // initialized correctly. if (this.dataOnGpu) { result.Gpu(shader); } } } if (autograd_pre_initialized) { this.ResetAutogradCounts(); result.ResetAutogradCounts(); x.ResetAutogradCounts(); } else { result.InitGraph(); result.creators.Add(this.id); result.creators.Add(x.id); result.creation_op = creation_op; children_indices.Add(result.Id); children_counts.Add(0); x.children_indices.Add(result.Id); x.children_counts.Add(0); this.sibling = x.id; } return(result); }