Example #1
0
        public FloatTensor HookGraph(ref FloatTensor result,
                                     string creation_op,
                                     bool inline,
                                     float scalar_input          = -1,
                                     FloatTensor[] tensor_inputs = null,
                                     int[] resultShape           = null,
                                     float[] resultData          = null,
                                     IntTensor[] indices         = null)
        {
            // no dynamic graph for inline operations
            if (inline)
            {
                return(this);
            }

            bool autograd_pre_initialized = false;

            // if we don't override with a result tensor being passed in, let's first look to see if we can reuse one
            // from a previous operation - if not - we'll create our own.
            if (result == null)
            {
                bool child_pre_initialized = false;
                int  child_index           = 0;


                // iterate through all children to see if any were created using the same parameters and creation_op
                // as is currently being requested
                for (int i = 0; i < this.children_indices.Count; i++)
                {
                    FloatTensor child = factory.Get(children_indices[i]);

                    if (child.creation_op == creation_op)
                    {
                        // if this creation_op requires no parameters - then we only have to match
                        // on the creation_op itself - which we have already done.
                        if (scalar_input == -1 && (tensor_inputs == null || tensor_inputs.Length == 0))
                        {
                            child_pre_initialized = true;
                            child_index           = children_indices[i];
                            break;
                        }

                        // since there are paremeters - now this child must match all parameters exactly
                        bool keep_looking = false;

                        if (scalar_input != -1)
                        {
                            if (child.creators.Count > 1)
                            {
                                if (factory.Get(child.creators[1]).data[0] != scalar_input)
                                {
                                    keep_looking = true;
                                }
                            }
                        }

                        if (tensor_inputs != null && tensor_inputs.Length >= 1)
                        {
                            foreach (FloatTensor tensor in tensor_inputs)
                            {
                                if (!child.creators.Contains(tensor.id))
                                {
                                    keep_looking = true;
                                }
                            }
                        }

                        if (keep_looking)
                        {
                            continue;
                        }

                        // found a child that matches all parameters
                        child_pre_initialized = true;
                        child_index           = children_indices[i];
                        break;
                    }
                }

                if (child_pre_initialized)
                {
                    autograd_pre_initialized = true;
                    result = factory.Get(child_index);
                    result.Zero_();
                }
                else
                {
                    bool resultAutograd = autograd;

                    if (tensor_inputs != null)
                    {
                        foreach (FloatTensor tensor in tensor_inputs)
                        {
                            resultAutograd = tensor.autograd && resultAutograd;
                        }
                    }

                    if (resultShape == null)
                    {
                        resultShape = this.shape;

                        if (resultData == null)
                        {
                            resultData = this.data;
                        }
                    }
                    else
                    {
                        // if shape is passed in - initialize a new dataset with that shape
                        resultData = null;
                    }

                    result = factory.Create(
                        _shape: resultShape,
                        _data: resultData,
                        _dataBuffer: dataBuffer,
                        _shapeBuffer: shapeBuffer,
                        _shader: shader,
                        _copyData: true,
                        _dataOnGpu: dataOnGpu,
                        _autograd: resultAutograd,                 // if either tensor doesn't have gradients
                        _keepgrads: keepgrads,                     // neither does the result. This might not end up being
                        _creation_op: creation_op);                // a good decision in the long run. We'll see.

                    if (this.dataOnGpu)
                    {
                        result.Gpu(shader);
                    }
                }
            }
            if (autograd_pre_initialized)
            {
                this.ResetAutogradCounts();
                result.ResetAutogradCounts();

                if (tensor_inputs != null)
                {
                    foreach (FloatTensor tensor in tensor_inputs)
                    {
                        tensor.ResetAutogradCounts();
                    }
                }
            }
            else
            {
                result.InitGraph();
                result.creators.Add(this.id);
                result.creation_op = creation_op;

                children_indices.Add(result.Id);
                children_counts.Add(0);

                // hook autograd one parents - one scalar
                if (scalar_input != -1)
                {
                    result.creators.Add(factory.Create(
                                            _shape: new int[] { 1 },
                                            _data: new float[] { scalar_input },
                                            _dataBuffer: dataBuffer,
                                            _shapeBuffer: shapeBuffer,
                                            _shader: shader,
                                            _copyData: true,
                                            _dataOnGpu: dataOnGpu,
                                            _autograd: autograd,
                                            _keepgrads: keepgrads,
                                            _creation_op: creation_op).id);
                }

                // hook autograd - two parents
                if (tensor_inputs != null)
                {
                    foreach (FloatTensor tensor in tensor_inputs)
                    {
                        result.creators.Add(tensor.id);
                        tensor.children_indices.Add(result.Id);
                        tensor.children_counts.Add(0);
                    }
                }


                // special storage for the graph so that we can know which indices of the parent to
                // backprop into. note that int_creators are expected to be non-differentiable and so we do
                // not backprop into them directly
                if (indices != null && indices.Length > 0)
                {
                    if (result.int_creators.Count == 0)
                    {
                        foreach (IntTensor ind in indices)
                        {
                            result.int_creators.Add(ind.Id);
                        }
                    }
                    else if (result.int_creators.Count == indices.Length)
                    {
                        // TODO: after dynamic graph works for IntTensor you should be able to simply check to see if
                        // the ids are the same - but at the time of writing we always creating new IntTensors so that
                        // wouldn't work yet.
                    }
                    else
                    {
                        throw new Exception("Something is wrong... int_creators already existed but had the wrong length");
                    }
                }

                // TODO: this is just used so that eventually if any inline operation was run on "indices" to change it
                // (before backpropagating), we could trigger a warning that backprop will be broken.
                //indices.children_indices.Add(result.id);
            }
            return(result);
        }
Example #2
0
        // hook autograd single parent
        public FloatTensor HookAutograd(ref FloatTensor result, string creation_op, bool inline = false, int[] resultShape = null)
        {
            if (inline)
            {
                return(this);
            }

            bool autograd_pre_initialized = false;

            //Debug.Log("Id:" + this.id + " Children:" + this.children.Count);
            if (result == null)
            {
                bool child_pre_initialized = false;
                int  child_index           = 0;
                if (this.children_indices.Count > 0)
                {
                    for (int i = 0; i < this.children_indices.Count; i++)
                    {
                        if (factory.Get(children_indices[i]).creation_op == creation_op)
                        {
                            child_pre_initialized = true;
                            child_index           = children_indices[i];
                        }
                    }
                }

                if (child_pre_initialized)
                {
                    autograd_pre_initialized = true;
                    result = factory.Get(child_index);
                    result.Zero_();
                    //Debug.Log("Graph:237:Fetching Tensor:" + result.id + " with creation_op:" + result.creation_op + " called under creation op:" + creation_op);
                }
                else
                {
                    if (resultShape != null)
                    {
                        result = factory.Create(
                            _shape: resultShape,
                            _dataOnGpu: dataOnGpu,
                            _autograd: autograd,
                            _keepgrads: keepgrads,
                            _creation_op: creation_op);
                        //Debug.Log("Graph:187:Creating Tensor:" + result.id + " with creation_op:" + result.creation_op);
                    }
                    else
                    {
                        result = factory.Create(
                            _shape: this.shape,
                            _data: data,
                            _dataBuffer: dataBuffer,
                            _shapeBuffer: shapeBuffer,
                            _shader: shader,
                            _copyData: true,
                            _dataOnGpu: dataOnGpu,
                            _autograd: autograd,
                            _keepgrads: keepgrads,
                            _creation_op: creation_op);

                        //Debug.Log("Graph:254:Creating Tensor:" + result.id + " with creation_op:" + result.creation_op);
                    }
                }
            }

            if (autograd_pre_initialized)
            {
                this.ResetAutogradCounts();
                result.ResetAutogradCounts();
            }
            else
            {
                result.InitGraph();
                result.creators.Add(this.id);
                result.creation_op = creation_op;

                children_indices.Add(result.Id);
                children_counts.Add(0);
            }

            return(result);
        }
Example #3
0
        // hook autograd one parents - one scalar
        public FloatTensor HookAutograd(ref FloatTensor result, float x, string creation_op, bool inline)
        {
            if (inline)
            {
                return(this);
            }

            bool autograd_pre_initialized = false;

            if (result == null)
            {
                bool child_pre_initialized = false;
                int  child_index           = 0;
                if (this.children_indices.Count > 0)

                {
                    for (int i = 0; i < this.children_indices.Count; i++)
                    {
                        FloatTensor temp = factory.Get(children_indices[i]);

                        if (temp.creation_op == creation_op)
                        {
                            if (temp.creators.Count > 1)
                            {
                                FloatTensor temp2 = factory.Get(temp.creators[1]);
                                if (temp2.data[0] == x)
                                {
                                    //if (temp2.autograd == temp.autograd)
                                    //{
                                    child_pre_initialized = true;
                                    child_index           = children_indices[i];
                                    //}
                                }
                            }
                        }
                    }
                }

                if (child_pre_initialized)
                {
                    autograd_pre_initialized = true;
                    result = factory.Get(child_index);
                    result.Zero_();
                    //Debug.Log("Graph:93:Fetching Tensor:" + result.id + " with creation_op:" + result.creation_op + " called under creation op:" + creation_op);
                }
                else
                {
                    result = factory.Create(_shape: this.shape,
                                            _data: data,
                                            _dataBuffer: dataBuffer,
                                            _shapeBuffer: shapeBuffer,
                                            _shader: shader,
                                            _copyData: true,
                                            _dataOnGpu: dataOnGpu,
                                            _autograd: autograd,
                                            _keepgrads: keepgrads,
                                            _creation_op: creation_op);


                    //Debug.Log("Graph:109:Creating Tensor:" + result.id + " with creation_op:" + result.creation_op);
                }
            }

            if (autograd_pre_initialized)
            {
                this.ResetAutogradCounts();
                result.ResetAutogradCounts();
            }
            else
            {
/*
 *                      FloatTensor new_child =
 *                              new FloatTensor(_controller: controller, _shape: , _data: new float[] {x});
 */

                FloatTensor new_child = factory.Create(
                    _shape: new int[] { 1 },
                    _data: new float[] { x },
                    _dataBuffer: dataBuffer,
                    _shapeBuffer: shapeBuffer,
                    _shader: shader,
                    _copyData: true,
                    _dataOnGpu: dataOnGpu,
                    _autograd: autograd,
                    _keepgrads: keepgrads,
                    _creation_op: creation_op);


                result.InitGraph();
                result.creators.Add(this.id);
                result.creators.Add(new_child.id);
                result.creation_op = creation_op;

                children_indices.Add(result.Id);
                children_counts.Add(0);
            }

            return(result);
        }
Example #4
0
        // hook autograd two parents
        public FloatTensor HookAutograd(ref FloatTensor result, ref FloatTensor x, string creation_op, bool inline = false, int[] resultShape = null)
        {
            if (inline)
            {
                return(this);
            }

            // checks to see if the input has been seen previously. If so, then it assumes
            // that we should just use the previous computation graph instead of initializing
            // a new result. The assumption here is that if the same tensors are used to perform
            // the same operation, then they should output to the same memory instead of allocating
            // new memory.
            bool autograd_pre_initialized = false;

            if (result == null)
            {
                bool child_pre_initialized = false;
                int  child_index           = 0;
                if (this.children_indices.Count > 0)
                {
                    // iterate through children
                    for (int i = 0; i < this.children_indices.Count; i++)
                    {
                        FloatTensor temp = factory.Get(children_indices[i]);

                        // if a child was created using the same op as the one currently being called
                        // and the child was also created using the same tensor as x
                        // then it's exactly the same operation and we can re-use variables.
                        if (temp.creation_op == creation_op && temp.creators.Contains(x.id))
                        {
                            child_pre_initialized = true;
                            child_index           = children_indices[i];
                        }
                    }
                }

                if (child_pre_initialized)
                {
                    //Debug.Log("Id:" + this.id + " Children:" + this.children_indices.Count);
                    autograd_pre_initialized = true;
                    result = factory.Get(child_index);
                    result.Zero_();
                    //Debug.Log("Graph:148:Fetching Tensor:" + result.id + " with creation_op:" + result.creation_op + " called under creation op:" + creation_op);
                }
                else
                {
                    if (resultShape != null)
                    {
                        // initializes an empty tensor with new shape
                        result = factory.Create(
                            _shape: resultShape,
                            _dataOnGpu: dataOnGpu,
                            _autograd: x.autograd && autograd,
                            _keepgrads: keepgrads,
                            _creation_op: creation_op);
                        //Debug.Log("Graph:187:Creating Tensor:" + result.id + " with creation_op:" + result.creation_op);
                    }
                    else
                    {
                        // initializes an empty tensor with identical shape
                        result = factory.Create(
                            _shape: this.shape,
                            _data: data,
                            _dataBuffer: dataBuffer,
                            _shapeBuffer: shapeBuffer,
                            _shader: shader,
                            _copyData: true,
                            _dataOnGpu: dataOnGpu,
                            _autograd: x.autograd && autograd,                             // if either tensor doesn't have gradients
                            _keepgrads: keepgrads,                                         // neither does the result. This might not end up being
                            _creation_op: creation_op);                                    // a good decision in the long run. We'll see.
                        //Debug.Log("Graph:202:Creating Tensor:" + result.id + " with creation_op:" + result.creation_op);
                    }


                    // this is sortof a backup check. In theory, the result tensor should have been
                    // initialized correctly.
                    if (this.dataOnGpu)
                    {
                        result.Gpu(shader);
                    }
                }
            }

            if (autograd_pre_initialized)
            {
                this.ResetAutogradCounts();
                result.ResetAutogradCounts();
                x.ResetAutogradCounts();
            }
            else
            {
                result.InitGraph();
                result.creators.Add(this.id);
                result.creators.Add(x.id);
                result.creation_op = creation_op;

                children_indices.Add(result.Id);
                children_counts.Add(0);

                x.children_indices.Add(result.Id);
                x.children_counts.Add(0);

                this.sibling = x.id;
            }


            return(result);
        }