Esempio n. 1
0
        public WeightTensor(int rows, int columns, float c, int deviceId, bool keepCache = true)
        {
            DeviceId = deviceId;
            Rows     = rows;
            Columns  = columns;

            var n = rows * columns;

            var allocator = TensorAllocator.Allocator(deviceId);

            TGradient = new Tensor(allocator, DType.Float32, Rows, Columns);
            Ops.Fill(TGradient, 0.0f);

            if (keepCache)
            {
                TCache = new Tensor(allocator, DType.Float32, Rows, Columns);
                Ops.Fill(TCache, 0.0f);

                TLrW = new Tensor(allocator, DType.Float32, Rows, Columns);
                Ops.Fill(TLrW, 0.0f);
            }

            TWeight = new Tensor(allocator, DType.Float32, Rows, Columns);
            Ops.Fill(TWeight, c);
        }
Esempio n. 2
0
        public WeightTensor(long[] sizes, int deviceId, string name = "", bool isTrainable = false, bool normal = false)
        {
            Name        = name;
            DeviceId    = deviceId;
            IsTrainable = isTrainable;
            allocator   = TensorAllocator.Allocator(DeviceId);

            Sizes = sizes;

            if (normal)
            {
                var     n      = Rows * Columns;
                float[] weight = new float[n];


                var scale = (float)Math.Sqrt(1.0 / (Rows * Columns));
                if (normal)
                {
                    scale = 0.08f;
                }
                for (int i = 0; i < n; i++)
                {
                    weight[i] = RandomGenerator.NormalRandom(0.0f, scale);
                }

                TGradient = new Tensor(allocator, DType.Float32, Sizes);
                Ops.Fill(TGradient, 0.0f);

                TWeight = Tensor.FromArray(allocator, weight).View(Sizes);
            }
        }
Esempio n. 3
0
        public WeightTensor(int rows, int columns, int deviceId, bool normal = false)
        {
            DeviceId = deviceId;
            Rows     = rows;
            Columns  = columns;
            var n = rows * columns;

            float[] weight = new float[n];


            var scale = (float)Math.Sqrt(1.0 / (rows * columns));

            if (normal)
            {
                scale = 0.08f;
            }
            for (int i = 0; i < n; i++)
            {
                weight[i] = RandomGenerator.NormalRandom(0.0f, scale);
            }

            var allocator = TensorAllocator.Allocator(deviceId);

            TGradient = new Tensor(allocator, DType.Float32, Rows, Columns);
            Ops.Fill(TGradient, 0.0f);

            TCash = new Tensor(allocator, DType.Float32, Rows, Columns);
            Ops.Fill(TCash, 0.0f);

            TLrW = new Tensor(allocator, DType.Float32, Rows, Columns);
            Ops.Fill(TLrW, 0.0f);

            TWeight = Tensor.FromArray(allocator, weight).View(Rows, Columns);
        }
Esempio n. 4
0
        public WeightTensor(long[] sizes, float c, int deviceId, string name = "", bool isTrainable = false)
        {
            Name        = name;
            DeviceId    = deviceId;
            IsTrainable = isTrainable;
            Sizes       = sizes;
            m_allocator = TensorAllocator.Allocator(DeviceId);

            TWeight = new Tensor(m_allocator, DType.Float32, Sizes);
            Ops.Fill(TWeight, c);
        }
Esempio n. 5
0
        public float UpdateCost(IWeightTensor m, int[] ids)
        {
            WeightTensor t = m as WeightTensor;

            using (Tensor idsTensor = new Tensor(TensorAllocator.Allocator(m_deviceId), DType.Int32, 1, ids.Length))
            {
                idsTensor.SetElementsAsInt(ids);
                using (Tensor costs = Ops.UpdateCost(null, t.TWeight, idsTensor))
                {
                    return(Ops.SumAll(costs));
                }
            }
        }
Esempio n. 6
0
 public void CopyOrAddGradient(WeightTensor src)
 {
     if (m_TGradient == null)
     {
         allocator   = TensorAllocator.Allocator(DeviceId);
         m_TGradient = new Tensor(allocator, DType.Float32, Rows, Columns);
         Ops.Copy(m_TGradient, src.TGradient);
     }
     else
     {
         Ops.Add(m_TGradient, m_TGradient, src.TGradient);
     }
 }
Esempio n. 7
0
 public void AddMulGradient(Tensor w, Tensor g)
 {
     if (m_TGradient == null)
     {
         allocator   = TensorAllocator.Allocator(DeviceId);
         m_TGradient = new Tensor(allocator, DType.Float32, w.Sizes);
         Ops.Mul(m_TGradient, w, g);
     }
     else
     {
         Ops.AddMul(m_TGradient, m_TGradient, w, g);
     }
 }
Esempio n. 8
0
 public void AddSigmoidGradient(WeightTensor src)
 {
     if (m_TGradient == null)
     {
         allocator   = TensorAllocator.Allocator(DeviceId);
         m_TGradient = new Tensor(allocator, DType.Float32, src.TWeight.Sizes);
         Ops.SigmoidD(m_TGradient, src.TWeight, src.TGradient);
     }
     else
     {
         Ops.AddSigmoidD(m_TGradient, m_TGradient, src.TWeight, src.TGradient);
     }
 }
Esempio n. 9
0
 public void AddSoftmaxGradient(WeightTensor src)
 {
     if (m_TGradient == null)
     {
         allocator   = TensorAllocator.Allocator(DeviceId);
         m_TGradient = new Tensor(allocator, DType.Float32, src.TGradient.Sizes);
         Ops.SoftmaxGrad(m_TGradient, src.TGradient, src.TWeight, false);
     }
     else
     {
         Ops.SoftmaxGrad(m_TGradient, src.TGradient, src.TWeight);
     }
 }
Esempio n. 10
0
 public void CopyOrAddGradient(Tensor src)
 {
     if (m_TGradient == null)
     {
         allocator   = TensorAllocator.Allocator(DeviceId);
         m_TGradient = new Tensor(allocator, DType.Float32, src.Sizes);
         Ops.Copy(m_TGradient, src);
     }
     else
     {
         Ops.Add(m_TGradient, m_TGradient, src);
     }
 }
Esempio n. 11
0
        public WeightTensor(int rows, int columns, int deviceId)
        {
            DeviceId = deviceId;
            Rows     = rows;
            Columns  = columns;

            var allocator = TensorAllocator.Allocator(deviceId);

            TGradient = new Tensor(allocator, DType.Float32, Rows, Columns);
            Ops.Fill(TGradient, 0.0f);

            TWeight = new Tensor(allocator, DType.Float32, Rows, Columns);
        }
Esempio n. 12
0
        public IWeightMatrix SoftmaxM(IWeightMatrix w, bool bp = true)
        {
            WeightTensor m   = w as WeightTensor;
            var          res = weightTensorFactory.CreateWeightTensor(m.Rows, m.Columns, deviceId, new Tensor(TensorAllocator.Allocator(deviceId), DType.Float32, m.Rows, m.Columns), bp);

            Tensor tTmp = new Tensor(TensorAllocator.Allocator(deviceId), DType.Float32, m.Rows, m.Columns);

            var maxval  = Ops.Max(null, m.TWeight, 1);
            var maxvalM = maxval.Expand(m.Rows, m.Columns);

            Ops.ExpSub2(tTmp, m.TWeight, maxvalM);

            var sumV = Ops.Sum(null, tTmp, 1);
            var sumM = sumV.Expand(m.Rows, m.Columns);

            Ops.Div(res.TWeight, tTmp, sumM);

            maxval.Dispose();
            maxvalM.Dispose();
            sumV.Dispose();
            sumM.Dispose();

            if (this.needs_backprop && bp)
            {
                Action backward = () =>
                {
                    Ops.Mul(tTmp, res.TGradient, res.TWeight);
                    Ops.Add(m.TGradient, m.TGradient, tTmp);

                    var ss  = Ops.Sum(null, tTmp, 1);
                    var ssN = Ops.Neg(null, ss);

                    var ssM = ssN.Expand(m.Rows, m.Columns);
                    Ops.AddMul(m.TGradient, m.TGradient, res.TWeight, ssM);


                    tTmp.Dispose();
                    ss.Dispose();
                    ssM.Dispose();
                    ssN.Dispose();
                };
                this.backprop.Add(backward);
            }
            else
            {
                tTmp.Dispose();
            }

            return(res);
        }
Esempio n. 13
0
        public WeightTensor(long[] sizes, float c, int deviceId, string name = "", bool isTrainable = false)
        {
            Name        = name;
            DeviceId    = deviceId;
            IsTrainable = isTrainable;
            allocator   = TensorAllocator.Allocator(DeviceId);
            Sizes       = sizes;

            var n = Rows * Columns;

            TGradient = new Tensor(allocator, DType.Float32, Sizes);
            Ops.Fill(TGradient, 0.0f);

            TWeight = new Tensor(allocator, DType.Float32, Sizes);
            Ops.Fill(TWeight, c);
        }
Esempio n. 14
0
        private Tensor BuildRandomTensor(int rows, int columns, int batchSize, float prob)
        {
            using (Tensor noise = new Tensor(TensorAllocator.Allocator(m_deviceId), DType.Float32, rows / batchSize, columns))
            {
                Ops.RandomBernoulli(noise, seedSource, prob);

                if (rows / batchSize == 1)
                {
                    return(noise.Expand(rows, columns));
                }
                else
                {
                    return(noise.RepeatTensor(batchSize, 1));
                }
            }
        }
Esempio n. 15
0
        public WeightTensor(int rows, int columns, Tensor weight, int deviceId, bool graident = true)
        {
            DeviceId = deviceId;
            Rows     = rows;
            Columns  = columns;

            TWeight = weight;

            if (graident)
            {
                var allocator = TensorAllocator.Allocator(deviceId);

                TGradient = new Tensor(allocator, DType.Float32, Rows, Columns);
                Ops.Fill(TGradient, 0.0f);
            }
        }
Esempio n. 16
0
        private Tensor BuildRandomTensor(int rows, int columns, int batchSize, float prob)
        {
            using (Tensor noise = new Tensor(TensorAllocator.Allocator(m_deviceId), DType.Float32, rows / batchSize, columns))
            {
                float[] w = TensorSharp.RandomGenerator.BuildRandomBernoulliWeight(new long[] { rows / batchSize, columns }, prob);
                noise.SetElementsAsFloat(w);

                if (rows / batchSize == 1)
                {
                    return(noise.Expand(rows, columns));
                }
                else
                {
                    return(noise.RepeatTensor(batchSize, 1));
                }
            }
        }
Esempio n. 17
0
        private Tensor BuildRandomTensor(int rows, int columns, double prob)
        {
            float[] weights = new float[rows * columns];
            for (int i = 0; i < weights.Length; i++)
            {
                double r = rnd.NextDouble();
                if (r < prob)
                {
                    weights[i] = 1.0f;
                }
            }

            Tensor noise = new Tensor(TensorAllocator.Allocator(deviceId), DType.Float32, rows, columns);

            noise.SetElementsAsFloat(weights);

            return(noise);
        }
Esempio n. 18
0
 public void AddMulGradient(Tensor w, Tensor g, bool inPlace = false)
 {
     if (m_TGradient == null)
     {
         m_allocator = TensorAllocator.Allocator(DeviceId);
         m_TGradient = new Tensor(m_allocator, DType.Float32, w.Sizes);
         Ops.Mul(m_TGradient, w, g);
     }
     else
     {
         if (inPlace)
         {
             Ops.Mul(m_TGradient, w, g);
         }
         else
         {
             Ops.AddMul(m_TGradient, m_TGradient, w, g);
         }
     }
 }
Esempio n. 19
0
        public WeightTensor(long[] sizes, int deviceId, string name = "", bool isTrainable = false, bool normal = false)
        {
            Name        = name;
            DeviceId    = deviceId;
            IsTrainable = isTrainable;
            m_allocator = TensorAllocator.Allocator(DeviceId);
            Sizes       = sizes;

            if (normal)
            {
                var     n      = Rows * Columns;
                float[] weight = new float[n];

                var scale = (float)Math.Sqrt(2.0 / Rows);
                for (int i = 0; i < n; i++)
                {
                    weight[i] = RandomGenerator.NormalRandom(0.0f, scale);
                }

                TWeight = Tensor.FromArray(m_allocator, weight).View(Sizes);
            }
        }
Esempio n. 20
0
        public IWeightMatrix Softmax(IWeightMatrix w, bool bp = true)
        {
            WeightTensor m   = w as WeightTensor;
            var          res = weightTensorFactory.CreateWeightTensor(m.Rows, m.Columns, deviceId, new Tensor(TensorAllocator.Allocator(deviceId), DType.Float32, m.Rows, m.Columns), bp);

            Ops.Softmax(res.TWeight, m.TWeight);

            if (this.needs_backprop && bp)
            {
                Action backward = () =>
                {
                    Ops.SoftmaxGrad(m.TGradient, res.TGradient, res.TWeight);
                    res.Dispose();
                };
                this.backprop.Add(backward);
            }

            return(res);
        }