Exemplo n.º 1
0
        public override void Update(int iteration, BaseLayer layer)
        {
            if (DecayRate > 0)
            {
                LearningRate = LearningRate * (1 / 1 + DecayRate * iteration);
            }

            float t    = iteration + 1;
            float lr_t = Convert.ToSingle(LearningRate / Math.Sqrt(1f - Math.Pow(Beta1, t)));

            foreach (var item in layer.Params)
            {
                var param = item.Value;

                if (!ms.ContainsKey(param.Name))
                {
                    ms[param.Name] = Tensor.Constant(0, Global.Device, DType.Float32, param.Data.Shape);
                    us[param.Name] = Tensor.Constant(0, Global.Device, DType.Float32, param.Data.Shape);

                    var m_t = (Beta1 * ms[param.Name]) + (1 - Beta1) * param.Grad;
                    var u_t = TOps.Maximum((Beta2 * us[param.Name]), Abs(param.Grad));

                    param.Data     = param.Data - lr_t * m_t / (u_t + EPSILON);
                    ms[param.Name] = m_t;
                    us[param.Name] = u_t;

                    param.ApplyConstraint();
                }
            }
        }
Exemplo n.º 2
0
 public void Exponential(NDArray result, int?seed, float lambda)
 {
     using (var cpuCopy = new NDArray(cpuAllocator, result.ElementType, result.Shape))
     {
         cpuRandom.Exponential(cpuCopy, seed, lambda);
         TOps.Copy(result, cpuCopy);
     }
 }
Exemplo n.º 3
0
 public void Normal(NDArray result, int?seed, float mean, float stdv)
 {
     using (var cpuCopy = new NDArray(cpuAllocator, result.ElementType, result.Shape))
     {
         cpuRandom.Normal(cpuCopy, seed, mean, stdv);
         TOps.Copy(result, cpuCopy);
     }
 }
Exemplo n.º 4
0
 public void Uniform(NDArray result, int?seed, float min, float max)
 {
     using (var cpuCopy = new NDArray(cpuAllocator, result.ElementType, result.Shape))
     {
         cpuRandom.Uniform(cpuCopy, seed, min, max);
         TOps.Copy(result, cpuCopy);
     }
 }
Exemplo n.º 5
0
 public void Bernoulli(NDArray result, int?seed, float p)
 {
     using (var cpuCopy = new NDArray(cpuAllocator, result.ElementType, result.Shape))
     {
         cpuRandom.Bernoulli(cpuCopy, seed, p);
         TOps.Copy(result, cpuCopy);
     }
 }
Exemplo n.º 6
0
 public void Cauchy(NDArray result, int?seed, float median, float sigma)
 {
     using (var cpuCopy = new NDArray(cpuAllocator, result.ElementType, result.Shape))
     {
         cpuRandom.Cauchy(cpuCopy, seed, median, sigma);
         TOps.Copy(result, cpuCopy);
     }
 }
Exemplo n.º 7
0
 public void Geometric(Tensor result, int?seed, float p)
 {
     using (var cpuCopy = new Tensor(cpuAllocator, result.ElementType, result.Shape))
     {
         cpuRandom.Geometric(cpuCopy, seed, p);
         TOps.Copy(result, cpuCopy);
     }
 }
Exemplo n.º 8
0
 public void LogNormal(Tensor result, int?seed, float mean, float stdv)
 {
     using (var cpuCopy = new Tensor(cpuAllocator, result.ElementType, result.Shape))
     {
         cpuRandom.LogNormal(cpuCopy, seed, mean, stdv);
         TOps.Copy(result, cpuCopy);
     }
 }
Exemplo n.º 9
0
        public void Fit(IFrameIter train, int epochs, int batchSize, IFrameIter val = null)
        {
            DateTime     start         = DateTime.Now;
            List <float> train_losses  = new List <float>();
            List <float> train_metrics = new List <float>();
            List <float> val_losses    = new List <float>();
            List <float> val_metrics   = new List <float>();

            train.SetBatchSize(batchSize);
            for (int iteration = 1; iteration <= epochs; iteration++)
            {
                train.Reset();
                while (train.Next())
                {
                    var(x, y) = train.GetBatch();

                    using (Variable pred = Forward(x))
                        using (Tensor lossVal = LossFn.Call(pred.Data, y))
                            using (Tensor grad = LossFn.CalcGrad(pred.Data, y))
                                using (Tensor reg_loss = ApplyRegularizer(lossVal))
                                {
                                    //var metricVal = MetricFn.Call(pred.Data, y);
                                    train_losses.Add(reg_loss.TVar().ToScalar().Evaluate());
                                    //train_metrics.Add(metricVal.ToScalar().Evaluate());

                                    Backward(grad);

                                    ApplyDeltaRegularizer();

                                    foreach (var layer in Layers)
                                    {
                                        OptimizerFn.Update(iteration, layer);
                                    }
                                }

                    x.Dispose();
                    y.Dispose();
                }

                if (val != null)
                {
                    while (val.Next())
                    {
                        var(x, y) = val.GetBatch();

                        var pred = Forward(x);

                        var lossVal   = LossFn.Call(pred.Data, y);
                        var metricVal = MetricFn.Call(pred.Data, y);
                        val_losses.Add(TOps.MeanF(lossVal));
                        val_metrics.Add(TOps.MeanF(metricVal));
                    }
                }

                Console.WriteLine("Epoch: {0}, Loss: {1}", iteration, train_losses.Average());
            }
        }
Exemplo n.º 10
0
        public override void Update(int iteration, BaseLayer layer)
        {
            if (DecayRate > 0)
            {
                LearningRate = LearningRate * (1 / 1 + DecayRate * iteration);
            }

            float t    = iteration + 1;
            float lr_t = Convert.ToSingle(LearningRate * Math.Sqrt(1f - Math.Pow(Beta2, t)) / (1f - Math.Pow(Beta1, t)));

            foreach (var item in layer.Params)
            {
                var param = item.Value;

                if (!ms.ContainsKey(param.Name))
                {
                    ms[param.Name] = Tensor.Constant(0, Global.Device, DType.Float32, param.Data.Shape);
                    vs[param.Name] = Tensor.Constant(0, Global.Device, DType.Float32, param.Data.Shape);
                    if (AmsGrad)
                    {
                        vhats[param.Name] = Tensor.Constant(0, Global.Device, DType.Float32, param.Data.Shape);
                    }
                    else
                    {
                        vhats[param.Name] = Tensor.Constant(0, Global.Device, DType.Float32, 1);
                    }

                    var m_t = (Beta1 * ms[param.Name]) + (1 - Beta1) * param.Grad;
                    var v_t = (Beta2 * vs[param.Name]) + (1 - Beta2) * Square(param.Grad);

                    if (AmsGrad)
                    {
                        Tensor vhat_t = TOps.Maximum(vhats[param.Name], v_t);

                        param.Data        = param.Data - lr_t * m_t / (Sqrt(vhat_t) + EPSILON);
                        vhats[param.Name] = vhat_t;
                    }
                    else
                    {
                        param.Data = param.Data - lr_t * m_t / (Sqrt(v_t) + EPSILON);
                    }

                    ms[param.Name] = m_t;
                    vs[param.Name] = v_t;

                    param.ApplyConstraint();
                }
            }
        }
Exemplo n.º 11
0
        static void Main(string[] args)
        {
            Global.UseGpu();

            Tensor x = Tensor.FromArray(Global.Device, new float[] { 1, 2, 3, 4, 5, 6, 7, 8, 9 });

            x = x.Reshape(3, 3);

            var result = TOps.Diag(x);

            result.Print();

            string datasetFolder = @"C:\dataset\MNIST";
            bool   useDenseModel = false;

            var((trainX, trainY), (valX, valY)) = MNISTParser.LoadDataSet(datasetFolder, trainCount: 60000, testCount: 10000, flatten: useDenseModel);
            Console.WriteLine("Train and Test data loaded");
            DataFrameIter trainIter = new DataFrameIter(trainX, trainY);
            DataFrameIter valIter   = new DataFrameIter(valX, valY);

            Sequential model = null;

            if (useDenseModel)
            {
                model = BuildFCModel();
            }
            else
            {
                model = BuildConvModel();
            }

            model.Compile(OptimizerType.Adam, LossType.CategorialCrossEntropy, MetricType.Accuracy);
            Console.WriteLine("Model compiled.. initiating training");

            model.EpochEnd += Model_EpochEnd;

            model.Train(trainIter, 10, 32, valIter);

            Console.ReadLine();
        }