コード例 #1
0
ファイル: Updater.cs プロジェクト: fcmai/brightwire
        public override void Update(IMatrix biasDelta, IMatrix weightDelta, ITrainingContext context)
        {
            var t = context.CurrentEpoch;

            using (var deltaSquared = weightDelta.PointwiseMultiply(weightDelta)) {
                _cache.AddInPlace(weightDelta, _decay, 1 - _decay);
                _cache2.AddInPlace(deltaSquared, _decay2, 1 - _decay2);

                using (var mb = _cache.Clone())
                    using (var vb = _cache2.Clone()) {
                        mb.Multiply(1f / (1f - Convert.ToSingle(Math.Pow(_decay, t))));
                        vb.Multiply(1f / (1f - Convert.ToSingle(Math.Pow(_decay2, t))));
                        using (var vbSqrt = vb.Sqrt(1e-8f))
                            using (var delta = mb.PointwiseDivide(vbSqrt)) {
                                _layerUpdater.Update(biasDelta, delta, context);
                            }
                    }
            }
        }
コード例 #2
0
            public IMatrix Execute(IMatrix curr, ITrainingContext context, bool calculateOutput, INeuralNetworkUpdateAccumulator updateAccumulator)
            {
                using (var ad = _activation.Derivative(_output, curr)) {
                    // clip the gradient
                    ad.Constrain(-1f, 1f);

                    var delta        = curr.PointwiseMultiply(ad);
                    var prevDelta    = delta.TransposeAndMultiply(_memoryUpdater.Layer.Weight);
                    var memoryUpdate = _memory.TransposeThisAndMultiply(delta);
                    var inputUpdate  = _input.TransposeThisAndMultiply(delta);

                    //_input.Dispose();
                    _memory.Dispose();
                    _output.Dispose();
                    updateAccumulator.Record(_memoryUpdater, delta, memoryUpdate);
                    updateAccumulator.Record(_inputUpdater, delta.Clone(), inputUpdate);

                    return(prevDelta);
                }
            }
コード例 #3
0
        protected bool _CalculateTestScore(ITrainingContext context, float[] forwardMemory, float[] backwardMemory, ISequentialTrainingDataProvider data, INeuralNetworkBidirectionalBatchTrainer network, IRecurrentTrainingContext recurrentContext, ref double bestScore, ref BidirectionalNetwork output)
        {
            bool flag        = false;
            var  score       = _GetScore(data, network, forwardMemory, backwardMemory, recurrentContext);
            var  errorMetric = recurrentContext.TrainingContext.ErrorMetric;

            if ((errorMetric.HigherIsBetter && score > bestScore) || (!errorMetric.HigherIsBetter && score < bestScore))
            {
                bestScore            = score;
                output               = network.NetworkInfo;
                output.ForwardMemory = new FloatArray {
                    Data = forwardMemory
                };
                output.BackwardMemory = new FloatArray {
                    Data = backwardMemory
                };
                flag = true;
            }
            context.WriteScore(score, errorMetric.DisplayAsPercentage, flag);
            return(flag);
        }
コード例 #4
0
        public ParallelTreeTrainingOperation(
            Random random,
            ITrainingContext <F, S> trainingContext,
            TrainingParameters parameters,
            int maxThreads,
            IDataPointCollection data,
            ProgressWriter progress)
        {
            data_            = data;
            trainingContext_ = trainingContext;
            parameters_      = parameters;

            maxThreads_ = maxThreads;

            random_ = random;

            progress_ = progress;

            parentStatistics_ = trainingContext_.GetStatisticsAggregator();

            leftChildStatistics_  = trainingContext_.GetStatisticsAggregator();
            rightChildStatistics_ = trainingContext_.GetStatisticsAggregator();

            responses_ = new float[data.Count()];

            indices_ = new int[data.Count()];
            for (int i = 0; i < indices_.Length; i++)
            {
                indices_[i] = i;
            }

            threadLocals_ = new ThreadLocalData[maxThreads_];
            for (int threadIndex = 0; threadIndex < maxThreads_; threadIndex++)
            {
                threadLocals_[threadIndex] = new ThreadLocalData(random_, trainingContext_, parameters_, data_);
            }
        }
コード例 #5
0
        /// <summary>
        /// Train a new decision tree given some training data and a training
        /// problem described by an ITrainingContext instance.
        /// </summary>
        /// <param name="random">The single random number generator.</param>
        /// <param name="progress">Progress reporting target.</param>
        /// <param name="context">The ITrainingContext instance by which
        /// the training framework interacts with the training data.
        /// Implemented within client code.</param>
        /// <param name="parameters">Training parameters.</param>
        /// <param name="maxThreads">The maximum number of threads to use.</param>
        /// <param name="data">The training data.</param>
        /// <returns>A new decision tree.</returns>
        static public Tree <F, S> TrainTree(
            Random random,
            ITrainingContext <F, S> context,
            TrainingParameters parameters,
            int maxThreads,
            IDataPointCollection data,
            ProgressWriter progress = null)
        {
            if (progress == null)
            {
                progress = new ProgressWriter(Verbosity.Interest, Console.Out);
            }

            ParallelTreeTrainingOperation <F, S> trainingOperation = new ParallelTreeTrainingOperation <F, S>(
                random, context, parameters, maxThreads, data, progress);

            Tree <F, S> tree = new Tree <F, S>(parameters.MaxDecisionLevels);

            trainingOperation.TrainNodesRecurse(tree.nodes_, 0, 0, data.Count(), 0); // will recurse until termination criterion is met

            tree.CheckValid();

            return(tree);
        }
コード例 #6
0
        private void OnEpochComplete(ITrainingContext context)
        {
            var score       = _GetScore(context);
            var flag        = false;
            var errorMetric = context.ErrorMetric;

            if ((errorMetric.HigherIsBetter && score > _bestScore) || (!errorMetric.HigherIsBetter && score < _bestScore))
            {
                _bestScore  = score;
                _bestOutput = _trainer.NetworkInfo;
                flag        = true;
            }
            if ((context.CurrentEpoch % _reportCadence) == 0)
            {
                context.WriteScore(score, errorMetric.DisplayAsPercentage, flag);
            }

            if (flag)
            {
                using (var stream = new FileStream(_dataFile, FileMode.Create, FileAccess.Write))
                    Serializer.Serialize(stream, _bestOutput);
                _noChange = 0;
            }
            else
            {
                ++_noChange;
            }

            if (_autoAdjustOnNoChangeCount.HasValue && _noChange >= _autoAdjustOnNoChangeCount.Value)
            {
                _noChange = 0;
                ApplyBestParams();
                context.ReduceTrainingRate();
                Console.WriteLine("Reducing training rate to " + context.TrainingRate);
            }
        }
コード例 #7
0
        public IMatrix Backpropagate(IMatrix input, IMatrix output, IMatrix errorSignal, ITrainingContext context, bool calculateOutput, INeuralNetworkUpdateAccumulator updates = null)
        {
            IMatrix ret = null;

            // calculate the derivative to determine the gradients
            using (var od = _layerUpdater.Layer.Activation?.Derivative(output, errorSignal) ?? output) {
                if (_verifyDerivatives)
                {
                    var dError = _VerifyDerivatives(od);
                    Debug.Assert(dError < 0.001f);
                }

                // use the gradients to determine the direction of each change
                var delta = errorSignal.PointwiseMultiply(od);
                if (calculateOutput)
                {
                    ret = _layerUpdater.Layer.CalculateErrorSignal(delta);
                }

                // update the layer
                _UpdateLayer(input, delta, context, updates);
            }
            return(ret);
        }
コード例 #8
0
 public RecurrentContext(ILinearAlgebraProvider lap, ITrainingContext trainingContext)
 {
     _lap             = lap;
     _trainingContext = trainingContext;
 }
コード例 #9
0
 public IMatrix Execute(IMatrix error, ITrainingContext context, bool calculateOutput, INeuralNetworkUpdateAccumulator updateAccumulator)
 {
     //using (var input = _input)
     using (var output = _output)
         return(_trainer.Backpropagate(_input, output, error, context, calculateOutput, updateAccumulator));
 }
コード例 #10
0
ファイル: Updater.cs プロジェクト: fcmai/brightwire
        public override void Update(IMatrix biasDelta, IMatrix weightDelta, ITrainingContext context)
        {
            var l2 = 1.0f - (context.TrainingRate * _lambda / context.TrainingSamples);

            Update(biasDelta, weightDelta, l2, context);
        }
コード例 #11
0
ファイル: UpdateAccumulator.cs プロジェクト: fcmai/brightwire
 public UpdateAccumulator(ITrainingContext context)
 {
     _context = context;
 }
コード例 #12
0
ファイル: Updater.cs プロジェクト: fcmai/brightwire
 public override void Update(IMatrix biasDelta, IMatrix weightDelta, ITrainingContext context)
 {
     _cache.AddInPlace(weightDelta, _momentum);
     _layerUpdater.Update(biasDelta, _cache, context);
 }
コード例 #13
0
ファイル: Updater.cs プロジェクト: fcmai/brightwire
 public virtual void Update(IMatrix biasDelta, IMatrix weightDelta, ITrainingContext context)
 {
     Update(biasDelta, weightDelta, 1, context);
 }
コード例 #14
0
ファイル: Updater.cs プロジェクト: fcmai/brightwire
 public abstract void Update(IMatrix biasDelta, IMatrix weightDelta, ITrainingContext context);
コード例 #15
0
 public TrainingController(ITrainingContext context)
 {
     dbContext = context;
 }
コード例 #16
0
 public float CalculateCost(ITrainingDataProvider data, ITrainingContext trainingContext)
 {
     return(Execute(data).Select(r => trainingContext.ErrorMetric.Compute(r.Output, r.ExpectedOutput)).Average());
 }
コード例 #17
0
        public void Train(ISequentialTrainingDataProvider trainingData, int numEpochs, ITrainingContext context, IRecurrentTrainingContext recurrentContext = null)
        {
            if (recurrentContext == null)
            {
                recurrentContext = new RecurrentContext(_trainer.LinearAlgebraProvider, context);
            }

            _bestScore = _GetScore(_testData, _trainer, _memory, recurrentContext);
            Console.WriteLine(context.ErrorMetric.DisplayAsPercentage ? "Initial score: {0:P}" : "Initial score: {0}", _bestScore);

            _bestOutput = null;
            recurrentContext.TrainingContext.RecurrentEpochComplete += OnEpochComplete;
            _memory = _trainer.Train(trainingData, _memory, numEpochs, recurrentContext);
            recurrentContext.TrainingContext.RecurrentEpochComplete -= OnEpochComplete;

            // ensure best values are current
            ApplyBestParams();
        }
コード例 #18
0
ファイル: Updater.cs プロジェクト: fcmai/brightwire
 protected void Update(IMatrix biasDelta, IMatrix weightDelta, float weightCoefficient, ITrainingContext context)
 {
     _layer.Update(biasDelta, weightDelta, weightCoefficient, context.TrainingRate / context.MiniBatchSize);
 }
コード例 #19
0
 double _GetScore(ITrainingContext context)
 {
     return(Math.Abs(_trainer.Execute(_testData).Average(d => context.ErrorMetric.Compute(d.Output, d.ExpectedOutput))));
 }
コード例 #20
0
 public IRecurrentTrainingContext CreateRecurrentTrainingContext(ITrainingContext trainingContext)
 {
     return(new RecurrentContext(_lap, trainingContext));
 }