Exemple #1
0
        /// <summary>
        /// Gets a method which converts a <see cref="Discrete"/> distribution into a point estimate.
        /// </summary>
        /// <param name="lossFunction"> The <see cref="LossFunction"/>, which determines the loss to minimize.</param>
        /// <returns>The point estimator.</returns>
        public static Func <Discrete, int> ForDiscrete(LossFunction lossFunction)
        {
            switch (lossFunction)
            {
            // Zero-one loss
            case LossFunction.ZeroOne:
                return(distribution => distribution.GetMode());

            // Quadratic loss
            case LossFunction.Squared:
                return(distribution => Convert.ToInt32(distribution.GetMean()));

            // Absolute loss
            case LossFunction.Absolute:
                return(distribution => distribution.GetMedian());

            // Custom loss
            case LossFunction.Custom:
                throw new InvalidOperationException("Call PointEstimator.ForDiscrete with actual custom loss function instead.");
            }

            // Should never be reached
            Debug.Fail(string.Format("Loss function {0} not supported", lossFunction));
            return(null);
        }
 public Model(LossFunction lossFunction, string name, params LayerBase[] layers)
 {
     this.layers     = layers;
     this.layerCount = layers.Length;
     this.LossFunc   = lossFunction;
     this.name       = name;
 }
Exemple #3
0
 public CustomNetworkProperties(ActivationFunction af, ActivationFunction daf, LossFunction lf, float learningRate)
 {
     _activationFunction   = af;
     _derivativeActivation = daf;
     _lossFunction         = lf;
     LearningRate          = learningRate;
 }
Exemple #4
0
 public Network(LossFunctionType lossFunctionType, Optimizer optimizer, int numberOfClasses)
 {
     _numberOfClasses = numberOfClasses;
     _lossFunction    = LossFunctionFactory.Produce(lossFunctionType);
     _layers          = new List <IFullLayer>();
     _optimizer       = optimizer;
 }
Exemple #5
0
        /// <summary>
        /// Gets a method which converts a <see cref="Bernoulli"/> distribution into a point estimate.
        /// </summary>
        /// <param name="lossFunction"> The <see cref="LossFunction"/>, which determines the loss to minimize.</param>
        /// <returns>The point estimator.</returns>
        public static Func <Bernoulli, bool> ForBernoulli(LossFunction lossFunction)
        {
            switch (lossFunction)
            {
            // Zero-one loss
            case LossFunction.ZeroOne:
                return(distribution => distribution.GetMode());

            // Quadratic loss
            case LossFunction.Squared:
                return(distribution => distribution.GetMean() >= 0.5);

            // Absolute loss
            case LossFunction.Absolute:
                return(distribution => distribution.GetMode());    // For Bernoulli distributions in Infer.NET, the median equals the mode!

            // Custom loss
            case LossFunction.Custom:
                throw new InvalidOperationException("Call PointEstimator.ForBernoulli with actual custom loss function instead.");
            }

            // Should never be reached
            Debug.Fail(string.Format("Loss function {0} not supported", lossFunction));
            return(null);
        }
Exemple #6
0
 /// <summary>
 /// Add a set of evaluation metrics to the set of observations.
 /// </summary>
 /// <param name="metrics">The observed regression evaluation metric</param>
 void IMetricsStatistics <RegressionMetrics> .Add(RegressionMetrics metrics)
 {
     MeanAbsoluteError.Add(metrics.MeanAbsoluteError);
     MeanSquaredError.Add(metrics.MeanSquaredError);
     RootMeanSquaredError.Add(metrics.RootMeanSquaredError);
     LossFunction.Add(metrics.LossFunction);
     RSquared.Add(metrics.RSquared);
 }
Exemple #7
0
 public NN(Table DInputs, Table DLabels, List <HiddenLayer> HiddenLayers = null,
           ActivationFunction Activation = ActivationFunction.ReLU, LossFunction LossFunc = LossFunction.SGD, double LearningRate = .05f, int Lepochs = 1)
 {
     Inputs = DInputs;
     Labels = DLabels;
     AF     = Activation;
     LF     = LossFunc;
     LR     = LearningRate;
     epochs = Lepochs;
 }
 public OutputLayerDense(int outputNumber, LayerDef activationFunction, LossFunction lossFunction)
 {
     if (!(activationFunction is SoftmaxDef) && lossFunction == LossFunction.CrossEntropy)
     {
         Debug.LogError("Cross Entropy only support softmax activation for now.");
         activationFunction = new SoftmaxDef();
     }
     HiddenSize     = outputNumber;
     Activation     = activationFunction;
     Loss           = lossFunction;
     ParameterNames = new List <string>();
 }
        public void PrepareTrain(TensorOld X, TensorOld y)
        {
            trainInputShape = X.shape;
            trainYShape     = y.shape;

            for (int i = 0; i < TrainingLayers.Count; i++)
            {
                X = TrainingLayers[i].PrepareTrain(X);
            }

            LossFunction.PrepareTrain(X, y);
        }
Exemple #10
0
 public NN(Model AModel, int TrainingSetSize, int InputDim, int OutputDim, double RegularizationStrength,
           double LearningRate = .05f, int Lepochs = 1)
 {
     Inputs          = AModel.inputs;
     Labels          = AModel.labels;
     AF              = AModel.activationFunction;
     LF              = AModel.lossFunction;
     LR              = LearningRate;
     epochs          = Lepochs;
     trainingSize    = TrainingSetSize;
     inputDimension  = InputDim;
     outputDimension = OutputDim;
     regLambda       = RegularizationStrength;
 }
    public override void SetInspector(GameObject inspector, GameObject signleLineInput, GameObject signleLineSelect)
    {
        Inspector inspector1 = inspector.GetComponent <Inspector>();

        inspector.transform.Find("Quad/NodeName").GetComponent <TextMesh>().text = "Criterion Node";

        var lossFunctionObj = Instantiate(signleLineSelect);

        inspector1.Add(lossFunctionObj.transform);
        lossFunctionObj.GetComponent <ParamSelect>().SetType(typeof(LossFunction), "Loss Function", (int)lossFunction);

        inspector1.OnSave = () =>
        {
            lossFunction = (LossFunction)lossFunctionObj.GetComponent <ParamSelect>().GetValue();
        };
    }
        public void Step(TensorOld X, TensorOld y)
        {
            CheckTrainShape(X, y);

            //正向传播
            var yHat = Forward(X);

            //计算Loss
            LossFunction.Compute(y, yHat);
            //反向传播
            Backward(LossFunction.BackwardOutput);
            //正则化
            Regularize();
            //更新参数
            Optimize();
        }
Exemple #13
0
        ////////////////////////////////////////////////////////////////////////////////////////////////////
        /// <summary>   Do a learning process with a batch. </summary>
        ///
        /// <param name="functionStack">    Stack of functions. This cannot be null. </param>
        /// <param name="input">            The input. This may be null. </param>
        /// <param name="teach">            The teach. This may be null. </param>
        /// <param name="lossFunction">     The loss function. This cannot be null. </param>
        /// <param name="isUpdate">         (Optional) True if this object is update. </param>
        ///
        /// <returns>   A Real. </returns>
        ////////////////////////////////////////////////////////////////////////////////////////////////////

        public static Real Train([NotNull] SortedFunctionStack functionStack, [CanBeNull] NdArray input,
                                 [CanBeNull] NdArray teach, [NotNull] LossFunction lossFunction, bool isUpdate = true,
                                 bool verbose = true)
        {
            if (verbose)
            {
                RILogManager.Default?.EnterMethod("Training " + functionStack.Name);
            }
            // for preserving the error of the result
            if (verbose)
            {
                RILogManager.Default?.SendDebug("Forward propagation");
            }
            NdArray[] result = functionStack.Forward(verbose, input);
            if (verbose)
            {
                RILogManager.Default?.SendDebug("Evaluating loss");
            }
            Real sumLoss = lossFunction.Evaluate(result, teach);

            // Run Backward batch
            if (verbose)
            {
                RILogManager.Default?.SendDebug("Backward propagation");
            }
            functionStack.Backward(verbose, result);

            if (isUpdate)
            {
                if (verbose)
                {
                    RILogManager.Default?.SendDebug("Updating stack");
                }
                functionStack.Update();
            }

            if (verbose)
            {
                RILogManager.Default?.ExitMethod("Training " + functionStack.Name);
                RILogManager.Default?.ViewerSendWatch("Local Loss", sumLoss.ToString(), sumLoss);
            }

            return(sumLoss);
        }
Exemple #14
0
            public Model(Table DInputs, Table DLabels, List <HiddenLayer> HiddenLayerStructure, int OutputDim, double RegularizationLambda, int PassCount, double LearningRate, bool PrintLoss
                         , ActivationFunction MActivationFunction, LossFunction MLossFunction)
            {
                inputs             = DInputs;
                labels             = DLabels;
                hiddenLayers       = HiddenLayerStructure;
                lossFunction       = MLossFunction;
                activationFunction = MActivationFunction;
                var examplesCount = DInputs.tableStructure.Ys;

                Rand.SetRandomSeed(0);
                weights.Add(Table.Random(hiddenLayers[0].neurons, examplesCount) / Math.Sqrt(examplesCount)); //SQRT?
                biases.Add(Table.Zeros(1, hiddenLayers[0].neurons));
                weights.Add(Table.Random(OutputDim, hiddenLayers[0].neurons) / Math.Sqrt(hiddenLayers[0].neurons));
                biases.Add(Table.Zeros(1, OutputDim));
                regLambda = RegularizationLambda;
                passCount = PassCount;
                epsilon   = LearningRate;
                printLoss = PrintLoss;
            }
        /// <summary>
        /// 3Layer Backpropagation Class
        /// </summary>
        /// <param name="inputWeight">Input Weight.</param>
        /// <param name="outputWeight">Output Weight.</param>
        /// <param name="hiddenLayer">Hidden layer.</param>
        /// <param name="outputLayer">Output layer.</param>
        /// <param name="hiddenLogisticFunc">Hidden Layer Logistic Function. Default = SigmoidFunc</param>
        /// <param name="outputLogisticFunc">Output Layer Logistic Function. Default = SigmoidFunc</param>
        /// <param name="lossFunc">Output Layer Loss Function. Default = MSE</param>
        /// <param name="learnRate">Learn rate. Default = 0.01</param>
        public BackPropagation(Matrix inputWeight, Matrix outputWeight, Matrix hiddenLayer, Matrix outputLayer,
                               LogisticFunctions hiddenLogisticFunc = LogisticFunctions.Sigmoid,
                               LogisticFunctions outputLogisticFunc = LogisticFunctions.Sigmoid,
                               LossFunctions lossFunc = LossFunctions.MSE, double learnRate = 0.01)
        {
            _inputWeight  = inputWeight;
            _outputWeight = outputWeight;
            _hiddenLayer  = hiddenLayer;
            _outputLayer  = outputLayer;

            HiddenLogisticFunc = hiddenLogisticFunc;
            OutputLogisticFunc = outputLogisticFunc;
            LossFunc           = lossFunc;

            _hiddenLogisticFunc = GetLogisticFunction(hiddenLogisticFunc);
            _outputLogisticFunc = GetLogisticFunction(outputLogisticFunc);
            _lossFunc           = GetLossFunction(lossFunc);

            LearnRate = learnRate;
        }
Exemple #16
0
        /// <summary>
        /// Sets the loss function which determines how a prediction in the form of a distribution is converted into a point prediction.
        /// </summary>
        /// <param name="lossFunction">The loss function.</param>
        /// <param name="customLossFunction">
        /// An optional custom loss function. This can only be set when <paramref name="lossFunction"/> is set to 'Custom'.
        /// The custom loss function returns the loss incurred when choosing an estimate instead of the true value,
        /// where the first argument is the true value and the second argument is the estimate of the true value.
        /// </param>
        public void SetPredictionLossFunction(LossFunction lossFunction, Func <TLabel, TLabel, double> customLossFunction = null)
        {
            if (lossFunction == LossFunction.Custom)
            {
                if (customLossFunction == null)
                {
                    throw new ArgumentNullException("customLossFunction");
                }
            }
            else
            {
                if (customLossFunction != null)
                {
                    throw new InvalidOperationException("Loss function must be set to '" + LossFunction.Custom + "' when providing a custom loss function.");
                }
            }

            this.customLossFunction = customLossFunction;
            this.lossFunction       = lossFunction;
        }
Exemple #17
0
        public Population<T> Optimize(Population<T> population, out IEnumerable<PerformanceResult<T>> ranking)
        {
            int decimatedPopulationSize = (int)(population.Size * SurvivalPercentage);
            int emptyPopulationSpace = population.Size - decimatedPopulationSize;

            ranking = population.Entities
                .Select(x => new PerformanceResult<T>(x, LossFunction.CalculateLoss(x)))
                .OrderBy(x => x.Loss);

            var performances = ranking
                .Take(decimatedPopulationSize)
                .ToList();

            var totalErrorOfSurvived = performances.Select(x => x.Loss).Sum();

            var newPopulationEntities = new List<T>(population.Size);

            //generate the new population, prioritise according to ~ 1 / error
            foreach (var performance in performances)
            {
                newPopulationEntities.Add(performance.Entity);

                double offsetPercentage = (1 - performance.Loss / totalErrorOfSurvived) / (decimatedPopulationSize - 1);
                int offsetAmount = (int)(offsetPercentage * emptyPopulationSpace);

                for(int i = 0; i < offsetAmount; i++)
                {
                    newPopulationEntities.Add((T)performance.Entity.Mutate());
                }
            }

            //fill the rest of the population beginning from the baddest survivor
            var restPopulationSpace = population.Size - newPopulationEntities.Count;
            for(int i = 0; i < restPopulationSpace; i++)
            {
                newPopulationEntities.Add((T)performances[i % performances.Count()].Entity.Mutate());
            }

            return new Population<T>(newPopulationEntities, population.Size);
        }
Exemple #18
0
 /// <summary>
 /// Initializes a new instance of the <see cref="BayesPointMachineClassifierPredictionSettings{TLabel}"/> class.
 /// </summary>
 protected BayesPointMachineClassifierPredictionSettings()
 {
     this.lossFunction       = LossFunctionDefault; // Classification error
     this.customLossFunction = null;
 }
 public void FinishBuilding(LossFunction lossFunction)
 {
     this.lossFunction = lossFunction;
 }
Exemple #20
0
 /// <summary>
 /// Initializes a new instance of the <see cref="MatchboxRecommenderPredictionSettings"/> class.
 /// </summary>
 public MatchboxRecommenderPredictionSettings()
 {
     this.lossFunction       = LossFunctionDefault;
     this.customLossFunction = null;
 }
 /// <summary>
 /// Calculate the loss/error of the output nodes.
 /// </summary>
 /// <param name="layer">the output layer</param>
 /// <returns>a vector containing the errors</returns>
 public static Vector OutputLoss(Layer layer, Vector desired, LossFunction f)
 {
     return(new Vector(VectorMath.ElementWiseOperation(layer.Nodes.Values, desired.Values, (a, b) => f(a, b))));
 }
Exemple #22
0
 /// <summary>
 /// Create a <see cref="CustomNetworkProperties"/> instance.
 /// </summary>
 /// <param name="af"></param>
 /// <param name="lf"></param>
 /// <returns></returns>
 public static NetworkProperties Create(ActivationFunction af, ActivationFunction daf, LossFunction lf, float learningRate)
 {
     return(new CustomNetworkProperties(af, daf, lf, learningRate));
 }
Exemple #23
0
 protected override void ApplyLossFunction(ref float score, float label, ref double loss)
 {
     loss = LossFunction.Loss(score, label);
 }
        protected override void BeginProcessing()
        {
            if (Context == null)
            {
                Context = Context.CurrentContext;
            }

            var stopWatch = new Stopwatch();

            stopWatch.Start();

            for (var epoch = 1; epoch <= MaxEpoch; ++epoch)
            {
                TrainingData.Reset();
                var totalLoss = 0.0f;
                var dataSize  = 0;

                while (!TrainingData.End())
                {
                    var batch = TrainingData.Next();
                    var data  = batch.Data[0].AsInContext(Context);
                    var label = batch.Label[0].AsInContext(Context);

                    using (Autograd.Record())
                    {
                        var output = Model.Call(data);
                        var loss   = (NDArray)LossFunction.Call(output, label);
                        loss.Backward();
                        totalLoss += loss.Sum();
                        dataSize  += data.Shape[0];

                        Trainer.Step(batch.Data[0].Shape[0]);
                    }
                }

                if (epoch % DisplayStep == 0 || epoch == MaxEpoch)
                {
                    totalLoss /= dataSize;

                    ValidationData.Reset();
                    var totalValidLoss = 0.0f;
                    var validDataSize  = 0;

                    if (MetricFunction != null)
                    {
                        MetricFunction.Reset();
                    }

                    while (!ValidationData.End())
                    {
                        var batch = ValidationData.Next();
                        var data  = batch.Data[0].AsInContext(Context);
                        var label = batch.Label[0].AsInContext(Context);

                        var output    = Model.Call(data);
                        var validLoss = (NDArray)LossFunction.Call(output, label);
                        totalValidLoss += validLoss.Sum();
                        validDataSize  += data.Shape[0];

                        if (MetricFunction != null)
                        {
                            MetricFunction.Update(label, output);
                        }
                    }
                    totalValidLoss /= validDataSize;

                    string metricName = null;
                    float  metric     = float.NaN;

                    if (MetricFunction != null)
                    {
                        (metricName, metric) = MetricFunction.Get();
                    }

                    var status = new TrainingStatus(epoch,
                                                    (float)Math.Round(totalLoss, DisplayDigits),
                                                    (float)Math.Round(totalValidLoss, DisplayDigits),
                                                    metricName,
                                                    (float)Math.Round(metric, DisplayDigits),
                                                    stopWatch.Elapsed);

                    WriteObject(status);
                }
            }
        }
Exemple #25
0
        ////////////////////////////////////////////////////////////////////////////////////////////////////
        /// <summary>   Do a learning process with a batch. </summary>
        ///
        /// <param name="functionStack">    Stack of functions. </param>
        /// <param name="input">            The input. </param>
        /// <param name="teach">            The teach. </param>
        /// <param name="lossFunction">     The loss function. </param>
        /// <param name="isUpdate">         (Optional) True if this object is update. </param>
        ///
        /// <returns>   A Real. </returns>
        ////////////////////////////////////////////////////////////////////////////////////////////////////

        public static Real Train([CanBeNull] FunctionStack functionStack, [NotNull] Array[] input, [NotNull] Array[] teach, [CanBeNull] LossFunction lossFunction, bool isUpdate = true)
        {
            return(Train(functionStack, NdArray.FromArrays(input), NdArray.FromArrays(teach), lossFunction, isUpdate));
        }
Exemple #26
0
 //バッチで学習処理を行う
 public static Real Train(FunctionStack functionStack, Array[] input, Array[] teach, LossFunction lossFunction, bool isUpdate = true)
 {
     return(Train(functionStack, NdArray.FromArrays(input), NdArray.FromArrays(teach), lossFunction, isUpdate));
 }
Exemple #27
0
        //バッチで学習処理を行う
        public static Real Train(FunctionStack functionStack, NdArray input, NdArray teach, LossFunction lossFunction, bool isUpdate = true)
        {
            //結果の誤差保存用
            NdArray[] result  = functionStack.Forward(input);
            Real      sumLoss = lossFunction.Evaluate(result, teach);

            //Backwardのバッチを実行
            functionStack.Backward(result);

            //更新
            if (isUpdate)
            {
                functionStack.Update();
            }

            return(sumLoss);
        }
 protected override void ApplyLossFunction(ref VBuffer <float> score, float label, ref VBuffer <Double> loss)
 {
     VBufferUtils.PairManipulator <Float, Double> lossFn =
         (int slot, Float src, ref Double dst) => dst = LossFunction.Loss(src, label);
     VBufferUtils.ApplyWith(ref score, ref loss, lossFn);
 }
 public NeuralNetwork UseLossFunction(LossFunction loss)
 {
     LossFunction = loss;
     return(this);
 }
Exemple #30
0
        /// <summary>
        /// Gets a method which converts a <see cref="Discrete"/> distribution into a point estimate.
        /// </summary>
        /// <typeparam name="TLabel">The type of a label.</typeparam>
        /// <param name="lossFunction"> The <see cref="LossFunction"/>, which determines the loss to minimize.</param>
        /// <returns>The point estimator.</returns>
        public static Func <IDictionary <TLabel, double>, TLabel> ForDiscrete <TLabel>(LossFunction lossFunction)
        {
            switch (lossFunction)
            {
            // Zero-one loss
            case LossFunction.ZeroOne:
                return(GetMode);

            // Quadratic loss
            case LossFunction.Squared:
                throw new InvalidOperationException("Call PointEstimator.ForDiscrete with actual custom loss function instead.");

            // Absolute loss
            case LossFunction.Absolute:
                throw new InvalidOperationException("Call PointEstimator.ForDiscrete with actual custom loss function instead.");

            // Custom loss
            case LossFunction.Custom:
                throw new InvalidOperationException("Call PointEstimator.ForDiscrete with actual custom loss function instead.");
            }

            // Should never be reached
            Debug.Fail(string.Format("Loss function {0} not supported", lossFunction));
            return(null);
        }