/// <summary>
/// Gets a method which converts a <see cref="Discrete"/> distribution into a point estimate.
/// </summary>
/// <param name="lossFunction"> The <see cref="LossFunction"/>, which determines the loss to minimize.</param>
/// <returns>The point estimator.</returns>
public static Func <Discrete, int> ForDiscrete(LossFunction lossFunction)
{
switch (lossFunction)
{
// Zero-one loss
case LossFunction.ZeroOne:
return(distribution => distribution.GetMode());
// Quadratic loss
case LossFunction.Squared:
return(distribution => Convert.ToInt32(distribution.GetMean()));
// Absolute loss
case LossFunction.Absolute:
return(distribution => distribution.GetMedian());
// Custom loss
case LossFunction.Custom:
throw new InvalidOperationException("Call PointEstimator.ForDiscrete with actual custom loss function instead.");
}
// Should never be reached
Debug.Fail(string.Format("Loss function {0} not supported", lossFunction));
return(null);
}
public Model(LossFunction lossFunction, string name, params LayerBase[] layers)
{
this.layers = layers;
this.layerCount = layers.Length;
this.LossFunc = lossFunction;
this.name = name;
}
public CustomNetworkProperties(ActivationFunction af, ActivationFunction daf, LossFunction lf, float learningRate)
{
_activationFunction = af;
_derivativeActivation = daf;
_lossFunction = lf;
LearningRate = learningRate;
}
public Network(LossFunctionType lossFunctionType, Optimizer optimizer, int numberOfClasses)
{
_numberOfClasses = numberOfClasses;
_lossFunction = LossFunctionFactory.Produce(lossFunctionType);
_layers = new List <IFullLayer>();
_optimizer = optimizer;
}
/// <summary>
/// Gets a method which converts a <see cref="Bernoulli"/> distribution into a point estimate.
/// </summary>
/// <param name="lossFunction"> The <see cref="LossFunction"/>, which determines the loss to minimize.</param>
/// <returns>The point estimator.</returns>
public static Func <Bernoulli, bool> ForBernoulli(LossFunction lossFunction)
{
switch (lossFunction)
{
// Zero-one loss
case LossFunction.ZeroOne:
return(distribution => distribution.GetMode());
// Quadratic loss
case LossFunction.Squared:
return(distribution => distribution.GetMean() >= 0.5);
// Absolute loss
case LossFunction.Absolute:
return(distribution => distribution.GetMode()); // For Bernoulli distributions in Infer.NET, the median equals the mode!
// Custom loss
case LossFunction.Custom:
throw new InvalidOperationException("Call PointEstimator.ForBernoulli with actual custom loss function instead.");
}
// Should never be reached
Debug.Fail(string.Format("Loss function {0} not supported", lossFunction));
return(null);
}
/// <summary>
/// Add a set of evaluation metrics to the set of observations.
/// </summary>
/// <param name="metrics">The observed regression evaluation metric</param>
void IMetricsStatistics <RegressionMetrics> .Add(RegressionMetrics metrics)
{
MeanAbsoluteError.Add(metrics.MeanAbsoluteError);
MeanSquaredError.Add(metrics.MeanSquaredError);
RootMeanSquaredError.Add(metrics.RootMeanSquaredError);
LossFunction.Add(metrics.LossFunction);
RSquared.Add(metrics.RSquared);
}
public NN(Table DInputs, Table DLabels, List <HiddenLayer> HiddenLayers = null,
ActivationFunction Activation = ActivationFunction.ReLU, LossFunction LossFunc = LossFunction.SGD, double LearningRate = .05f, int Lepochs = 1)
{
Inputs = DInputs;
Labels = DLabels;
AF = Activation;
LF = LossFunc;
LR = LearningRate;
epochs = Lepochs;
}
public OutputLayerDense(int outputNumber, LayerDef activationFunction, LossFunction lossFunction)
{
if (!(activationFunction is SoftmaxDef) && lossFunction == LossFunction.CrossEntropy)
{
Debug.LogError("Cross Entropy only support softmax activation for now.");
activationFunction = new SoftmaxDef();
}
HiddenSize = outputNumber;
Activation = activationFunction;
Loss = lossFunction;
ParameterNames = new List <string>();
}
public void PrepareTrain(TensorOld X, TensorOld y)
{
trainInputShape = X.shape;
trainYShape = y.shape;
for (int i = 0; i < TrainingLayers.Count; i++)
{
X = TrainingLayers[i].PrepareTrain(X);
}
LossFunction.PrepareTrain(X, y);
}
public NN(Model AModel, int TrainingSetSize, int InputDim, int OutputDim, double RegularizationStrength,
double LearningRate = .05f, int Lepochs = 1)
{
Inputs = AModel.inputs;
Labels = AModel.labels;
AF = AModel.activationFunction;
LF = AModel.lossFunction;
LR = LearningRate;
epochs = Lepochs;
trainingSize = TrainingSetSize;
inputDimension = InputDim;
outputDimension = OutputDim;
regLambda = RegularizationStrength;
}
public override void SetInspector(GameObject inspector, GameObject signleLineInput, GameObject signleLineSelect)
{
Inspector inspector1 = inspector.GetComponent <Inspector>();
inspector.transform.Find("Quad/NodeName").GetComponent <TextMesh>().text = "Criterion Node";
var lossFunctionObj = Instantiate(signleLineSelect);
inspector1.Add(lossFunctionObj.transform);
lossFunctionObj.GetComponent <ParamSelect>().SetType(typeof(LossFunction), "Loss Function", (int)lossFunction);
inspector1.OnSave = () =>
{
lossFunction = (LossFunction)lossFunctionObj.GetComponent <ParamSelect>().GetValue();
};
}
public void Step(TensorOld X, TensorOld y)
{
CheckTrainShape(X, y);
//正向传播
var yHat = Forward(X);
//计算Loss
LossFunction.Compute(y, yHat);
//反向传播
Backward(LossFunction.BackwardOutput);
//正则化
Regularize();
//更新参数
Optimize();
}
////////////////////////////////////////////////////////////////////////////////////////////////////
/// <summary> Do a learning process with a batch. </summary>
///
/// <param name="functionStack"> Stack of functions. This cannot be null. </param>
/// <param name="input"> The input. This may be null. </param>
/// <param name="teach"> The teach. This may be null. </param>
/// <param name="lossFunction"> The loss function. This cannot be null. </param>
/// <param name="isUpdate"> (Optional) True if this object is update. </param>
///
/// <returns> A Real. </returns>
////////////////////////////////////////////////////////////////////////////////////////////////////
public static Real Train([NotNull] SortedFunctionStack functionStack, [CanBeNull] NdArray input,
[CanBeNull] NdArray teach, [NotNull] LossFunction lossFunction, bool isUpdate = true,
bool verbose = true)
{
if (verbose)
{
RILogManager.Default?.EnterMethod("Training " + functionStack.Name);
}
// for preserving the error of the result
if (verbose)
{
RILogManager.Default?.SendDebug("Forward propagation");
}
NdArray[] result = functionStack.Forward(verbose, input);
if (verbose)
{
RILogManager.Default?.SendDebug("Evaluating loss");
}
Real sumLoss = lossFunction.Evaluate(result, teach);
// Run Backward batch
if (verbose)
{
RILogManager.Default?.SendDebug("Backward propagation");
}
functionStack.Backward(verbose, result);
if (isUpdate)
{
if (verbose)
{
RILogManager.Default?.SendDebug("Updating stack");
}
functionStack.Update();
}
if (verbose)
{
RILogManager.Default?.ExitMethod("Training " + functionStack.Name);
RILogManager.Default?.ViewerSendWatch("Local Loss", sumLoss.ToString(), sumLoss);
}
return(sumLoss);
}
public Model(Table DInputs, Table DLabels, List <HiddenLayer> HiddenLayerStructure, int OutputDim, double RegularizationLambda, int PassCount, double LearningRate, bool PrintLoss
, ActivationFunction MActivationFunction, LossFunction MLossFunction)
{
inputs = DInputs;
labels = DLabels;
hiddenLayers = HiddenLayerStructure;
lossFunction = MLossFunction;
activationFunction = MActivationFunction;
var examplesCount = DInputs.tableStructure.Ys;
Rand.SetRandomSeed(0);
weights.Add(Table.Random(hiddenLayers[0].neurons, examplesCount) / Math.Sqrt(examplesCount)); //SQRT?
biases.Add(Table.Zeros(1, hiddenLayers[0].neurons));
weights.Add(Table.Random(OutputDim, hiddenLayers[0].neurons) / Math.Sqrt(hiddenLayers[0].neurons));
biases.Add(Table.Zeros(1, OutputDim));
regLambda = RegularizationLambda;
passCount = PassCount;
epsilon = LearningRate;
printLoss = PrintLoss;
}
/// <summary>
/// 3Layer Backpropagation Class
/// </summary>
/// <param name="inputWeight">Input Weight.</param>
/// <param name="outputWeight">Output Weight.</param>
/// <param name="hiddenLayer">Hidden layer.</param>
/// <param name="outputLayer">Output layer.</param>
/// <param name="hiddenLogisticFunc">Hidden Layer Logistic Function. Default = SigmoidFunc</param>
/// <param name="outputLogisticFunc">Output Layer Logistic Function. Default = SigmoidFunc</param>
/// <param name="lossFunc">Output Layer Loss Function. Default = MSE</param>
/// <param name="learnRate">Learn rate. Default = 0.01</param>
public BackPropagation(Matrix inputWeight, Matrix outputWeight, Matrix hiddenLayer, Matrix outputLayer,
LogisticFunctions hiddenLogisticFunc = LogisticFunctions.Sigmoid,
LogisticFunctions outputLogisticFunc = LogisticFunctions.Sigmoid,
LossFunctions lossFunc = LossFunctions.MSE, double learnRate = 0.01)
{
_inputWeight = inputWeight;
_outputWeight = outputWeight;
_hiddenLayer = hiddenLayer;
_outputLayer = outputLayer;
HiddenLogisticFunc = hiddenLogisticFunc;
OutputLogisticFunc = outputLogisticFunc;
LossFunc = lossFunc;
_hiddenLogisticFunc = GetLogisticFunction(hiddenLogisticFunc);
_outputLogisticFunc = GetLogisticFunction(outputLogisticFunc);
_lossFunc = GetLossFunction(lossFunc);
LearnRate = learnRate;
}
/// <summary>
/// Sets the loss function which determines how a prediction in the form of a distribution is converted into a point prediction.
/// </summary>
/// <param name="lossFunction">The loss function.</param>
/// <param name="customLossFunction">
/// An optional custom loss function. This can only be set when <paramref name="lossFunction"/> is set to 'Custom'.
/// The custom loss function returns the loss incurred when choosing an estimate instead of the true value,
/// where the first argument is the true value and the second argument is the estimate of the true value.
/// </param>
public void SetPredictionLossFunction(LossFunction lossFunction, Func <TLabel, TLabel, double> customLossFunction = null)
{
if (lossFunction == LossFunction.Custom)
{
if (customLossFunction == null)
{
throw new ArgumentNullException("customLossFunction");
}
}
else
{
if (customLossFunction != null)
{
throw new InvalidOperationException("Loss function must be set to '" + LossFunction.Custom + "' when providing a custom loss function.");
}
}
this.customLossFunction = customLossFunction;
this.lossFunction = lossFunction;
}
public Population<T> Optimize(Population<T> population, out IEnumerable<PerformanceResult<T>> ranking)
{
int decimatedPopulationSize = (int)(population.Size * SurvivalPercentage);
int emptyPopulationSpace = population.Size - decimatedPopulationSize;
ranking = population.Entities
.Select(x => new PerformanceResult<T>(x, LossFunction.CalculateLoss(x)))
.OrderBy(x => x.Loss);
var performances = ranking
.Take(decimatedPopulationSize)
.ToList();
var totalErrorOfSurvived = performances.Select(x => x.Loss).Sum();
var newPopulationEntities = new List<T>(population.Size);
//generate the new population, prioritise according to ~ 1 / error
foreach (var performance in performances)
{
newPopulationEntities.Add(performance.Entity);
double offsetPercentage = (1 - performance.Loss / totalErrorOfSurvived) / (decimatedPopulationSize - 1);
int offsetAmount = (int)(offsetPercentage * emptyPopulationSpace);
for(int i = 0; i < offsetAmount; i++)
{
newPopulationEntities.Add((T)performance.Entity.Mutate());
}
}
//fill the rest of the population beginning from the baddest survivor
var restPopulationSpace = population.Size - newPopulationEntities.Count;
for(int i = 0; i < restPopulationSpace; i++)
{
newPopulationEntities.Add((T)performances[i % performances.Count()].Entity.Mutate());
}
return new Population<T>(newPopulationEntities, population.Size);
}
/// <summary>
/// Initializes a new instance of the <see cref="BayesPointMachineClassifierPredictionSettings{TLabel}"/> class.
/// </summary>
protected BayesPointMachineClassifierPredictionSettings()
{
this.lossFunction = LossFunctionDefault; // Classification error
this.customLossFunction = null;
}
public void FinishBuilding(LossFunction lossFunction)
{
this.lossFunction = lossFunction;
}
/// <summary>
/// Initializes a new instance of the <see cref="MatchboxRecommenderPredictionSettings"/> class.
/// </summary>
public MatchboxRecommenderPredictionSettings()
{
this.lossFunction = LossFunctionDefault;
this.customLossFunction = null;
}
/// <summary>
/// Calculate the loss/error of the output nodes.
/// </summary>
/// <param name="layer">the output layer</param>
/// <returns>a vector containing the errors</returns>
public static Vector OutputLoss(Layer layer, Vector desired, LossFunction f)
{
return(new Vector(VectorMath.ElementWiseOperation(layer.Nodes.Values, desired.Values, (a, b) => f(a, b))));
}
/// <summary>
/// Create a <see cref="CustomNetworkProperties"/> instance.
/// </summary>
/// <param name="af"></param>
/// <param name="lf"></param>
/// <returns></returns>
public static NetworkProperties Create(ActivationFunction af, ActivationFunction daf, LossFunction lf, float learningRate)
{
return(new CustomNetworkProperties(af, daf, lf, learningRate));
}
protected override void ApplyLossFunction(ref float score, float label, ref double loss)
{
loss = LossFunction.Loss(score, label);
}
protected override void BeginProcessing()
{
if (Context == null)
{
Context = Context.CurrentContext;
}
var stopWatch = new Stopwatch();
stopWatch.Start();
for (var epoch = 1; epoch <= MaxEpoch; ++epoch)
{
TrainingData.Reset();
var totalLoss = 0.0f;
var dataSize = 0;
while (!TrainingData.End())
{
var batch = TrainingData.Next();
var data = batch.Data[0].AsInContext(Context);
var label = batch.Label[0].AsInContext(Context);
using (Autograd.Record())
{
var output = Model.Call(data);
var loss = (NDArray)LossFunction.Call(output, label);
loss.Backward();
totalLoss += loss.Sum();
dataSize += data.Shape[0];
Trainer.Step(batch.Data[0].Shape[0]);
}
}
if (epoch % DisplayStep == 0 || epoch == MaxEpoch)
{
totalLoss /= dataSize;
ValidationData.Reset();
var totalValidLoss = 0.0f;
var validDataSize = 0;
if (MetricFunction != null)
{
MetricFunction.Reset();
}
while (!ValidationData.End())
{
var batch = ValidationData.Next();
var data = batch.Data[0].AsInContext(Context);
var label = batch.Label[0].AsInContext(Context);
var output = Model.Call(data);
var validLoss = (NDArray)LossFunction.Call(output, label);
totalValidLoss += validLoss.Sum();
validDataSize += data.Shape[0];
if (MetricFunction != null)
{
MetricFunction.Update(label, output);
}
}
totalValidLoss /= validDataSize;
string metricName = null;
float metric = float.NaN;
if (MetricFunction != null)
{
(metricName, metric) = MetricFunction.Get();
}
var status = new TrainingStatus(epoch,
(float)Math.Round(totalLoss, DisplayDigits),
(float)Math.Round(totalValidLoss, DisplayDigits),
metricName,
(float)Math.Round(metric, DisplayDigits),
stopWatch.Elapsed);
WriteObject(status);
}
}
}
////////////////////////////////////////////////////////////////////////////////////////////////////
/// <summary> Do a learning process with a batch. </summary>
///
/// <param name="functionStack"> Stack of functions. </param>
/// <param name="input"> The input. </param>
/// <param name="teach"> The teach. </param>
/// <param name="lossFunction"> The loss function. </param>
/// <param name="isUpdate"> (Optional) True if this object is update. </param>
///
/// <returns> A Real. </returns>
////////////////////////////////////////////////////////////////////////////////////////////////////
public static Real Train([CanBeNull] FunctionStack functionStack, [NotNull] Array[] input, [NotNull] Array[] teach, [CanBeNull] LossFunction lossFunction, bool isUpdate = true)
{
return(Train(functionStack, NdArray.FromArrays(input), NdArray.FromArrays(teach), lossFunction, isUpdate));
}
//バッチで学習処理を行う
public static Real Train(FunctionStack functionStack, Array[] input, Array[] teach, LossFunction lossFunction, bool isUpdate = true)
{
return(Train(functionStack, NdArray.FromArrays(input), NdArray.FromArrays(teach), lossFunction, isUpdate));
}
//バッチで学習処理を行う
public static Real Train(FunctionStack functionStack, NdArray input, NdArray teach, LossFunction lossFunction, bool isUpdate = true)
{
//結果の誤差保存用
NdArray[] result = functionStack.Forward(input);
Real sumLoss = lossFunction.Evaluate(result, teach);
//Backwardのバッチを実行
functionStack.Backward(result);
//更新
if (isUpdate)
{
functionStack.Update();
}
return(sumLoss);
}
protected override void ApplyLossFunction(ref VBuffer <float> score, float label, ref VBuffer <Double> loss)
{
VBufferUtils.PairManipulator <Float, Double> lossFn =
(int slot, Float src, ref Double dst) => dst = LossFunction.Loss(src, label);
VBufferUtils.ApplyWith(ref score, ref loss, lossFn);
}
public NeuralNetwork UseLossFunction(LossFunction loss)
{
LossFunction = loss;
return(this);
}
/// <summary>
/// Gets a method which converts a <see cref="Discrete"/> distribution into a point estimate.
/// </summary>
/// <typeparam name="TLabel">The type of a label.</typeparam>
/// <param name="lossFunction"> The <see cref="LossFunction"/>, which determines the loss to minimize.</param>
/// <returns>The point estimator.</returns>
public static Func <IDictionary <TLabel, double>, TLabel> ForDiscrete <TLabel>(LossFunction lossFunction)
{
switch (lossFunction)
{
// Zero-one loss
case LossFunction.ZeroOne:
return(GetMode);
// Quadratic loss
case LossFunction.Squared:
throw new InvalidOperationException("Call PointEstimator.ForDiscrete with actual custom loss function instead.");
// Absolute loss
case LossFunction.Absolute:
throw new InvalidOperationException("Call PointEstimator.ForDiscrete with actual custom loss function instead.");
// Custom loss
case LossFunction.Custom:
throw new InvalidOperationException("Call PointEstimator.ForDiscrete with actual custom loss function instead.");
}
// Should never be reached
Debug.Fail(string.Format("Loss function {0} not supported", lossFunction));
return(null);
}
