protected void CalculateRegressionResults()
{
IEnumerable <double> estimatedTrainingValues = EstimatedTrainingValues; // cache values
IEnumerable <double> originalTrainingValues = ProblemData.Dataset.GetDoubleValues(ProblemData.TargetVariable, ProblemData.TrainingIndices);
IEnumerable <double> estimatedTestValues = EstimatedTestValues; // cache values
IEnumerable <double> originalTestValues = ProblemData.Dataset.GetDoubleValues(ProblemData.TargetVariable, ProblemData.TestIndices);
OnlineCalculatorError errorState;
double trainingMSE = OnlineMeanSquaredErrorCalculator.Calculate(originalTrainingValues, estimatedTrainingValues, out errorState);
TrainingMeanSquaredError = errorState == OnlineCalculatorError.None ? trainingMSE : double.NaN;
double testMSE = OnlineMeanSquaredErrorCalculator.Calculate(originalTestValues, estimatedTestValues, out errorState);
TestMeanSquaredError = errorState == OnlineCalculatorError.None ? testMSE : double.NaN;
double trainingMAE = OnlineMeanAbsoluteErrorCalculator.Calculate(originalTrainingValues, estimatedTrainingValues, out errorState);
TrainingMeanAbsoluteError = errorState == OnlineCalculatorError.None ? trainingMAE : double.NaN;
double testMAE = OnlineMeanAbsoluteErrorCalculator.Calculate(originalTestValues, estimatedTestValues, out errorState);
TestMeanAbsoluteError = errorState == OnlineCalculatorError.None ? testMAE : double.NaN;
double trainingR = OnlinePearsonsRCalculator.Calculate(originalTrainingValues, estimatedTrainingValues, out errorState);
TrainingRSquared = errorState == OnlineCalculatorError.None ? trainingR * trainingR : double.NaN;
double testR = OnlinePearsonsRCalculator.Calculate(originalTestValues, estimatedTestValues, out errorState);
TestRSquared = errorState == OnlineCalculatorError.None ? testR * testR : double.NaN;
double trainingRelError = OnlineMeanAbsolutePercentageErrorCalculator.Calculate(originalTrainingValues, estimatedTrainingValues, out errorState);
TrainingRelativeError = errorState == OnlineCalculatorError.None ? trainingRelError : double.NaN;
double testRelError = OnlineMeanAbsolutePercentageErrorCalculator.Calculate(originalTestValues, estimatedTestValues, out errorState);
TestRelativeError = errorState == OnlineCalculatorError.None ? testRelError : double.NaN;
double trainingNMSE = OnlineNormalizedMeanSquaredErrorCalculator.Calculate(originalTrainingValues, estimatedTrainingValues, out errorState);
TrainingNormalizedMeanSquaredError = errorState == OnlineCalculatorError.None ? trainingNMSE : double.NaN;
double testNMSE = OnlineNormalizedMeanSquaredErrorCalculator.Calculate(originalTestValues, estimatedTestValues, out errorState);
TestNormalizedMeanSquaredError = errorState == OnlineCalculatorError.None ? testNMSE : double.NaN;
TrainingRootMeanSquaredError = Math.Sqrt(TrainingMeanSquaredError);
TestRootMeanSquaredError = Math.Sqrt(TestMeanSquaredError);
// BackwardsCompatibility3.3
#region Backwards compatible code, remove with 3.5
if (ContainsKey(TrainingMeanErrorResultName))
{
double trainingME = OnlineMeanErrorCalculator.Calculate(originalTrainingValues, estimatedTrainingValues, out errorState);
TrainingMeanError = errorState == OnlineCalculatorError.None ? trainingME : double.NaN;
}
if (ContainsKey(TestMeanErrorResultName))
{
double testME = OnlineMeanErrorCalculator.Calculate(originalTestValues, estimatedTestValues, out errorState);
TestMeanError = errorState == OnlineCalculatorError.None ? testME : double.NaN;
}
#endregion
}
public static double Calculate(IEnumerable <double> first, IEnumerable <double> second, out OnlineCalculatorError errorState)
{
IEnumerator <double> firstEnumerator = first.GetEnumerator();
IEnumerator <double> secondEnumerator = second.GetEnumerator();
var calculator = new OnlinePearsonsRCalculator();
// always move forward both enumerators (do not use short-circuit evaluation!)
while (firstEnumerator.MoveNext() & secondEnumerator.MoveNext())
{
double original = firstEnumerator.Current;
double estimated = secondEnumerator.Current;
calculator.Add(original, estimated);
if (calculator.ErrorState != OnlineCalculatorError.None)
{
break;
}
}
// check if both enumerators are at the end to make sure both enumerations have the same length
if (calculator.ErrorState == OnlineCalculatorError.None &&
(secondEnumerator.MoveNext() || firstEnumerator.MoveNext()))
{
throw new ArgumentException("Number of elements in first and second enumeration doesn't match.");
}
else
{
errorState = calculator.ErrorState;
return(calculator.R);
}
}
protected OnlinePearsonsRCalculator(OnlinePearsonsRCalculator original, Cloner cloner)
: base(original, cloner)
{
covCalculator = cloner.Clone(original.covCalculator);
sxCalculator = cloner.Clone(original.sxCalculator);
syCalculator = cloner.Clone(original.syCalculator);
}
public static IEnumerable <Tuple <string, double> > CalculateImpacts(IRegressionSolution solution,
DataPartitionEnum data = DataPartitionEnum.Training,
ReplacementMethodEnum replacement = ReplacementMethodEnum.Median)
{
var problemData = solution.ProblemData;
var dataset = problemData.Dataset;
IEnumerable <int> rows;
IEnumerable <double> targetValues;
double originalR2 = -1;
OnlineCalculatorError error;
switch (data)
{
case DataPartitionEnum.All:
rows = solution.ProblemData.AllIndices;
targetValues = problemData.TargetVariableValues.ToList();
originalR2 = OnlinePearsonsRCalculator.Calculate(problemData.TargetVariableValues, solution.EstimatedValues, out error);
if (error != OnlineCalculatorError.None)
{
throw new InvalidOperationException("Error during R² calculation.");
}
originalR2 = originalR2 * originalR2;
break;
case DataPartitionEnum.Training:
rows = problemData.TrainingIndices;
targetValues = problemData.TargetVariableTrainingValues.ToList();
originalR2 = solution.TrainingRSquared;
break;
case DataPartitionEnum.Test:
rows = problemData.TestIndices;
targetValues = problemData.TargetVariableTestValues.ToList();
originalR2 = solution.TestRSquared;
break;
default: throw new ArgumentException(string.Format("DataPartition {0} cannot be handled.", data));
}
var impacts = new Dictionary <string, double>();
var modifiableDataset = ((Dataset)dataset).ToModifiable();
foreach (var inputVariable in problemData.AllowedInputVariables)
{
var newEstimates = EvaluateModelWithReplacedVariable(solution.Model, inputVariable, modifiableDataset, rows, replacement);
var newR2 = OnlinePearsonsRCalculator.Calculate(targetValues, newEstimates, out error);
if (error != OnlineCalculatorError.None)
{
throw new InvalidOperationException("Error during R² calculation with replaced inputs.");
}
newR2 = newR2 * newR2;
var impact = originalR2 - newR2;
impacts[inputVariable] = impact;
}
return(impacts.OrderByDescending(i => i.Value).Select(i => Tuple.Create(i.Key, i.Value)));
}
private void CalculateTrainingPrognosisResults()
{
OnlineCalculatorError errorState;
var problemData = Solution.ProblemData;
if (!problemData.TrainingIndices.Any())
{
return;
}
var model = Solution.Model;
//mean model
double trainingMean = problemData.Dataset.GetDoubleValues(problemData.TargetVariable, problemData.TrainingIndices).Average();
var meanModel = new ConstantModel(trainingMean);
//AR1 model
double alpha, beta;
IEnumerable <double> trainingStartValues = problemData.Dataset.GetDoubleValues(problemData.TargetVariable, problemData.TrainingIndices.Select(r => r - 1).Where(r => r > 0)).ToList();
OnlineLinearScalingParameterCalculator.Calculate(problemData.Dataset.GetDoubleValues(problemData.TargetVariable, problemData.TrainingIndices.Where(x => x > 0)), trainingStartValues, out alpha, out beta, out errorState);
var AR1model = new TimeSeriesPrognosisAutoRegressiveModel(problemData.TargetVariable, new double[] { beta }, alpha);
var trainingHorizions = problemData.TrainingIndices.Select(r => Math.Min(trainingHorizon, problemData.TrainingPartition.End - r)).ToList();
IEnumerable <IEnumerable <double> > trainingTargetValues = problemData.TrainingIndices.Zip(trainingHorizions, Enumerable.Range).Select(r => problemData.Dataset.GetDoubleValues(problemData.TargetVariable, r)).ToList();
IEnumerable <IEnumerable <double> > trainingEstimatedValues = model.GetPrognosedValues(problemData.Dataset, problemData.TrainingIndices, trainingHorizions).ToList();
IEnumerable <IEnumerable <double> > trainingMeanModelPredictions = meanModel.GetPrognosedValues(problemData.Dataset, problemData.TrainingIndices, trainingHorizions).ToList();
IEnumerable <IEnumerable <double> > trainingAR1ModelPredictions = AR1model.GetPrognosedValues(problemData.Dataset, problemData.TrainingIndices, trainingHorizions).ToList();
IEnumerable <double> originalTrainingValues = trainingTargetValues.SelectMany(x => x).ToList();
IEnumerable <double> estimatedTrainingValues = trainingEstimatedValues.SelectMany(x => x).ToList();
double trainingMSE = OnlineMeanSquaredErrorCalculator.Calculate(originalTrainingValues, estimatedTrainingValues, out errorState);
PrognosisTrainingMeanSquaredError = errorState == OnlineCalculatorError.None ? trainingMSE : double.NaN;
double trainingMAE = OnlineMeanAbsoluteErrorCalculator.Calculate(originalTrainingValues, estimatedTrainingValues, out errorState);
PrognosisTrainingMeanAbsoluteError = errorState == OnlineCalculatorError.None ? trainingMAE : double.NaN;
double trainingR = OnlinePearsonsRCalculator.Calculate(originalTrainingValues, estimatedTrainingValues, out errorState);
PrognosisTrainingRSquared = errorState == OnlineCalculatorError.None ? trainingR * trainingR : double.NaN;
double trainingRelError = OnlineMeanAbsolutePercentageErrorCalculator.Calculate(originalTrainingValues, estimatedTrainingValues, out errorState);
PrognosisTrainingRelativeError = errorState == OnlineCalculatorError.None ? trainingRelError : double.NaN;
double trainingNMSE = OnlineNormalizedMeanSquaredErrorCalculator.Calculate(originalTrainingValues, estimatedTrainingValues, out errorState);
PrognosisTrainingNormalizedMeanSquaredError = errorState == OnlineCalculatorError.None ? trainingNMSE : double.NaN;
double trainingME = OnlineMeanErrorCalculator.Calculate(originalTrainingValues, estimatedTrainingValues, out errorState);
PrognosisTrainingMeanError = errorState == OnlineCalculatorError.None ? trainingME : double.NaN;
PrognosisTrainingDirectionalSymmetry = OnlineDirectionalSymmetryCalculator.Calculate(trainingStartValues, trainingTargetValues, trainingEstimatedValues, out errorState);
PrognosisTrainingDirectionalSymmetry = errorState == OnlineCalculatorError.None ? PrognosisTrainingDirectionalSymmetry : 0.0;
PrognosisTrainingWeightedDirectionalSymmetry = OnlineWeightedDirectionalSymmetryCalculator.Calculate(trainingStartValues, trainingTargetValues, trainingEstimatedValues, out errorState);
PrognosisTrainingWeightedDirectionalSymmetry = errorState == OnlineCalculatorError.None ? PrognosisTrainingWeightedDirectionalSymmetry : 0.0;
PrognosisTrainingTheilsUStatisticAR1 = OnlineTheilsUStatisticCalculator.Calculate(trainingStartValues, trainingTargetValues, trainingAR1ModelPredictions, trainingEstimatedValues, out errorState);
PrognosisTrainingTheilsUStatisticAR1 = errorState == OnlineCalculatorError.None ? PrognosisTrainingTheilsUStatisticAR1 : double.PositiveInfinity;
PrognosisTrainingTheilsUStatisticMean = OnlineTheilsUStatisticCalculator.Calculate(trainingStartValues, trainingTargetValues, trainingMeanModelPredictions, trainingEstimatedValues, out errorState);
PrognosisTrainingTheilsUStatisticMean = errorState == OnlineCalculatorError.None ? PrognosisTrainingTheilsUStatisticMean : double.PositiveInfinity;
}
public double Calculate(IEnumerable<Tuple<double, double>> values, out OnlineCalculatorError errorState) {
var calculator = new OnlinePearsonsRCalculator();
foreach (var tuple in values) {
calculator.Add(tuple.Item1, tuple.Item2);
if (calculator.ErrorState != OnlineCalculatorError.None) break;
}
errorState = calculator.ErrorState;
return calculator.R;
}
public static double CalculateQuality(IEnumerable <double> targetValues, IEnumerable <double> estimatedValues)
{
OnlineCalculatorError errorState;
var ret = OnlinePearsonsRCalculator.Calculate(targetValues, estimatedValues, out errorState);
if (errorState != OnlineCalculatorError.None)
{
throw new InvalidOperationException("Error during calculation with replaced inputs.");
}
return(ret * ret);
}
public double Calculate(IEnumerable <Tuple <double, double> > values, out OnlineCalculatorError errorState)
{
var calculator = new OnlinePearsonsRCalculator();
foreach (var tuple in values)
{
calculator.Add(tuple.Item1, tuple.Item2);
if (calculator.ErrorState != OnlineCalculatorError.None)
{
break;
}
}
errorState = calculator.ErrorState;
return(calculator.R);
}
public static double Calculate(IEnumerable<double> first, IEnumerable<double> second, out OnlineCalculatorError errorState) {
IEnumerator<double> firstEnumerator = first.GetEnumerator();
IEnumerator<double> secondEnumerator = second.GetEnumerator();
var calculator = new OnlinePearsonsRCalculator();
// always move forward both enumerators (do not use short-circuit evaluation!)
while (firstEnumerator.MoveNext() & secondEnumerator.MoveNext()) {
double original = firstEnumerator.Current;
double estimated = secondEnumerator.Current;
calculator.Add(original, estimated);
if (calculator.ErrorState != OnlineCalculatorError.None) break;
}
// check if both enumerators are at the end to make sure both enumerations have the same length
if (calculator.ErrorState == OnlineCalculatorError.None &&
(secondEnumerator.MoveNext() || firstEnumerator.MoveNext())) {
throw new ArgumentException("Number of elements in first and second enumeration doesn't match.");
} else {
errorState = calculator.ErrorState;
return calculator.R;
}
}
protected void CalculateRegressionResults()
{
double[] estimatedTrainingValues = EstimatedTrainingValues.ToArray(); // cache values
double[] originalTrainingValues = ProblemData.Dataset.GetDoubleValues(ProblemData.TargetVariable, ProblemData.TrainingIndices).ToArray();
double[] estimatedTestValues = EstimatedTestValues.ToArray(); // cache values
double[] originalTestValues = ProblemData.Dataset.GetDoubleValues(ProblemData.TargetVariable, ProblemData.TestIndices).ToArray();
OnlineCalculatorError errorState;
double trainingMSE = OnlineMeanSquaredErrorCalculator.Calculate(originalTrainingValues, estimatedTrainingValues, out errorState);
TrainingMeanSquaredError = errorState == OnlineCalculatorError.None ? trainingMSE : double.NaN;
double testMSE = OnlineMeanSquaredErrorCalculator.Calculate(originalTestValues, estimatedTestValues, out errorState);
TestMeanSquaredError = errorState == OnlineCalculatorError.None ? testMSE : double.NaN;
double trainingR = OnlinePearsonsRCalculator.Calculate(originalTrainingValues, estimatedTrainingValues, out errorState);
TrainingRSquared = errorState == OnlineCalculatorError.None ? trainingR * trainingR : double.NaN;
double testR = OnlinePearsonsRCalculator.Calculate(originalTestValues, estimatedTestValues, out errorState);
TestRSquared = errorState == OnlineCalculatorError.None ? testR * testR : double.NaN;
double trainingNormalizedGini = NormalizedGiniCalculator.Calculate(originalTrainingValues, estimatedTrainingValues, out errorState);
if (errorState != OnlineCalculatorError.None)
{
trainingNormalizedGini = double.NaN;
}
double testNormalizedGini = NormalizedGiniCalculator.Calculate(originalTestValues, estimatedTestValues, out errorState);
if (errorState != OnlineCalculatorError.None)
{
testNormalizedGini = double.NaN;
}
TrainingNormalizedGiniCoefficient = trainingNormalizedGini;
TestNormalizedGiniCoefficient = testNormalizedGini;
}
public static IEnumerable <Tuple <string, double> > CalculateImpacts(
IRegressionSolution solution,
DataPartitionEnum data = DataPartitionEnum.Training,
ReplacementMethodEnum replacementMethod = ReplacementMethodEnum.Median,
FactorReplacementMethodEnum factorReplacementMethod = FactorReplacementMethodEnum.Best)
{
var problemData = solution.ProblemData;
var dataset = problemData.Dataset;
IEnumerable <int> rows;
IEnumerable <double> targetValues;
double originalR2 = -1;
OnlineCalculatorError error;
switch (data)
{
case DataPartitionEnum.All:
rows = solution.ProblemData.AllIndices;
targetValues = problemData.TargetVariableValues.ToList();
originalR2 = OnlinePearsonsRCalculator.Calculate(problemData.TargetVariableValues, solution.EstimatedValues, out error);
if (error != OnlineCalculatorError.None)
{
throw new InvalidOperationException("Error during R² calculation.");
}
originalR2 = originalR2 * originalR2;
break;
case DataPartitionEnum.Training:
rows = problemData.TrainingIndices;
targetValues = problemData.TargetVariableTrainingValues.ToList();
originalR2 = solution.TrainingRSquared;
break;
case DataPartitionEnum.Test:
rows = problemData.TestIndices;
targetValues = problemData.TargetVariableTestValues.ToList();
originalR2 = solution.TestRSquared;
break;
default: throw new ArgumentException(string.Format("DataPartition {0} cannot be handled.", data));
}
var impacts = new Dictionary <string, double>();
var modifiableDataset = ((Dataset)dataset).ToModifiable();
var inputvariables = new HashSet <string>(problemData.AllowedInputVariables.Union(solution.Model.VariablesUsedForPrediction));
var allowedInputVariables = dataset.VariableNames.Where(v => inputvariables.Contains(v)).ToList();
// calculate impacts for double variables
foreach (var inputVariable in allowedInputVariables.Where(problemData.Dataset.VariableHasType <double>))
{
var newEstimates = EvaluateModelWithReplacedVariable(solution.Model, inputVariable, modifiableDataset, rows, replacementMethod);
var newR2 = OnlinePearsonsRCalculator.Calculate(targetValues, newEstimates, out error);
if (error != OnlineCalculatorError.None)
{
throw new InvalidOperationException("Error during R² calculation with replaced inputs.");
}
newR2 = newR2 * newR2;
var impact = originalR2 - newR2;
impacts[inputVariable] = impact;
}
// calculate impacts for string variables
foreach (var inputVariable in allowedInputVariables.Where(problemData.Dataset.VariableHasType <string>))
{
if (factorReplacementMethod == FactorReplacementMethodEnum.Best)
{
// try replacing with all possible values and find the best replacement value
var smallestImpact = double.PositiveInfinity;
foreach (var repl in problemData.Dataset.GetStringValues(inputVariable, rows).Distinct())
{
var newEstimates = EvaluateModelWithReplacedVariable(solution.Model, inputVariable, modifiableDataset, rows,
Enumerable.Repeat(repl, dataset.Rows));
var newR2 = OnlinePearsonsRCalculator.Calculate(targetValues, newEstimates, out error);
if (error != OnlineCalculatorError.None)
{
throw new InvalidOperationException("Error during R² calculation with replaced inputs.");
}
newR2 = newR2 * newR2;
var impact = originalR2 - newR2;
if (impact < smallestImpact)
{
smallestImpact = impact;
}
}
impacts[inputVariable] = smallestImpact;
}
else
{
// for replacement methods shuffle and mode
// calculate impacts for factor variables
var newEstimates = EvaluateModelWithReplacedVariable(solution.Model, inputVariable, modifiableDataset, rows,
factorReplacementMethod);
var newR2 = OnlinePearsonsRCalculator.Calculate(targetValues, newEstimates, out error);
if (error != OnlineCalculatorError.None)
{
throw new InvalidOperationException("Error during R² calculation with replaced inputs.");
}
newR2 = newR2 * newR2;
var impact = originalR2 - newR2;
impacts[inputVariable] = impact;
}
} // foreach
return(impacts.OrderByDescending(i => i.Value).Select(i => Tuple.Create(i.Key, i.Value)));
}
public double Calculate(IEnumerable <double> originalValues, IEnumerable <double> estimatedValues, out OnlineCalculatorError errorState)
{
var r = OnlinePearsonsRCalculator.Calculate(originalValues, estimatedValues, out errorState);
return(r * r);
}
public static double Calculate(IEnumerable <double> first, IEnumerable <double> second, out OnlineCalculatorError errorState)
{
var r = OnlinePearsonsRCalculator.Calculate(first, second, out errorState);
return(r * r);
}
protected OnlinePearsonsRSquaredCalculator(OnlinePearsonsRSquaredCalculator original, Cloner cloner)
: base(original, cloner)
{
rCalculator = cloner.Clone(original.rCalculator);
}