private void computeInformation(double[][] inputData, double[] outputData, double[] weights)
{
// Store model information
#pragma warning disable 612, 618
result = regression.Compute(inputData);
#pragma warning restore 612, 618
if (weights == null)
{
this.deviance = regression.GetDeviance(inputData, outputData);
this.logLikelihood = regression.GetLogLikelihood(inputData, outputData);
this.chiSquare = regression.ChiSquare(inputData, outputData);
}
else
{
this.deviance = regression.GetDeviance(inputData, outputData, weights);
this.logLikelihood = regression.GetLogLikelihood(inputData, outputData, weights);
this.chiSquare = regression.ChiSquare(inputData, outputData, weights);
}
// Store coefficient information
for (int i = 0; i < regression.Coefficients.Length; i++)
{
this.standardErrors[i] = regression.StandardErrors[i];
this.waldTests[i] = regression.GetWaldTest(i);
this.coefficients[i] = regression.Coefficients[i];
this.confidences[i] = regression.GetConfidenceInterval(i);
this.oddsRatios[i] = regression.GetOddsRatio(i);
}
}
private static double[] finiteDifferences(double[][] input, double[] output, bool stochastic)
{
LogisticRegression regression;
LogisticGradientDescent teacher;
regression = new LogisticRegression(inputs: 2);
teacher = new LogisticGradientDescent(regression)
{
Stochastic = stochastic,
LearningRate = 1e-4,
};
FiniteDifferences diff = new FiniteDifferences(3);
diff.Function = (x) =>
{
for (int i = 0; i < x.Length; i++)
{
regression.Coefficients[i] = x[i];
}
return(regression.GetLogLikelihood(input, output));
};
return(diff.Compute(regression.Coefficients));
}
#pragma warning restore 612, 618
private void computeInner(double[][] inputData, double[] outputData, double[] weights)
{
for (int i = 0; i < NumberOfInputs; i++)
{
// Create a diminished inner model without the current variable
double[][] data = inputData.RemoveColumn(i);
// Perform likelihood-ratio tests against diminished nested models
var innerModel = new LogisticRegression()
{
NumberOfInputs = NumberOfInputs - 1
};
var learning = new IterativeReweightedLeastSquares(innerModel)
{
Iterations = iterations,
Tolerance = tolerance,
Regularization = regularization
};
learning.Learn(data, outputData, weights);
double ratio = 2.0 * (logLikelihood - innerModel.GetLogLikelihood(data, outputData));
ratioTests[i + 1] = new ChiSquareTest(ratio, 1);
}
innerComputed = true;
}
private void computeInner()
{
double limit = tolerance;
int maxIterations = iterations;
// Perform likelihood-ratio tests against diminished nested models
var innerModel = new LogisticRegression(inputCount - 1);
var learning = new IterativeReweightedLeastSquares(innerModel);
learning.Regularization = regularization;
for (int i = 0; i < inputCount; i++)
{
// Create a diminished inner model without the current variable
double[][] data = inputData.RemoveColumn(i);
int iteration = 0;
double delta = 0;
do // learning iterations until convergence
{
delta = learning.Run(data, outputData, weights);
iteration++;
} while (delta > limit && iteration < maxIterations);
double ratio = 2.0 * (logLikelihood - innerModel.GetLogLikelihood(data, outputData));
ratioTests[i + 1] = new ChiSquareTest(ratio, 1);
}
innerComputed = true;
}
/// <summary>
/// Computes the Logistic Regression Analysis.
/// </summary>
/// <remarks>The likelihood surface for the
/// logistic regression learning is convex, so there will be only one
/// peak. Any local maxima will be also a global maxima.
/// </remarks>
/// <param name="limit">
/// The difference between two iterations of the regression algorithm
/// when the algorithm should stop. If not specified, the value of
/// 10e-4 will be used. The difference is calculated based on the largest
/// absolute parameter change of the regression.
/// </param>
/// <param name="maxIterations">
/// The maximum number of iterations to be performed by the regression
/// algorithm.
/// </param>
/// <returns>
/// True if the model converged, false otherwise.
/// </returns>
///
public bool Compute(double limit, int maxIterations)
{
double delta;
int iteration = 0;
do // learning iterations until convergence
{
delta = regression.Regress(inputData, outputData);
iteration++;
} while (delta > limit && iteration < maxIterations);
// Check if the full model has converged
bool converged = iteration <= maxIterations;
// Store model information
this.result = regression.Compute(inputData);
this.deviance = regression.GetDeviance(inputData, outputData);
this.logLikelihood = regression.GetLogLikelihood(inputData, outputData);
this.chiSquare = regression.ChiSquare(inputData, outputData);
// Store coefficient information
for (int i = 0; i < regression.Coefficients.Length; i++)
{
this.waldTests[i] = regression.GetWaldTest(i);
this.standardErrors[i] = regression.GetStandardError(i);
this.coefficients[i] = regression.Coefficients[i];
this.confidences[i] = regression.GetConfidenceInterval(i);
this.oddsRatios[i] = regression.GetOddsRatio(i);
}
// Perform likelihood-ratio tests against diminished nested models
for (int i = 0; i < inputCount; i++)
{
// Create a diminished inner model without the current variable
double[][] data = inputData.RemoveColumn(i);
LogisticRegression inner = new LogisticRegression(inputCount - 1);
iteration = 0;
do // learning iterations until convergence
{
delta = inner.Regress(data, outputData);
iteration++;
} while (delta > limit && iteration < maxIterations);
double ratio = 2.0 * (logLikelihood - inner.GetLogLikelihood(data, outputData));
ratioTests[i + 1] = new ChiSquareTest(ratio, 1);
}
// Returns true if the full model has converged, false otherwise.
return(converged);
}
private static double[] finiteDifferences(double[][] input, double[] output, bool stochastic)
{
var regression = new LogisticRegression(inputs: 2);
var diff = new FiniteDifferences(3);
diff.Function = (x) =>
{
for (int i = 0; i < x.Length; i++)
{
regression.Coefficients[i] = x[i];
}
return(regression.GetLogLikelihood(input, output));
};
return(diff.Compute(regression.Coefficients));
}
private void computeInformation()
{
// Store model information
this.result = regression.Compute(inputData);
this.deviance = regression.GetDeviance(inputData, outputData);
this.logLikelihood = regression.GetLogLikelihood(inputData, outputData);
this.chiSquare = regression.ChiSquare(inputData, outputData);
// Store coefficient information
for (int i = 0; i < regression.Coefficients.Length; i++)
{
this.standardErrors[i] = regression.StandardErrors[i];
this.waldTests[i] = regression.GetWaldTest(i);
this.coefficients[i] = regression.Coefficients[i];
this.confidences[i] = regression.GetConfidenceInterval(i);
this.oddsRatios[i] = regression.GetOddsRatio(i);
}
}
/// <summary>
/// Computes one step of the Stepwise Logistic Regression Analysis.
/// </summary>
/// <returns>
/// Returns the index of the variable discarded in the step or -1
/// in case no variable could be discarded.
/// </returns>
///
public int DoStep()
{
ChiSquareTest[] tests = null;
// Check if we are performing the first step
if (currentModel == null)
{
// This is the first step. We should create the full model.
int inputCount = inputData[0].Length;
LogisticRegression regression = new LogisticRegression(inputCount);
int[] variables = Matrix.Indices(0, inputCount);
fit(regression, inputData, outputData);
ChiSquareTest test = regression.ChiSquare(inputData, outputData);
fullLikelihood = regression.GetLogLikelihood(inputData, outputData);
if (Double.IsNaN(fullLikelihood))
{
throw new ConvergenceException(
"Perfect separation detected. Please rethink the use of logistic regression.");
}
tests = new ChiSquareTest[regression.Coefficients.Length];
currentModel = new StepwiseLogisticRegressionModel(this, regression, variables, test, tests);
completeModel = currentModel;
}
// Verify first if a variable reduction is possible
if (currentModel.Regression.Inputs == 1)
{
return(-1); // cannot reduce further
}
// Now go and create the diminished nested models
var nestedModels = new StepwiseLogisticRegressionModel[currentModel.Regression.Inputs];
for (int i = 0; i < nestedModels.Length; i++)
{
// Create a diminished nested model without the current variable
LogisticRegression regression = new LogisticRegression(currentModel.Regression.Inputs - 1);
int[] variables = currentModel.Variables.RemoveAt(i);
double[][] subset = inputData.Submatrix(0, inputData.Length - 1, variables);
fit(regression, subset, outputData);
// Check the significance of the nested model
double logLikelihood = regression.GetLogLikelihood(subset, outputData);
double ratio = 2.0 * (fullLikelihood - logLikelihood);
ChiSquareTest test = new ChiSquareTest(ratio, inputNames.Length - variables.Length)
{
Size = threshold
};
if (tests != null)
{
tests[i + 1] = test;
}
// Store the nested model
nestedModels[i] = new StepwiseLogisticRegressionModel(this, regression, variables, test, null);
}
// Select the model with the highest p-value
double pmax = 0; int imax = -1;
for (int i = 0; i < nestedModels.Length; i++)
{
if (nestedModels[i].ChiSquare.PValue >= pmax)
{
imax = i;
pmax = nestedModels[i].ChiSquare.PValue;
}
}
// Create the read-only nested model collection
this.nestedModelCollection = new StepwiseLogisticRegressionModelCollection(nestedModels);
// If the model with highest p-value is not significant,
if (imax >= 0 && pmax > threshold)
{
// Then this means the variable can be safely discarded from the full model
int removed = currentModel.Variables[imax];
// Our diminished nested model will become our next full model.
this.currentModel = nestedModels[imax];
// Finally, return the index of the removed variable
return(removed);
}
else
{
// Else we can not safely remove any variable from the model.
return(-1);
}
}