/// <summary>
/// Runs a single pass of the gradient descent algorithm.
/// </summary>
///
public double Run(double[] input, double output)
{
// Initial definitions and memory allocations
double[] coefficients = regression.Coefficients;
this.previous = (double[])coefficients.Clone();
// 1. Compute local gradient estimate
double actual = regression.Compute(input);
double error = output - actual;
gradient[0] = error;
for (int i = 0; i < input.Length; i++)
{
gradient[i + 1] = input[i] * error;
}
// 2. Update using the local estimate
for (int i = 0; i < coefficients.Length; i++)
{
coefficients[i] += rate * gradient[i];
}
// 3. Return maximum parameter change
for (int i = 0; i < previous.Length; i++)
{
deltas[i] = Math.Abs(coefficients[i] - previous[i]) / Math.Abs(previous[i]);
}
return(Matrix.Max(deltas));
}
private void computeInformation(double[][] inputData, double[] outputData, double[] weights)
{
// Store model information
#pragma warning disable 612, 618
result = regression.Compute(inputData);
#pragma warning restore 612, 618
if (weights == null)
{
this.deviance = regression.GetDeviance(inputData, outputData);
this.logLikelihood = regression.GetLogLikelihood(inputData, outputData);
this.chiSquare = regression.ChiSquare(inputData, outputData);
}
else
{
this.deviance = regression.GetDeviance(inputData, outputData, weights);
this.logLikelihood = regression.GetLogLikelihood(inputData, outputData, weights);
this.chiSquare = regression.ChiSquare(inputData, outputData, weights);
}
// Store coefficient information
for (int i = 0; i < regression.Coefficients.Length; i++)
{
this.standardErrors[i] = regression.StandardErrors[i];
this.waldTests[i] = regression.GetWaldTest(i);
this.coefficients[i] = regression.Coefficients[i];
this.confidences[i] = regression.GetConfidenceInterval(i);
this.oddsRatios[i] = regression.GetOddsRatio(i);
}
}
public void OneVariableLogisticRegressionRegularizedTest(double lambda)
{
var revenuePerPopulationFilePath = FileManagerTest.GetTestDataByName("ResultExamsAdmission.txt");
var reader = new CsvReader <ResultExamsAdmission>();
var revenuePerPopulationRows = reader.GetData(revenuePerPopulationFilePath, ",");
var dataMatrix = new Matrix(revenuePerPopulationRows.Select(x => new ResultExamsAdmissionRow(x)));
var dataY = dataMatrix.Columns[0].ToMatrix();
var dataX = dataMatrix.ToMatrix(new int[2] {
1, 2
});
var logisticRegression = new LogisticRegression();
var logisticRegressionParameters = new LogisticRegressionParameters
{
IterationCount = 100,
ThetaInit = new Matrix(new double[3, 1] {
{ 0 }, { 0 }, { 0 }
}),
X = dataX,
Y = dataY,
Lambda = lambda
};
var output = logisticRegression.Compute(logisticRegressionParameters);
var expectedOutput = new Matrix(new double[3, 1] {
{ -25.052148050018360 }, { 0.205354461994741 }, { 0.200583555605940 }
});
AssertMatrixAreEqual(expectedOutput, output.Theta);
}
public void OneVariableLogisticRegressionTest()
{
var revenuePerPopulationFilePath = FileManagerTest.GetTestDataByName("ResultExamsAdmission.txt");
var reader = new CsvReader <ResultExamsAdmission>();
var revenuePerPopulationRows = reader.GetData(revenuePerPopulationFilePath, ",");
var dataMatrix = new Matrix(revenuePerPopulationRows.Select(x => new ResultExamsAdmissionRow(x)));
var dataY = dataMatrix.Columns[0].ToMatrix();
var dataX = dataMatrix.ToMatrix(new int[2] {
1, 2
});
var logisticRegression = new LogisticRegression();
var logisticRegressionParameters = new LogisticRegressionParameters
{
IterationCount = 100,
ThetaInit = new Matrix(new double[3, 1] {
{ 0 }, { 0 }, { 0 }
}),
X = dataX,
Y = dataY
};
var output = logisticRegression.Compute(logisticRegressionParameters);
var expectedOutput = new Matrix(new double[3, 1] {
{ -25.161333566639530 }, { 0.206231713293983 }, { 0.201471600441963 }
});
AssertMatrixAreEqual(expectedOutput, output.Theta);
}
/// <summary>
/// Computes the Logistic Regression Analysis.
/// </summary>
/// <remarks>The likelihood surface for the
/// logistic regression learning is convex, so there will be only one
/// peak. Any local maxima will be also a global maxima.
/// </remarks>
/// <param name="limit">
/// The difference between two iterations of the regression algorithm
/// when the algorithm should stop. If not specified, the value of
/// 10e-4 will be used. The difference is calculated based on the largest
/// absolute parameter change of the regression.
/// </param>
/// <param name="maxIterations">
/// The maximum number of iterations to be performed by the regression
/// algorithm.
/// </param>
/// <returns>
/// True if the model converged, false otherwise.
/// </returns>
///
public bool Compute(double limit, int maxIterations)
{
double delta;
int iteration = 0;
do // learning iterations until convergence
{
delta = regression.Regress(inputData, outputData);
iteration++;
} while (delta > limit && iteration < maxIterations);
// Check if the full model has converged
bool converged = iteration <= maxIterations;
// Store model information
this.result = regression.Compute(inputData);
this.deviance = regression.GetDeviance(inputData, outputData);
this.logLikelihood = regression.GetLogLikelihood(inputData, outputData);
this.chiSquare = regression.ChiSquare(inputData, outputData);
// Store coefficient information
for (int i = 0; i < regression.Coefficients.Length; i++)
{
this.waldTests[i] = regression.GetWaldTest(i);
this.standardErrors[i] = regression.GetStandardError(i);
this.coefficients[i] = regression.Coefficients[i];
this.confidences[i] = regression.GetConfidenceInterval(i);
this.oddsRatios[i] = regression.GetOddsRatio(i);
}
// Perform likelihood-ratio tests against diminished nested models
for (int i = 0; i < inputCount; i++)
{
// Create a diminished inner model without the current variable
double[][] data = inputData.RemoveColumn(i);
LogisticRegression inner = new LogisticRegression(inputCount - 1);
iteration = 0;
do // learning iterations until convergence
{
delta = inner.Regress(data, outputData);
iteration++;
} while (delta > limit && iteration < maxIterations);
double ratio = 2.0 * (logLikelihood - inner.GetLogLikelihood(data, outputData));
ratioTests[i + 1] = new ChiSquareTest(ratio, 1);
}
// Returns true if the full model has converged, false otherwise.
return(converged);
}
static void Main(string[] args)
{
if (args.Length != 2)
{
Console.WriteLine("TestLogisticRegression [train corpus file name] [test corpus file name]");
return;
}
double[][] inputs;
double[] outputs;
Console.WriteLine("Loading train corpus...");
//Load training corpus
LoadCorpusFromFile(args[0], out inputs, out outputs);
//Try to normalize output value
Normalize(outputs);
Console.WriteLine("Logistic regression...");
LogisticRegression lr = new LogisticRegression(inputs[0].Length);
double error = lr.Regress(inputs, outputs);
Console.WriteLine("Parameter list:");
for (int i = 0; i < lr.Coefficients.Length; i++)
{
Console.WriteLine("Coefficient {0}: {1}", i, lr.Coefficients[i]);
}
Console.WriteLine("Delta: {0}", error);
Console.WriteLine();
//Load test corpus
Console.WriteLine("Testing regress result:");
LoadCorpusFromFile(args[1], out inputs, out outputs);
//Try to normalize output value
Normalize(outputs);
for (int i = 0; i < outputs.Length; i++)
{
StringBuilder sb = new StringBuilder();
double output = lr.Compute(inputs[i]);
for (int j = 0; j < inputs[i].Length; j++)
{
sb.Append(inputs[i][j].ToString() + " ");
}
sb.Append(outputs[i] + " RV:" + output);
sb.Append(" Err:" + (output - outputs[i]).ToString());
Console.WriteLine(sb.ToString());
}
}
private void computeInformation()
{
// Store model information
this.result = regression.Compute(inputData);
this.deviance = regression.GetDeviance(inputData, outputData);
this.logLikelihood = regression.GetLogLikelihood(inputData, outputData);
this.chiSquare = regression.ChiSquare(inputData, outputData);
// Store coefficient information
for (int i = 0; i < regression.Coefficients.Length; i++)
{
this.standardErrors[i] = regression.StandardErrors[i];
this.waldTests[i] = regression.GetWaldTest(i);
this.coefficients[i] = regression.Coefficients[i];
this.confidences[i] = regression.GetConfidenceInterval(i);
this.oddsRatios[i] = regression.GetOddsRatio(i);
}
}
public void When_Compute_Logistic_Regression()
{
double[][] inputs =
{
new double[] { 55, 0 },
new double[] { 28, 0 },
new double[] { 65, 1 },
new double[] { 46, 0 },
new double[] { 86, 1 },
new double[] { 56, 1 },
new double[] { 85, 0 },
new double[] { 33, 0 },
new double[] { 21, 1 },
new double[] { 42, 1 }
};
double[] outputs =
{
0, 0, 0, 1, 1, 1, 0, 0, 0, 1
};
var logisticRegression = new LogisticRegression();
logisticRegression.Regress(inputs, outputs);
var result = logisticRegression.Compute(new double[] { 87, 1 });
var oddsRatio = logisticRegression.GetOddsRatio();
var standardErrors = logisticRegression.GetStandardErrors();
Assert.Equal(0.75143272858390264, result);
Assert.Equal(0.085627701183141239, oddsRatio[0]);
Assert.Equal(1.0208597029292656, oddsRatio[1]);
Assert.Equal(5.8584748981778869, oddsRatio[2]);
Assert.Equal(2.1590686019476122, standardErrors[0]);
Assert.Equal(0.0337904223210436, standardErrors[1]);
Assert.Equal(1.4729903935788495, standardErrors[2]);
}
public void LargeCoefficientsTest()
{
double[,] data =
{
{ 48, 1, 4.40, 0 },
{ 60, 0, 7.89, 1 },
{ 51, 0, 3.48, 0 },
{ 66, 0, 8.41, 1 },
{ 40, 1, 3.05, 0 },
{ 44, 1, 4.56, 0 },
{ 80, 0, 6.91, 1 },
{ 52, 0, 5.69, 0 },
{ 58, 0, 4.01, 0 },
{ 58, 0, 4.48, 0 },
{ 72, 1, 5.97, 0 },
{ 57, 0, 6.71, 1 },
{ 55, 1, 5.36, 0 },
{ 71, 0, 5.68, 0 },
{ 44, 1, 4.61, 0 },
{ 65, 1, 4.80, 0 },
{ 38, 0, 5.06, 0 },
{ 50, 0, 6.40, 0 },
{ 80, 0, 6.67, 1 },
{ 69, 1, 5.79, 0 },
{ 39, 0, 5.42, 0 },
{ 68, 0, 7.61, 1 },
{ 47, 1, 3.24, 0 },
{ 45, 1, 4.29, 0 },
{ 79, 1, 7.44, 1 },
{ 41, 1, 4.60, 0 },
{ 45, 0, 5.91, 0 },
{ 54, 0, 4.77, 0 },
{ 43, 1, 5.62, 0 },
{ 62, 1, 7.92, 1 },
{ 72, 1, 7.92, 1 },
{ 57, 1, 6.19, 0 },
{ 39, 1, 2.37, 0 },
{ 51, 0, 5.84, 0 },
{ 73, 1, 5.94, 0 },
{ 41, 1, 3.82, 0 },
{ 35, 0, 2.35, 0 },
{ 69, 0, 6.57, 1 },
{ 75, 1, 7.96, 1 },
{ 51, 1, 3.96, 0 },
{ 61, 1, 4.36, 0 },
{ 55, 0, 3.84, 0 },
{ 45, 1, 3.02, 0 },
{ 48, 0, 4.65, 0 },
{ 77, 0, 7.93, 1 },
{ 40, 1, 2.46, 0 },
{ 37, 1, 2.32, 0 },
{ 78, 0, 7.88, 1 },
{ 39, 1, 4.55, 0 },
{ 41, 0, 2.45, 0 },
{ 54, 1, 5.62, 0 },
{ 59, 1, 5.03, 0 },
{ 78, 0, 8.08, 1 },
{ 56, 1, 6.96, 1 },
{ 49, 1, 3.07, 0 },
{ 48, 0, 4.75, 0 },
{ 63, 1, 5.64, 0 },
{ 50, 0, 3.35, 0 },
{ 59, 1, 5.08, 0 },
{ 60, 0, 6.58, 1 },
{ 64, 0, 5.19, 0 },
{ 76, 1, 6.69, 1 },
{ 58, 0, 5.18, 0 },
{ 48, 1, 4.47, 0 },
{ 72, 0, 8.70, 1 },
{ 40, 1, 5.14, 0 },
{ 53, 0, 3.40, 0 },
{ 79, 0, 9.77, 1 },
{ 61, 1, 7.79, 1 },
{ 59, 0, 7.42, 1 },
{ 44, 0, 2.55, 0 },
{ 52, 1, 3.71, 0 },
{ 80, 1, 7.56, 1 },
{ 76, 0, 7.80, 1 },
{ 51, 0, 5.94, 0 },
{ 46, 1, 5.52, 0 },
{ 48, 0, 3.25, 0 },
{ 58, 1, 4.71, 0 },
{ 44, 1, 2.52, 0 },
{ 68, 0, 8.38, 1 },
};
double[][] input = data.Submatrix(null, 0, 2).ToJagged();
double[] output = data.GetColumn(3);
var regression = new LogisticRegression(3);
var teacher = new IterativeReweightedLeastSquares(regression);
teacher.Regularization = 1e-10;
var errors = new List <double>();
for (int i = 0; i < 1000; i++)
{
errors.Add(teacher.Run(input, output));
}
double error = 0;
for (int i = 0; i < output.Length; i++)
{
double expected = output[i];
double actual = System.Math.Round(regression.Compute(input[i]));
if (expected != actual)
{
error++;
}
}
error /= output.Length;
Assert.AreEqual(error, 0);
Assert.AreEqual(-490.30977151704076, regression.Coefficients[0], 1e-7);
Assert.AreEqual(1.7763049293456503, regression.Coefficients[1], 1e-7);
Assert.AreEqual(-14.882619671822592, regression.Coefficients[2], 1e-7);
Assert.AreEqual(60.5066623676452, regression.Coefficients[3], 1e-7);
}
public void ComputeTest()
{
// Suppose we have the following data about some patients.
// The first variable is continuous and represent patient
// age. The second variable is dichotomic and give whether
// they smoke or not (This is completely fictional data).
double[][] input =
{
new double[] { 55, 0 }, // 0 - no cancer
new double[] { 28, 0 }, // 0
new double[] { 65, 1 }, // 0
new double[] { 46, 0 }, // 1 - have cancer
new double[] { 86, 1 }, // 1
new double[] { 56, 1 }, // 1
new double[] { 85, 0 }, // 0
new double[] { 33, 0 }, // 0
new double[] { 21, 1 }, // 0
new double[] { 42, 1 }, // 1
};
// We also know if they have had lung cancer or not, and
// we would like to know whether smoking has any connection
// with lung cancer (This is completely fictional data).
double[] output =
{
0, 0, 0, 1, 1, 1, 0, 0, 0, 1
};
// To verify this hypothesis, we are going to create a logistic
// regression model for those two inputs (age and smoking).
LogisticRegression regression = new LogisticRegression(inputs: 2);
// Next, we are going to estimate this model. For this, we
// will use the Iteratively Reweighted Least Squares method.
var teacher = new IterativeReweightedLeastSquares(regression);
teacher.Regularization = 0;
// Now, we will iteratively estimate our model. The Run method returns
// the maximum relative change in the model parameters and we will use
// it as the convergence criteria.
double delta = 0;
do
{
// Perform an iteration
delta = teacher.Run(input, output);
} while (delta > 0.001);
// At this point, we can compute the odds ratio of our variables.
// In the model, the variable at 0 is always the intercept term,
// with the other following in the sequence. Index 1 is the age
// and index 2 is whether the patient smokes or not.
// For the age variable, we have that individuals with
// higher age have 1.021 greater odds of getting lung
// cancer controlling for cigarette smoking.
double ageOdds = regression.GetOddsRatio(1); // 1.0208597028836701
// For the smoking/non smoking category variable, however, we
// have that individuals who smoke have 5.858 greater odds
// of developing lung cancer compared to those who do not
// smoke, controlling for age (remember, this is completely
// fictional and for demonstration purposes only).
double smokeOdds = regression.GetOddsRatio(2); // 5.8584748789881331
double[] actual = new double[output.Length];
for (int i = 0; i < input.Length; i++)
{
actual[i] = regression.Compute(input[i]);
}
double[] expected =
{
0.21044171560168326,
0.13242527535212373,
0.65747803433771812,
0.18122484822324372,
0.74755661773156912,
0.61450041841477232,
0.33116705418194975,
0.14474110902457912,
0.43627109657399382,
0.54419383282533118
};
for (int i = 0; i < actual.Length; i++)
{
Assert.AreEqual(expected[i], actual[i]);
}
Assert.AreEqual(1.0208597028836701, ageOdds, 1e-10);
Assert.AreEqual(5.8584748789881331, smokeOdds, 1e-8);
Assert.AreEqual(-2.4577464307294092, regression.Intercept, 1e-8);
Assert.AreEqual(-2.4577464307294092, regression.Coefficients[0], 1e-8);
Assert.AreEqual(0.020645118265359252, regression.Coefficients[1], 1e-10);
Assert.AreEqual(1.7678893101571855, regression.Coefficients[2], 1e-8);
bool[] actualOutput = regression.Decide(input);
Assert.IsFalse(actualOutput[0]);
Assert.IsFalse(actualOutput[1]);
Assert.IsTrue(actualOutput[2]);
Assert.IsFalse(actualOutput[3]);
Assert.IsTrue(actualOutput[4]);
Assert.IsTrue(actualOutput[5]);
Assert.IsFalse(actualOutput[6]);
Assert.IsFalse(actualOutput[7]);
Assert.IsFalse(actualOutput[8]);
Assert.IsTrue(actualOutput[9]);
}
public StrategySignal getPrediction(double[] input)
{
return(new StrategySignal(logisticBuy.Compute(input), logisticSell.Compute(input)));
}
public void LargeCoefficientsTest()
{
double[,] data =
{
{ 48, 1, 4.40, 0 },
{ 60, 0, 7.89, 1 },
{ 51, 0, 3.48, 0 },
{ 66, 0, 8.41, 1 },
{ 40, 1, 3.05, 0 },
{ 44, 1, 4.56, 0 },
{ 80, 0, 6.91, 1 },
{ 52, 0, 5.69, 0 },
{ 58, 0, 4.01, 0 },
{ 58, 0, 4.48, 0 },
{ 72, 1, 5.97, 0 },
{ 57, 0, 6.71, 1 },
{ 55, 1, 5.36, 0 },
{ 71, 0, 5.68, 0 },
{ 44, 1, 4.61, 0 },
{ 65, 1, 4.80, 0 },
{ 38, 0, 5.06, 0 },
{ 50, 0, 6.40, 0 },
{ 80, 0, 6.67, 1 },
{ 69, 1, 5.79, 0 },
{ 39, 0, 5.42, 0 },
{ 68, 0, 7.61, 1 },
{ 47, 1, 3.24, 0 },
{ 45, 1, 4.29, 0 },
{ 79, 1, 7.44, 1 },
{ 41, 1, 4.60, 0 },
{ 45, 0, 5.91, 0 },
{ 54, 0, 4.77, 0 },
{ 43, 1, 5.62, 0 },
{ 62, 1, 7.92, 1 },
{ 72, 1, 7.92, 1 },
{ 57, 1, 6.19, 0 },
{ 39, 1, 2.37, 0 },
{ 51, 0, 5.84, 0 },
{ 73, 1, 5.94, 0 },
{ 41, 1, 3.82, 0 },
{ 35, 0, 2.35, 0 },
{ 69, 0, 6.57, 1 },
{ 75, 1, 7.96, 1 },
{ 51, 1, 3.96, 0 },
{ 61, 1, 4.36, 0 },
{ 55, 0, 3.84, 0 },
{ 45, 1, 3.02, 0 },
{ 48, 0, 4.65, 0 },
{ 77, 0, 7.93, 1 },
{ 40, 1, 2.46, 0 },
{ 37, 1, 2.32, 0 },
{ 78, 0, 7.88, 1 },
{ 39, 1, 4.55, 0 },
{ 41, 0, 2.45, 0 },
{ 54, 1, 5.62, 0 },
{ 59, 1, 5.03, 0 },
{ 78, 0, 8.08, 1 },
{ 56, 1, 6.96, 1 },
{ 49, 1, 3.07, 0 },
{ 48, 0, 4.75, 0 },
{ 63, 1, 5.64, 0 },
{ 50, 0, 3.35, 0 },
{ 59, 1, 5.08, 0 },
{ 60, 0, 6.58, 1 },
{ 64, 0, 5.19, 0 },
{ 76, 1, 6.69, 1 },
{ 58, 0, 5.18, 0 },
{ 48, 1, 4.47, 0 },
{ 72, 0, 8.70, 1 },
{ 40, 1, 5.14, 0 },
{ 53, 0, 3.40, 0 },
{ 79, 0, 9.77, 1 },
{ 61, 1, 7.79, 1 },
{ 59, 0, 7.42, 1 },
{ 44, 0, 2.55, 0 },
{ 52, 1, 3.71, 0 },
{ 80, 1, 7.56, 1 },
{ 76, 0, 7.80, 1 },
{ 51, 0, 5.94, 0 },
{ 46, 1, 5.52, 0 },
{ 48, 0, 3.25, 0 },
{ 58, 1, 4.71, 0 },
{ 44, 1, 2.52, 0 },
{ 68, 0, 8.38, 1 },
};
double[][] input = data.Submatrix(null, 0, 2).ToArray();
double[] output = data.GetColumn(3);
LogisticRegression regression = new LogisticRegression(3);
var teacher = new IterativeReweightedLeastSquares(regression);
teacher.Regularization = 1e-5;
var errors = new List <double>();
for (int i = 0; i < 1000; i++)
{
errors.Add(teacher.Run(input, output));
}
double error = 0;
for (int i = 0; i < output.Length; i++)
{
double expected = output[i];
double actual = System.Math.Round(regression.Compute(input[i]));
if (expected != actual)
{
error++;
}
}
error /= output.Length;
Assert.AreEqual(error, 0);
Assert.AreEqual(-58.817944701474687, regression.Coefficients[0]);
Assert.AreEqual(0.13783960821658245, regression.Coefficients[1]);
Assert.AreEqual(-1.532885090757945, regression.Coefficients[2]);
Assert.AreEqual(7.9460105648631973, regression.Coefficients[3]);
}
/// <summary>
/// Runs one iteration of the Reweighted Least Squares algorithm.
/// </summary>
/// <param name="inputs">The input data.</param>
/// <param name="outputs">The outputs associated with each input vector.</param>
/// <returns>The maximum relative change in the parameters after the iteration.</returns>
///
public double Run(double[][] inputs, double[] outputs)
{
// Regress using Iteratively Reweighted Least Squares estimation.
// References:
// - Bishop, Christopher M.; Pattern Recognition
// and Machine Learning. Springer; 1st ed. 2006.
// Initial definitions and memory allocations
int N = inputs.Length;
double[][] design = new double[N][];
double[] errors = new double[N];
double[] weights = new double[N];
double[] coefficients = this.regression.Coefficients;
double[] deltas;
// Compute the regression matrix
for (int i = 0; i < inputs.Length; i++)
{
double[] row = design[i] = new double[parameterCount];
row[0] = 1; // for intercept
for (int j = 0; j < inputs[i].Length; j++)
{
row[j + 1] = inputs[i][j];
}
}
// Compute errors and weighing matrix
for (int i = 0; i < inputs.Length; i++)
{
double y = regression.Compute(inputs[i]);
// Calculate error vector
errors[i] = y - outputs[i];
// Calculate weighting matrix
weights[i] = y * (1.0 - y);
}
// Reset Hessian matrix and gradient
for (int i = 0; i < gradient.Length; i++)
{
gradient[i] = 0;
for (int j = 0; j < gradient.Length; j++)
{
hessian[i, j] = 0;
}
}
// (Re-) Compute error gradient
for (int j = 0; j < design.Length; j++)
{
for (int i = 0; i < gradient.Length; i++)
{
gradient[i] += design[j][i] * errors[j];
}
}
// (Re-) Compute weighted "Hessian" matrix
for (int k = 0; k < weights.Length; k++)
{
double[] rk = design[k];
for (int j = 0; j < rk.Length; j++)
{
for (int i = 0; i < rk.Length; i++)
{
hessian[j, i] += rk[i] * rk[j] * weights[k];
}
}
}
// Decompose to solve the linear system. Usually the hessian will
// be invertible and LU will succeed. However, sometimes the hessian
// may be singular and a Singular Value Decomposition may be needed.
LuDecomposition lu = new LuDecomposition(hessian);
// The SVD is very stable, but is quite expensive, being on average
// about 10-15 times more expensive than LU decomposition. There are
// other ways to avoid a singular Hessian. For a very interesting
// reading on the subject, please see:
//
// - Jeff Gill & Gary King, "What to Do When Your Hessian Is Not Invertible",
// Sociological Methods & Research, Vol 33, No. 1, August 2004, 54-87.
// Available in: http://gking.harvard.edu/files/help.pdf
//
// Moreover, the computation of the inverse is optional, as it will
// be used only to compute the standard errors of the regression.
if (lu.Nonsingular)
{
// Solve using LU decomposition
deltas = lu.Solve(gradient);
decomposition = lu;
}
else
{
// Hessian Matrix is singular, try pseudo-inverse solution
decomposition = new SingularValueDecomposition(hessian);
deltas = decomposition.Solve(gradient);
}
previous = (double[])coefficients.Clone();
// Update coefficients using the calculated deltas
for (int i = 0; i < coefficients.Length; i++)
{
coefficients[i] -= deltas[i];
}
if (computeStandardErrors)
{
// Grab the regression information matrix
double[,] inverse = decomposition.Inverse();
// Calculate coefficients' standard errors
double[] standardErrors = regression.StandardErrors;
for (int i = 0; i < standardErrors.Length; i++)
{
standardErrors[i] = Math.Sqrt(inverse[i, i]);
}
}
// Return the relative maximum parameter change
for (int i = 0; i < deltas.Length; i++)
{
deltas[i] = Math.Abs(deltas[i]) / Math.Abs(previous[i]);
}
return(Matrix.Max(deltas));
}
static void Main(string[] args)
{
if (args.Length != 2)
{
Console.WriteLine("TestLogisticRegression [train corpus file name] [test corpus file name]");
return;
}
double[][] inputs;
double[] outputs;
Console.WriteLine("Loading train corpus...");
//Load training corpus
LoadCorpusFromFile(args[0], out inputs, out outputs);
//Try to normalize output value
Normalize(outputs);
Console.WriteLine("Logistic regression...");
LogisticRegression lr = new LogisticRegression(inputs[0].Length);
double error = lr.Regress(inputs, outputs);
Console.WriteLine("Parameter list:");
for (int i = 0; i < lr.Coefficients.Length; i++)
{
Console.WriteLine("Coefficient {0}: {1}", i, lr.Coefficients[i]);
}
Console.WriteLine("Delta: {0}", error);
Console.WriteLine();
//Load test corpus
Console.WriteLine("Testing regress result:");
LoadCorpusFromFile(args[1], out inputs, out outputs);
//Try to normalize output value
Normalize(outputs);
for (int i = 0; i < outputs.Length; i++)
{
StringBuilder sb = new StringBuilder();
double output = lr.Compute(inputs[i]);
for (int j = 0; j < inputs[i].Length; j++)
{
sb.Append(inputs[i][j].ToString() + " ");
}
sb.Append(outputs[i] + " RV:" + output);
sb.Append(" Err:" + (output - outputs[i]).ToString());
Console.WriteLine(sb.ToString());
}
}
public double[] getPrediction(double[] input)
{
return(new double[] { logisticBuy.Compute(input), logisticSell.Compute(input) });
}