public void ProbabilityDensityFunctionTest() { MultivariateNormalDistribution[] components = new MultivariateNormalDistribution[2]; components[0] = new MultivariateNormalDistribution(new double[] { 2 }, new double[, ] { { 1 } }); components[1] = new MultivariateNormalDistribution(new double[] { 5 }, new double[, ] { { 1 } }); double[] coefficients = { 0.3, 0.7 }; var mixture = new MultivariateMixture <MultivariateNormalDistribution>(coefficients, components); double[] x = { 1.2 }; double expected = 0.3 * components[0].ProbabilityDensityFunction(x) + 0.7 * components[1].ProbabilityDensityFunction(x); double actual = mixture.ProbabilityDensityFunction(x); Assert.AreEqual(expected, actual); }
public void DistributionFunctionTestPerComponent() { MultivariateNormalDistribution[] components = new MultivariateNormalDistribution[2]; components[0] = new MultivariateNormalDistribution(new double[] { 2 }, new double[, ] { { 1 } }); components[1] = new MultivariateNormalDistribution(new double[] { 5 }, new double[, ] { { 1 } }); double[] coefficients = { 0.3, 0.7 }; var mixture = new MultivariateMixture <MultivariateNormalDistribution>(coefficients, components); double[] x = { 1.2 }; double expected = mixture.DistributionFunction(0, x) + mixture.DistributionFunction(1, x); double actual = mixture.DistributionFunction(x); Assert.AreEqual(expected, actual); }
public void GenerateTest() { Accord.Math.Tools.SetupGenerator(0); var normal = new MultivariateNormalDistribution( new double[] { 2, 6 }, new double[, ] { { 2, 1 }, { 1, 5 } }); double[][] sample = normal.Generate(1000000); double[] mean = sample.Mean(); double[,] cov = sample.Covariance(); Assert.AreEqual(2, mean[0], 1e-2); Assert.AreEqual(6, mean[1], 1e-2); Assert.AreEqual(2, cov[0, 0], 1e-2); Assert.AreEqual(1, cov[0, 1], 1e-2); Assert.AreEqual(1, cov[1, 0], 1e-2); Assert.AreEqual(5, cov[1, 1], 2e-2); }
private static double[, ][] calculateZXMatrix <TFilter, TState, TMeasurement>(this IList <TFilter> kalmanFilters, IList <TMeasurement> measurements, out double[,] probsZX) where TFilter : KalmanFilter <TState, TMeasurement> { probsZX = new double[measurements.Count, kalmanFilters.Count]; var innovZX = new double[measurements.Count, kalmanFilters.Count][]; for (int tIdx = 0; tIdx < kalmanFilters.Count; tIdx++) { var kalman = kalmanFilters[tIdx]; var zeroCoordinate = new double[kalman.MeasurementVectorDimension]; var mvnPDF = new MultivariateNormalDistribution(zeroCoordinate, kalman.ResidualCovariance); var mulCorrectionFactor = (double)1 / mvnPDF.ProbabilityDensityFunction(zeroCoordinate); for (int mIdx = 0; mIdx < measurements.Count; mIdx++) { var measurement = measurements[mIdx]; //delta' / S^-1 * delta; this expression has ChiSquare distribution (Mahalanobis distance) double[] delta; double mahalanobisDistancce;//not used var isInsideGate = kalman.IsMeasurementInsideGate(measurement, out delta, out mahalanobisDistancce); innovZX[mIdx, tIdx] = delta; if (isInsideGate) { probsZX[mIdx, tIdx] = mvnPDF.ProbabilityDensityFunction(delta) * mulCorrectionFactor; //modification (added mul correction factor) } //else probsZX[mIdx, tIdx] = 0 (by default) } } return(innovZX); }
public void LogProbabilityDensityFunctionTestPerComponent() { MultivariateNormalDistribution[] components = new MultivariateNormalDistribution[2]; components[0] = new MultivariateNormalDistribution(new double[] { 2 }, new double[, ] { { 1 } }); components[1] = new MultivariateNormalDistribution(new double[] { 5 }, new double[, ] { { 1 } }); double[] coefficients = { 0.3, 0.7 }; var mixture = new MultivariateMixture <MultivariateNormalDistribution>(coefficients, components); double[] x = { 1.2 }; double expected = System.Math.Log( mixture.ProbabilityDensityFunction(0, x) + mixture.ProbabilityDensityFunction(1, x)); double actual = mixture.LogProbabilityDensityFunction(x); Assert.AreEqual(expected, actual, 1e-10); Assert.IsFalse(double.IsNaN(actual)); }
public Cluster(List <Point> points, KMeans kmeans = null, int index = 0) { Dimensions = points.First().Coordinates.Length; K = Dimensions * (Dimensions + 3) / 2; Points = points; Centroid = kmeans?.Clusters[index].Centroid; Covariance = kmeans?.Clusters[index].Covariance; Minimums = new double[Dimensions]; Maximums = new double[Dimensions]; var pointsMatrix = points.Select(p => p.Coordinates).ToArray(); Means = pointsMatrix.Mean(pointsMatrix.Transpose().Select(x => x.Sum()).ToArray()); var covariance = pointsMatrix.Covariance(Means); for (var i = 0; i < Dimensions; i++) { Minimums[i] = points.Min(p => p[i]); Maximums[i] = points.Max(p => p[i]); } _multivariateNormalDistribution = new MultivariateNormalDistribution(Means, covariance); }
private static double[, ][] calculateZXMatrix <TFilter, TState, TMeasurement>(this IList <TFilter> kalmanFilters, IList <TMeasurement> measurements, out double[,] probsZX) where TFilter : KalmanFilter <TState, TMeasurement> { probsZX = new double[measurements.Count, kalmanFilters.Count]; var innovZX = new double[measurements.Count, kalmanFilters.Count][]; for (int tIdx = 0; tIdx < kalmanFilters.Count; tIdx++) { var kalman = kalmanFilters[tIdx]; var zeroCoordinate = new double[kalman.MeasurementVectorDimension]; var mvnPDF = new MultivariateNormalDistribution(zeroCoordinate, kalman.CovarianceMatrix); var mulCorrectionFactor = (double)1 / mvnPDF.ProbabilityDensityFunction(zeroCoordinate); for (int mIdx = 0; mIdx < measurements.Count; mIdx++) { var measurement = measurements[mIdx]; var delta = kalman.CalculatePredictionError(measurement); innovZX[mIdx, tIdx] = delta; //delta' / S^-1 * delta; this expression has ChiSquare distribution var gate = delta.Multiply(kalman.CovarianceMatrixInv).Multiply(delta.Transpose()).Sum(); if (gate < gateThreshold) { probsZX[mIdx, tIdx] = mvnPDF.ProbabilityDensityFunction(delta) * mulCorrectionFactor; //modification (added mul correction factor) } //else probsZX[mIdx, tIdx] = 0 (by default) } } return(innovZX); }
private void btnGenerateRandom_Click(object sender, EventArgs e) { k = (int)numClusters.Value; // Generate data with n Gaussian distributions double[][][] data = new double[k][][]; for (int i = 0; i < k; i++) { // Create random centroid to place the Gaussian distribution var mean = Matrix.Random(2, -6.0, +6.0); // Create random covariance matrix for the distribution double[,] covariance; do { covariance = Matrix.Random(2, true, 0.0, 3.0); }while (!covariance.IsPositiveDefinite()); // Create the Gaussian distribution var gaussian = new MultivariateNormalDistribution(mean, covariance); int samples = Accord.Math.Tools.Random.Next(150, 250); data[i] = gaussian.Generate(samples); } // Join the generated data mixture = Matrix.Stack(data); // Update the scatterplot CreateScatterplot(graph, mixture, k); // Forget previous initialization kmeans = null; }
public void FittingOptionsTest() { // Create a degenerate problem double[][] sequences = new double[][] { new double[] { 1, 1, 1, 1, 1, 0, 1, 1, 1, 1 }, new double[] { 1, 1, 1, 1, 0, 1, 1, 1, 1, 1 }, new double[] { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 }, new double[] { 1, 1, 1, 1, 1, 1 }, new double[] { 1, 1, 1, 1, 1, 1, 1 }, new double[] { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 }, new double[] { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1 }, }; // Creates a continuous hidden Markov Model with two states organized in a ergodic // topology and an underlying multivariate Normal distribution as density. var density = new MultivariateNormalDistribution(1); var model = new HiddenMarkovModel <MultivariateNormalDistribution>(new Ergodic(2), density); // Configure the learning algorithms to train the sequence classifier var teacher = new BaumWelchLearning <MultivariateNormalDistribution>(model) { Tolerance = 0.0001, Iterations = 0, // Configure options for fitting the normal distribution FittingOptions = new NormalOptions() { Regularization = 0.0001, } }; // Fit the model. No exceptions will be thrown double logLikelihood = teacher.Run(sequences); double likelihood = Math.Exp(logLikelihood); Assert.AreEqual(47.434837528491286, logLikelihood, 1e-15); Assert.IsFalse(double.IsNaN(logLikelihood)); Assert.AreEqual(0.0001, (teacher.FittingOptions as NormalOptions).Regularization); // Try without a regularization constant to get an exception bool thrown; thrown = false; density = new MultivariateNormalDistribution(1); model = new HiddenMarkovModel <MultivariateNormalDistribution>(new Ergodic(2), density); teacher = new BaumWelchLearning <MultivariateNormalDistribution>(model) { Tolerance = 0.0001, Iterations = 0, }; Assert.IsNull(teacher.FittingOptions); try { teacher.Run(sequences); } catch { thrown = true; } Assert.IsTrue(thrown); thrown = false; density = new Accord.Statistics.Distributions.Multivariate.MultivariateNormalDistribution(1); model = new HiddenMarkovModel <MultivariateNormalDistribution>(new Ergodic(2), density); teacher = new BaumWelchLearning <MultivariateNormalDistribution>(model) { Tolerance = 0.0001, Iterations = 0, FittingOptions = new NormalOptions() { Regularization = 0 } }; Assert.IsNotNull(teacher.FittingOptions); try { teacher.Run(sequences); } catch { thrown = true; } Assert.IsTrue(thrown); }
public void sequence_parsing_test() { #region doc_learn_fraud_analysis // Ensure results are reproducible Accord.Math.Random.Generator.Seed = 0; // Let's say we have the following data about credit card transactions, // where the data is organized in order of transaction, per credit card // holder. Everytime the "Time" column starts at zero, denotes that the // sequence of observations follow will correspond to transactions of the // same person: double[,] data = { // "Time", "V1", "V2", "V3", "V4", "V5", "Amount", "Fraud" { 0, 0.521, 0.124, 0.622, 15.2, 25.6, 2.70, 0 }, // first person, ok { 1, 0.121, 0.124, 0.822, 12.2, 25.6, 42.0, 0 }, // first person, ok { 0, 0.551, 0.124, 0.422, 17.5, 25.6, 20.0, 0 }, // second person, ok { 1, 0.136, 0.154, 0.322, 15.3, 25.6, 50.0, 0 }, // second person, ok { 2, 0.721, 0.240, 0.422, 12.2, 25.6, 100.0, 1 }, // second person, fraud! { 3, 0.222, 0.126, 0.722, 18.1, 25.8, 10.0, 0 }, // second person, ok }; // Transform the above data into a jagged matrix double[][][] input; int[][] states; transform(data, out input, out states); // Determine here the number of dimensions in the observations (in this case, 6) int observationDimensions = 6; // 6 columns: "V1", "V2", "V3", "V4", "V5", "Amount" // Create some prior distributions to help initialize our parameters var priorC = new WishartDistribution(dimension: observationDimensions, degreesOfFreedom: 10); // this 10 is just some random number, you might have to tune as if it was a hyperparameter var priorM = new MultivariateNormalDistribution(dimension: observationDimensions); // Configure the learning algorithms to train the sequence classifier var teacher = new MaximumLikelihoodLearning <MultivariateNormalDistribution, double[]>() { // Their emissions will be multivariate Normal distributions initialized using the prior distributions Emissions = (j) => new MultivariateNormalDistribution(mean: priorM.Generate(), covariance: priorC.Generate()), // We will prevent our covariance matrices from becoming degenerate by adding a small // regularization value to their diagonal until they become positive-definite again: FittingOptions = new NormalOptions() { Regularization = 1e-6 }, }; // Use the teacher to learn a new HMM var hmm = teacher.Learn(input, states); // Use the HMM to predict whether the transations were fradulent or not: int[] firstPerson = hmm.Decide(input[0]); // predict the first person, output should be: 0, 0 int[] secondPerson = hmm.Decide(input[1]); // predict the second person, output should be: 0, 0, 1, 0 #endregion Assert.AreEqual(new[] { 0, 0 }, firstPerson); Assert.AreEqual(new[] { 0, 0, 1, 0 }, secondPerson); }
public void LearnTest9() { var observations = new double[][][] { #region example new double[][] { new double[] { 2.58825719356537, -6.10018078957452, -3.51826652951428, }, new double[] { 1.5637531876564, -8.92844874836103, -9.09330631370717, }, new double[] { 2.12242007255554, -14.8117769726059, -9.04211363915664, }, new double[] { 0.39045587182045, -10.3548189544216, -7.69608701297759, }, new double[] { -0.553155690431595, -34.9185135663671, 14.6941023804174, }, new double[] { -0.923129916191101, -6.06337512248124, 8.28106954197084, }, new double[] { 0.478342920541763, -4.93066650122859, 3.1120912556361, }, }, new double[][] { new double[] { 1.89824998378754, -8.21581113387553, -7.88790716806936, }, new double[] { 2.24453508853912, -10.281886698766, -9.67846789539227, }, new double[] { 0.946296751499176, -22.0276392511088, -6.52238763834787, }, new double[] { -0.251136720180511, -13.3010653290676, 8.47499524273859, }, new double[] { -2.35625505447388, -18.1542111199742, 6.25564428645639, }, new double[] { 0.200483202934265, -5.48215328147925, 5.88811639894938, }, }, new double[][] { new double[] { 2.7240589261055, -3.71720542338046, -3.75092324997593, }, new double[] { 2.19917744398117, -7.18434871865373, -4.92539999824263, }, new double[] { 1.40723958611488, -11.5545592998714, -5.14780194932221, }, new double[] { 1.61909088492393, -12.5262932665595, -6.34366687651826, }, new double[] { -2.54745036363602, -8.64924529565274, 4.15127988308386, }, new double[] { 0.815489888191223, -33.8531051237431, 4.3954106953589, }, new double[] { -2.2090271115303, -7.17818258102413, 8.9117419130814, }, new double[] { -1.9000232219696, -2.4331659041997, 6.91224717766923, }, }, new double[][] { new double[] { 4.88746017217636, -4.36384651224969, -5.45526891285354, }, new double[] { 1.07786506414413, -12.9399071692788, -5.88248026843442, }, new double[] { 2.28888094425201, -15.4017823367163, -9.36490649113217, }, new double[] { -1.16468518972397, -35.4200913138333, 5.44735305966353, }, new double[] { -1.1483296751976, -13.5454911068913, 7.83577905727326, }, new double[] { -2.58188247680664, -1.10149600205281, 10.5928750605715, }, new double[] { -0.277529656887054, -6.96828661824016, 4.59381106840823, }, }, new double[][] { new double[] { 3.39118540287018, -2.9173207268871, -5.66795398530988, }, new double[] { 1.44856870174408, -9.21319243840922, -5.74986260778932, }, new double[] { 1.45215392112732, -10.3989582187704, -7.06932768129103, }, new double[] { 0.640938431024551, -15.319525165245, -7.68866476960221, }, new double[] { -0.77500119805336, -20.8335910793105, -1.56702420087282, }, new double[] { -3.48337143659592, -18.0461677940976, 12.3393172987974, }, new double[] { -1.17014795541763, -5.59624373275155, 6.09176828712909, }, }, new double[][] { new double[] { -3.984335064888, -6.2406475893692, -8.13815178201645, }, new double[] { -2.12110131978989, -5.60649378910647, -7.69551693188544, }, new double[] { -1.62762850522995, -24.1160212319193, -14.9683354815265, }, new double[] { -1.15231424570084, -17.1336790735458, -5.70731951079186, }, new double[] { 0.00514835119247437, -35.4256585588532, 11.0357975880744, }, new double[] { 0.247226655483246, -4.87705331087666, 8.47028869639136, }, new double[] { -1.28729045391083, -4.4684855254196, 4.45432778840328, }, }, new double[][] { new double[] { -5.14926165342331, -14.4168633009146, -14.4808205022332, }, new double[] { -3.93681302666664, -13.6040611430423, -9.52852874304709, }, new double[] { -4.0200162678957, -17.9772444010218, -10.9145425003168, }, new double[] { 2.99205146729946, -11.3995995445577, 10.0112700536762, }, new double[] { -1.80960297584534, -25.9626088707583, 3.84153700324761, }, new double[] { -0.47445073723793, -3.15995343875038, 3.81288679772555, }, }, new double[][] { new double[] { -3.10730338096619, -4.90623566171983, -7.71155001801384, }, new double[] { -2.58265435695648, -12.8249488039327, -7.81701695282102, }, new double[] { -3.70455086231232, -10.9642675851383, -10.3474496036822, }, new double[] { 2.34457105398178, -22.575668228196, -4.00681935468317, }, new double[] { -0.137023627758026, -22.8846781066673, 6.49448229892285, }, new double[] { -1.04487389326096, -10.8106353197974, 6.89123118904132, }, new double[] { -0.807777792215347, -6.72485967042486, 6.44026679233423, }, new double[] { -0.0864192843437195, -1.82784244477527, 5.21446167464657, }, }, new double[][] { new double[] { -3.68375554680824, -8.91158395500054, -9.35894038244743, }, new double[] { -3.42774018645287, -8.90966793048099, -12.0502934183779, }, new double[] { -2.21796408295631, -20.1283824753482, -9.3404551995806, }, new double[] { 0.275979936122894, -24.8898254667703, -1.95441472953041, }, new double[] { 2.8757631778717, -25.5929744730134, 15.9213204397452, }, new double[] { -0.0532664358615875, -5.41014381829368, 7.0702071664098, }, new double[] { -0.523447245359421, -2.21351362388411, 5.47910029515575, }, }, new double[][] { new double[] { -2.87790596485138, -4.67335526533981, -5.23215633615683, }, new double[] { -2.4156779050827, -3.99829080603495, -4.85576151355235, }, new double[] { -2.6987336575985, -7.76589206730162, -5.81054787011341, }, new double[] { -2.65482440590858, -10.5628263066491, -5.60468502395908, }, new double[] { -2.54620611667633, -13.0387387107748, -5.36223367466908, }, new double[] { -0.349991768598557, -6.54244110985515, -4.35843018634009, }, new double[] { 1.43021196126938, -14.1423935327282, 11.3171592025544, }, new double[] { -0.248833745718002, -25.6880129237476, 3.6943247495434, }, new double[] { -0.191526114940643, -7.40986142342928, 5.01053017361167, }, new double[] { 0.0262223184108734, -2.32355649224634, 5.02960958030255, }, }, new double[][] { new double[] { -0.491838902235031, -6.14010393559236, 0.827477332024586, }, new double[] { -0.806065648794174, -7.15029676810841, -1.19623376104369, }, new double[] { -0.376655906438828, -8.79062775480082, -1.90518908829517, }, new double[] { 0.0747844576835632, -8.78933441325732, -1.96265207353993, }, new double[] { -0.375023484230042, 3.89681155173501, 9.01643231817069, }, new double[] { -2.8106614947319, -11.460008093918, 2.27801912994775, }, new double[] { 8.87353122234344, -36.8569805718597, 6.36432395690119, }, new double[] { 2.17160433530808, -6.57312981892095, 6.99683358454453, }, }, new double[][] { new double[] { -2.59969010949135, -3.67992698430228, 1.09594294144671, }, new double[] { -1.09673067927361, -5.84256216502719, -0.576662929456575, }, new double[] { -1.31642892956734, -7.75851355520771, -2.38379618379558, }, new double[] { -0.119869410991669, -8.5749576027529, -1.84393133510667, }, new double[] { 1.6157403588295, -8.50491836461337, 1.75083250596366, }, new double[] { 1.66225507855415, -26.4882911957686, 1.98153904369032, }, new double[] { 2.55657434463501, -10.5098938623168, 11.632377227365, }, new double[] { 1.91832333803177, -9.98753621777953, 7.38483383044985, }, new double[] { 2.16058492660522, -2.7784029746222, 7.8378896386686, }, }, #endregion }; var density = new MultivariateNormalDistribution(3); var model = new HiddenMarkovModel <MultivariateNormalDistribution>(new Forward(5), density); var learning = new ViterbiLearning <MultivariateNormalDistribution>(model) { Tolerance = 0.0001, Iterations = 0, FittingOptions = new NormalOptions() { Regularization = 0.0001 } }; double logLikelihood = learning.Run(observations); Assert.IsFalse(Double.IsNaN(logLikelihood)); foreach (double value in model.Transitions) { Assert.IsFalse(Double.IsNaN(value)); } foreach (double value in model.Probabilities) { Assert.IsFalse(Double.IsNaN(value)); } }
public void learn_pendigits_normalization() { Console.WriteLine("Starting NormalQuasiNewtonHiddenLearningTest.learn_pendigits_normalization"); using (var travis = new KeepTravisAlive()) { #region doc_learn_pendigits // Ensure we get reproducible results Accord.Math.Random.Generator.Seed = 0; // Download the PENDIGITS dataset from UCI ML repository var pendigits = new Pendigits(path: Path.GetTempPath()); // Get and pre-process the training set double[][][] trainInputs = pendigits.Training.Item1; int[] trainOutputs = pendigits.Training.Item2; // Pre-process the digits so each of them is centered and scaled trainInputs = trainInputs.Apply(Accord.Statistics.Tools.ZScores); trainInputs = trainInputs.Apply((x) => x.Subtract(x.Min())); // make them positive // Create some prior distributions to help initialize our parameters var priorC = new WishartDistribution(dimension: 2, degreesOfFreedom: 5); var priorM = new MultivariateNormalDistribution(dimension: 2); // Create a new learning algorithm for creating continuous hidden Markov model classifiers var teacher1 = new HiddenMarkovClassifierLearning <MultivariateNormalDistribution, double[]>() { // This tells the generative algorithm how to train each of the component models. Note: The learning // algorithm is more efficient if all generic parameters are specified, including the fitting options Learner = (i) => new BaumWelchLearning <MultivariateNormalDistribution, double[], NormalOptions>() { Topology = new Forward(5), // Each model will have a forward topology with 5 states // Their emissions will be multivariate Normal distributions initialized using the prior distributions Emissions = (j) => new MultivariateNormalDistribution(mean: priorM.Generate(), covariance: priorC.Generate()), // We will train until the relative change in the average log-likelihood is less than 1e-6 between iterations Tolerance = 1e-6, MaxIterations = 1000, // or until we perform 1000 iterations (which is unlikely for this dataset) // We will prevent our covariance matrices from becoming degenerate by adding a small // regularization value to their diagonal until they become positive-definite again: FittingOptions = new NormalOptions() { Regularization = 1e-6 } } }; // The following line is only needed to ensure reproducible results. Please remove it to enable full parallelization teacher1.ParallelOptions.MaxDegreeOfParallelism = 1; // (Remove, comment, or change this line to enable full parallelism) // Use the learning algorithm to create a classifier var hmmc = teacher1.Learn(trainInputs, trainOutputs); // Create a new learning algorithm for creating HCRFs var teacher2 = new HiddenQuasiNewtonLearning <double[]>() { Function = new MarkovMultivariateFunction(hmmc), MaxIterations = 10 }; // The following line is only needed to ensure reproducible results. Please remove it to enable full parallelization teacher2.ParallelOptions.MaxDegreeOfParallelism = 1; // (Remove, comment, or change this line to enable full parallelism) // Use the learning algorithm to create a classifier var hcrf = teacher2.Learn(trainInputs, trainOutputs); // Compute predictions for the training set int[] trainPredicted = hcrf.Decide(trainInputs); // Check the performance of the classifier by comparing with the ground-truth: var m1 = new GeneralConfusionMatrix(predicted: trainPredicted, expected: trainOutputs); double trainAcc = m1.Accuracy; // should be 0.66523727844482561 // Prepare the testing set double[][][] testInputs = pendigits.Testing.Item1; int[] testOutputs = pendigits.Testing.Item2; // Apply the same normalizations testInputs = testInputs.Apply(Accord.Statistics.Tools.ZScores); testInputs = testInputs.Apply((x) => x.Subtract(x.Min())); // make them positive // Compute predictions for the test set int[] testPredicted = hcrf.Decide(testInputs); // Check the performance of the classifier by comparing with the ground-truth: var m2 = new GeneralConfusionMatrix(predicted: testPredicted, expected: testOutputs); double testAcc = m2.Accuracy; // should be 0.66506538564184681 #endregion Assert.AreEqual(0.66523727844482561, trainAcc, 1e-10); Assert.AreEqual(0.66506538564184681, testAcc, 1e-10); } }
public void learn_pendigits_normalization() { #region doc_learn_pendigits // Ensure we get reproducible results Accord.Math.Random.Generator.Seed = 0; // Download the PENDIGITS dataset from UCI ML repository var pendigits = new Pendigits(path: Path.GetTempPath()); // Get and pre-process the training set double[][][] trainInputs = pendigits.Training.Item1; int[] trainOutputs = pendigits.Training.Item2; // Pre-process the digits so each of them is centered and scaled trainInputs = trainInputs.Apply(Accord.Statistics.Tools.ZScores); trainInputs = trainInputs.Apply((x) => x.Subtract(x.Min())); // make them positive // Create some prior distributions to help initialize our parameters var priorC = new WishartDistribution(dimension: 2, degreesOfFreedom: 5); var priorM = new MultivariateNormalDistribution(dimension: 2); // Create a template Markov classifier that we can use as a base for the HCRF var hmmc = new HiddenMarkovClassifier <MultivariateNormalDistribution, double[]>( classes: pendigits.NumberOfClasses, topology: new Forward(5), initial: (i, j) => new MultivariateNormalDistribution(mean: priorM.Generate(), covariance: priorC.Generate())); // Create a new learning algorithm for creating HCRFs var teacher = new HiddenQuasiNewtonLearning <double[]>() { Function = new MarkovMultivariateFunction(hmmc), MaxIterations = 10 }; // The following line is only needed to ensure reproducible results. Please remove it to enable full parallelization teacher.ParallelOptions.MaxDegreeOfParallelism = 1; // (Remove, comment, or change this line to enable full parallelism) // Use the learning algorithm to create a classifier var hcrf = teacher.Learn(trainInputs, trainOutputs); // Compute predictions for the training set int[] trainPredicted = hcrf.Decide(trainInputs); // Check the performance of the classifier by comparing with the ground-truth: var m1 = new ConfusionMatrix(predicted: trainPredicted, expected: trainOutputs); double trainAcc = m1.Accuracy; // should be 0.89594053744997137 // Prepare the testing set double[][][] testInputs = pendigits.Testing.Item1; int[] testOutputs = pendigits.Testing.Item2; // Apply the same normalizations testInputs = testInputs.Apply(Accord.Statistics.Tools.ZScores); testInputs = testInputs.Apply((x) => x.Subtract(x.Min())); // make them positive // Compute predictions for the test set int[] testPredicted = hcrf.Decide(testInputs); // Check the performance of the classifier by comparing with the ground-truth: var m2 = new ConfusionMatrix(predicted: testPredicted, expected: testOutputs); double testAcc = m2.Accuracy; // should be 0.89594053744997137 #endregion Assert.AreEqual(0.89594053744997137, trainAcc, 1e-10); Assert.AreEqual(0.896050173472111, testAcc, 1e-10); }
/// <summary> /// Bhattacharyya distance between two Gaussian distributions. /// </summary> /// /// <param name="a">The first Normal distribution.</param> /// <param name="b">The second Normal distribution.</param> /// /// <returns>The Bhattacharyya distance between the two distributions.</returns> /// public double Distance(MultivariateNormalDistribution a, MultivariateNormalDistribution b) { return(Distance(a.Mean, a.Covariance, b.Mean, b.Covariance)); }
public Form1() { InitializeComponent(); //double[][][] sequences = new double[][][] //{ // new double[][] // { // // This is the first sequence with label = 0 // new double[] { 0, 1 }, // new double[] { 1, 2 }, // new double[] { 2, 3 }, // new double[] { 3, 4 }, // new double[] { 4, 5 }, // }, // new double[][] // { // // This is the second sequence with label = 1 // new double[] { 4, 3 }, // new double[] { 3, 2 }, // new double[] { 2, 1 }, // new double[] { 1, 0 }, // new double[] { 0, -1 }, // } //}; // Labels for the sequences //int[] labels = { 0, 1 }; #region Old //Dictionary<string, List<List<double[]>>> dictAll; //using (System.IO.FileStream fs = new System.IO.FileStream(@".\SkeletonsAsDouble.serialized", System.IO.FileMode.Open)) //{ // BinaryFormatter bf = new BinaryFormatter(); // dictAll = (Dictionary<string, List<List<double[]>>>)bf.Deserialize(fs); //} //#region Export to CSV //using (System.IO.StreamWriter sw = new System.IO.StreamWriter(@".\SkeletonsAsCSV.csv")) //{ // double[] rgFirstFrame = dictAll[dictAll.Keys.ElementAt(0)][0][0]; // string strHeader = string.Empty; // for (int i = 0; i < rgFirstFrame.Length; i++) // { // strHeader += string.Format("F{0},", i); // } // strHeader += "Label"; // sw.WriteLine(strHeader); // foreach (KeyValuePair<string, List<List<double[]>>> kvp in dictAll) // { // foreach (List<double[]> lstGesture in kvp.Value) // { // foreach (double[] frame in lstGesture) // { // sw.WriteLine(string.Format("{0},{1}", string.Join(",", frame), kvp.Key)); // } // } // } //} #endregion #region CHANGE THESE PARAMETERS const int nStartHiddenCount = 2; const int nEndHiddenCount = 12; const int nFeatureCount = 8; // Both DataOrig.dat and Data.dat have dimensionality of 8 #endregion string strFile; int nClasses; #if STRHC1 // Data.dat = Second STRHC (10 classes but not as noisy) strFile = @".\Data.dat"; nClasses = 10; #else // Data1.dat = First STRHC (4 classes but very noisy) strFile = @".\DataOrig.dat"; nClasses = 4; #endif Dictionary <int, Tuple <Dictionary <string, List <List <double[]> > >, Dictionary <string, List <List <double[]> > > > > dict; using (System.IO.FileStream fs = new System.IO.FileStream(strFile, System.IO.FileMode.Open, System.IO.FileAccess.Read)) { BinaryFormatter bf = new BinaryFormatter(); dict = bf.Deserialize(fs) as Dictionary <int, Tuple <Dictionary <string, List <List <double[]> > >, Dictionary <string, List <List <double[]> > > > >; } using (System.IO.StreamWriter sw = new System.IO.StreamWriter(@"HMMOutput.txt")) { for (int nHiddenCount = nStartHiddenCount; nHiddenCount <= nEndHiddenCount; nHiddenCount++) { double fAverage = 0.0; int nEpoch = 0; List <int> lstTrainTimes = new List <int>(); List <int> lstRecogTimes = new List <int>(); //for (int i = 1; i <= nEpochs; i++) foreach (KeyValuePair <int, Tuple <Dictionary <string, List <List <double[]> > >, Dictionary <string, List <List <double[]> > > > > kvpTT in dict) { nEpoch++; #region Old //Dictionary<string, List<List<double[]>>> dictTrain = new Dictionary<string, List<List<double[]>>>(); //Dictionary<string, List<List<double[]>>> dictTest = new Dictionary<string, List<List<double[]>>>(); //#region Divide the vectors into training and testing sets //foreach (KeyValuePair<string, List<List<double[]>>> kvp in dictAll) //{ // int nGroupSize = kvp.Value.Count / nEpochs; // List<List<double[]>> lstTrain; // List<List<double[]>> lstTest; // dictTrain.Add(kvp.Key, (lstTrain = new List<List<double[]>>())); // dictTest.Add(kvp.Key, (lstTest = new List<List<double[]>>())); // for (int j = 0; j < kvp.Value.Count; j++) // { // if (j < i * nGroupSize && j >= (i * nGroupSize) - nGroupSize) // { // lstTest.Add(kvp.Value[j]); // } // else // { // lstTrain.Add(kvp.Value[j]); // } // } //} //#endregion #endregion Dictionary <string, List <List <double[]> > > dictTrain = kvpTT.Value.Item1; Dictionary <string, List <List <double[]> > > dictTest = kvpTT.Value.Item2; double[][][] sequences = new double[dictTrain.Sum(kvp => kvp.Value.Count)][][]; int[] labels = new int[sequences.Length]; #region The Sequences int nIndex = 0; foreach (KeyValuePair <string, List <List <double[]> > > kvp in dictTrain) { foreach (List <double[]> lst in kvp.Value) { sequences[nIndex] = lst.ToArray(); labels[nIndex] = Array.IndexOf(dictTrain.Keys.ToArray(), kvp.Key); nIndex++; } } #endregion var initialDensity = new MultivariateNormalDistribution(nFeatureCount); // Creates a sequence classifier containing 2 hidden Markov Models with 2 states // and an underlying multivariate mixture of Normal distributions as density. var classifier = new HiddenMarkovClassifier <MultivariateNormalDistribution>( classes: nClasses, topology: new Forward(nHiddenCount), initial: initialDensity); // Configure the learning algorithms to train the sequence classifier var teacher = new HiddenMarkovClassifierLearning <MultivariateNormalDistribution>( classifier, // Train each model until the log-likelihood changes less than 0.0001 modelIndex => new BaumWelchLearning <MultivariateNormalDistribution>( classifier.Models[modelIndex]) { Tolerance = 0.0001, Iterations = 0, FittingOptions = new NormalOptions() { Diagonal = true, // only diagonal covariance matrices Regularization = 1e-5 // avoid non-positive definite errors } } ); // Train the sequence classifier using the algorithm System.Diagnostics.Stopwatch watchTrain = new System.Diagnostics.Stopwatch(); watchTrain.Start(); double logLikelihood = teacher.Run(sequences, labels); watchTrain.Stop(); lstTrainTimes.Add((int)watchTrain.ElapsedMilliseconds); //// Calculate the probability that the given //// sequences originated from the model //double likelihood, likelihood2; //// Try to classify the 1st sequence (output should be 0) //int c1 = classifier.Compute(sequences[0], out likelihood); //// Try to classify the 2nd sequence (output should be 1) //int c2 = classifier.Compute(sequences[1], out likelihood2); sw.WriteLine("Epoch: {0} -- Hidden State: {1}\t\tTotal Train Time: {2}", nEpoch, nHiddenCount, watchTrain.ElapsedMilliseconds.ToString()); int nCorrect = 0; int nIncorrect = 0; foreach (KeyValuePair <string, List <List <double[]> > > kvp in dictTest) { foreach (List <double[]> lst in kvp.Value) { System.Diagnostics.Stopwatch watch = new System.Diagnostics.Stopwatch(); watch.Start(); int nClassIndex = classifier.Compute(lst.ToArray()); watch.Stop(); sw.WriteLine(string.Format("Should be: {0}\tRecognized: {1}\t\tTime: {2} ms", kvp.Key, dictTest.Keys.ElementAt(nClassIndex), watch.ElapsedMilliseconds)); lstRecogTimes.Add((int)watch.ElapsedMilliseconds); if (dictTest.Keys.ElementAt(nClassIndex) == kvp.Key) { nCorrect++; } else { nIncorrect++; } } } fAverage += (double)nCorrect / (nCorrect + nIncorrect); sw.WriteLine(string.Format("Correct: {0} of {1} ({2:P3})", nCorrect, nCorrect + nIncorrect, (double)nCorrect / (nCorrect + nIncorrect))); sw.WriteLine(); sw.WriteLine(); } sw.WriteLine("Average Correct for {0} Hidden: {1:P3}", nHiddenCount, fAverage / 5); sw.WriteLine("Average Train Time for {0}: {1:F2}", nHiddenCount, lstTrainTimes.Select(v => (double)v).Average()); sw.WriteLine("Std. Dev. Train Time for {0}: {1:F2}", nHiddenCount, lstTrainTimes.Select(v => (double)v).StandardDeviation()); sw.WriteLine("Average Recog. Time for {0}: {1:F2}", nHiddenCount, lstRecogTimes.Select(v => (double)v).Average()); sw.WriteLine("Std. Dev. Recog. Time for {0}: {1:F2}", nHiddenCount, lstRecogTimes.Select(v => (double)v).StandardDeviation()); sw.WriteLine(); sw.WriteLine(); sw.WriteLine(); sw.WriteLine(); } } }
/// <summary> /// Bhattacharyya distance between two Gaussian distributions. /// </summary> /// /// <param name="x">The first Normal distribution.</param> /// <param name="y">The second Normal distribution.</param> /// /// <returns>The Bhattacharyya distance between the two distributions.</returns> /// public double Distance(MultivariateNormalDistribution x, MultivariateNormalDistribution y) { return(Distance(x.Mean, x.Covariance, y.Mean, y.Covariance)); }
public void LearnTest6() { // Create a Continuous density Hidden Markov Model Sequence Classifier // to detect a multivariate sequence and the same sequence backwards. double[][][] sequences = new double[][][] { new double[][] { // This is the first sequence with label = 0 new double[] { 0, 1 }, new double[] { 1, 2 }, new double[] { 2, 3 }, new double[] { 3, 4 }, new double[] { 4, 5 }, }, new double[][] { // This is the second sequence with label = 1 new double[] { 4, 3 }, new double[] { 3, 2 }, new double[] { 2, 1 }, new double[] { 1, 0 }, new double[] { 0, -1 }, } }; // Labels for the sequences int[] labels = { 0, 1 }; var density = new MultivariateNormalDistribution(2); // Creates a sequence classifier containing 2 hidden Markov Models with 2 states // and an underlying multivariate mixture of Normal distributions as density. var classifier = new HiddenMarkovClassifier <MultivariateNormalDistribution>( 2, new Custom(new double[2, 2], new double[2]), density); // Configure the learning algorithms to train the sequence classifier var teacher = new HiddenMarkovClassifierLearning <MultivariateNormalDistribution>( classifier, // Train each model until the log-likelihood changes less than 0.0001 modelIndex => new BaumWelchLearning <MultivariateNormalDistribution>( classifier.Models[modelIndex]) { Tolerance = 0.0001, Iterations = 0, FittingOptions = new NormalOptions() { Diagonal = true } } ); // Train the sequence classifier using the algorithm double logLikelihood = teacher.Run(sequences, labels); // Calculate the probability that the given // sequences originated from the model double response1, response2; // Try to classify the 1st sequence (output should be 0) int c1 = classifier.Compute(sequences[0], out response1); // Try to classify the 2nd sequence (output should be 1) int c2 = classifier.Compute(sequences[1], out response2); Assert.AreEqual(double.NegativeInfinity, logLikelihood); Assert.AreEqual(0, response1); Assert.AreEqual(0, response2); Assert.IsFalse(double.IsNaN(logLikelihood)); Assert.IsFalse(double.IsNaN(response1)); Assert.IsFalse(double.IsNaN(response2)); }
public void LearnTest3() { // Create a Continuous density Hidden Markov Model Sequence Classifier // to detect a multivariate sequence and the same sequence backwards. double[][][] sequences = new double[][][] { new double[][] { // This is the first sequence with label = 0 new double[] { 0 }, new double[] { 1 }, new double[] { 2 }, new double[] { 3 }, new double[] { 4 }, }, new double[][] { // This is the second sequence with label = 1 new double[] { 4 }, new double[] { 3 }, new double[] { 2 }, new double[] { 1 }, new double[] { 0 }, } }; // Labels for the sequences int[] labels = { 0, 1 }; // Creates a sequence classifier containing 2 hidden Markov Models // with 2 states and an underlying Normal distribution as density. MultivariateNormalDistribution density = new MultivariateNormalDistribution(1); var classifier = new HiddenMarkovClassifier <MultivariateNormalDistribution>(2, new Ergodic(2), density); // Configure the learning algorithms to train the sequence classifier var teacher = new HiddenMarkovClassifierLearning <MultivariateNormalDistribution>(classifier, // Train each model until the log-likelihood changes less than 0.001 modelIndex => new BaumWelchLearning <MultivariateNormalDistribution>(classifier.Models[modelIndex]) { Tolerance = 0.0001, Iterations = 0 } ); // Train the sequence classifier using the algorithm double logLikelihood = teacher.Run(sequences, labels); // Calculate the probability that the given // sequences originated from the model double likelihood1, likelihood2; // Try to classify the first sequence (output should be 0) int c1 = classifier.Compute(sequences[0], out likelihood1); // Try to classify the second sequence (output should be 1) int c2 = classifier.Compute(sequences[1], out likelihood2); Assert.AreEqual(0, c1); Assert.AreEqual(1, c2); Assert.AreEqual(-13.271981026832929, logLikelihood, 1e-14); Assert.AreEqual(0.99999791320102149, likelihood1, 1e-15); Assert.AreEqual(0.99999791320102149, likelihood2, 1e-15); Assert.IsFalse(double.IsNaN(logLikelihood)); Assert.IsFalse(double.IsNaN(likelihood1)); Assert.IsFalse(double.IsNaN(likelihood2)); }
public void LearnGesture(int valuesUsed, int modeUsed, int statesUsed) { double[][][] inputs = new double[storedGestures.Count][][]; int[] outputs = new int[storedGestures.Count]; for (int i = 0; i < inputs.Length; i++) { double[][] points = new double[storedGestures[i].points.Length][]; switch (modeUsed) { case 3: for (int j = 0; j < storedGestures[i].points.Length; j++) { points[j] = new double[3] { storedGestures[i].points[j][0], storedGestures[i].points[j][1], storedGestures[i].points[j][2] }; } break; case 33: for (int j = 0; j < storedGestures[i].points.Length; j++) { points[j] = new double[6] { storedGestures[i].points[j][0], storedGestures[i].points[j][1], storedGestures[i].points[j][2], storedGestures[i].points[j][6], storedGestures[i].points[j][7], storedGestures[i].points[j][8] }; } break; case 6: for (int j = 0; j < storedGestures[i].points.Length; j++) { points[j] = new double[6] { storedGestures[i].points[j][0], storedGestures[i].points[j][1], storedGestures[i].points[j][2], storedGestures[i].points[j][3], storedGestures[i].points[j][4], storedGestures[i].points[j][5] }; } break; case 66: for (int j = 0; j < storedGestures[i].points.Length; j++) { points[j] = new double[12] { storedGestures[i].points[j][0], storedGestures[i].points[j][1], storedGestures[i].points[j][2], storedGestures[i].points[j][3], storedGestures[i].points[j][4], storedGestures[i].points[j][5], storedGestures[i].points[j][6], storedGestures[i].points[j][7], storedGestures[i].points[j][8], storedGestures[i].points[j][9], storedGestures[i].points[j][10], storedGestures[i].points[j][11] }; } break; } inputs[i] = points; outputs[i] = storedGestures[i].index; } List <String> classes = new List <String>(); int states = gestureIndex.Count; MultivariateNormalDistribution dist = new MultivariateNormalDistribution(valuesUsed); hmm = new HiddenMarkovClassifier <MultivariateNormalDistribution, double[]> (states, new Forward(statesUsed), dist); var teacher = new HiddenMarkovClassifierLearning <MultivariateNormalDistribution, double[]>(hmm) { Learner = i => new BaumWelchLearning <MultivariateNormalDistribution, double[]>(hmm.Models[i]) { Tolerance = 0.01, MaxIterations = 0, FittingOptions = new NormalOptions() { Regularization = 1e-5 } } }; teacher.Empirical = true; teacher.Rejection = false; teacher.Learn(inputs, outputs); Debug.Log("Sequence Learned!"); }
private static double M(double x, double y, double rho) { return(MultivariateNormalDistribution.Bivariate(0, 0, 1, 1, rho).DistributionFunction(new double[] { x, y })); }
/// <summary> /// Generates a random vector of observations from a distribution with the given parameters. /// </summary> /// /// <param name="samples">The number of samples to generate.</param> /// <param name="mean">The mean vector μ (mu) for the distribution.</param> /// <param name="covariance">The covariance matrix Σ (sigma) for the distribution.</param> /// /// <returns>A random vector of observations drawn from this distribution.</returns> /// public static double[][] Generate(int samples, double[] mean, double[,] covariance) { var normal = new MultivariateNormalDistribution(mean, covariance); return(normal.Generate(samples)); }