public static void TestDeterminant() { Matrix m1 = Matrix.Parse("6 0 \n 0 1"); Matrix m2 = Matrix.Parse("3.0 2.0 \n 1.5 3.0"); Matrix m3 = Matrix.Parse("0.7493 0.5074\n -0.004 -0.4204"); Matrix m4 = PositiveDefiniteMatrix.Identity(4); Console.WriteLine(m1); // expect 6 Console.WriteLine("m1 det: {0}", m1.Determinant()); // expect 6 Console.WriteLine("m2 det: {0}", m2.Determinant()); // expect -0.313 Console.WriteLine("m3 det: {0}", m3.Determinant()); Console.WriteLine("m4 det: {0}", m4.Determinant()); // expect 6 Console.WriteLine("m1 my det: {0}", MatrixUtils.Determinant(m1)); // expect 6 Console.WriteLine("m2 my det: {0}", MatrixUtils.Determinant(m2)); // expect -0.313 Console.WriteLine("m3 my det: {0}", MatrixUtils.Determinant(m3)); Console.WriteLine("m4 my det: {0}", MatrixUtils.Determinant(m4)); // Use MathNet MNMatrix m5 = MNMatrix.Build.Dense(2, 2, new double[] { 6, 0, 0, 1 }); Console.WriteLine(m5); Console.WriteLine("m5 det: {0}", m5.Determinant()); }
//--------- public static void TestBlkDiag() { Matrix m1 = Matrix.Parse("1 2\n 3 4"); Matrix m2 = Matrix.Parse("5 6 7\n 8 9 10 "); Matrix m3 = Matrix.Parse("11\n12"); Matrix m4 = Matrix.Parse("13 14"); Matrix big = MatrixUtils.BlkDiag(m1, m2, m3, m4); Console.WriteLine(big); }
public static void TestMathNetInverse() { // http://numerics.mathdotnet.com/Matrix.html // inverse is // // ans = // // 0.5000 -0.3333 // -0.2500 0.5000 Matrix inferMat = Matrix.Parse("3 2 \n 1.5 3"); Console.WriteLine(inferMat); Matrix inv = MatrixUtils.Inverse(inferMat); Console.WriteLine("Inverse: "); Console.WriteLine(inv); }
public void LoadDataFromMat(string filePath, out Vector[] X, out bool[] Y) { // Expect X (dxn) and Y (1xn) in the mat file Dictionary <string, object> dict = MatlabReader.Read(filePath); Matrix xmat = (Matrix)dict["X"]; X = MatrixUtils.SplitColumns(xmat); int n = X.Length; // Y is actually a 1xn vector Matrix ymat = (Matrix)dict["Y"]; double[,] yarr = ymat.ToArray(); if (yarr.GetLength(1) != n) { throw new ArgumentException("length of X and Y do not match."); } Y = new bool[n]; for (int i = 0; i < n; i++) { // expect 0, 1 Y[i] = yarr[0, i] > 0.5; } }
public void RunOnlineKEPSampling() { // Kernel EP with importance sampling. /** * Only one W just like in Ali's paper. * In practice, we typically observe multiple sets of observations * where we want to do inference on the same model with the same * parameter. */ Rand.Restart(init_fixed_seed); Vector w = Vector.Zero(d); Rand.Normal(Vector.Zero(d), PositiveDefiniteMatrix.Identity(d), w); List <LogisticOpRecords> allRecs = new List <LogisticOpRecords>(); // Create the Logistic operator instance only one because we want to use the same // one after a new problem (new seed). // stopwatch for measuring inference time for each problem Stopwatch watch = new Stopwatch(); var logisticOpIns = new KEPOnlineISLogisticOpIns( new LogisticOpRecords(), watch, -8.5); logisticOpIns.SetImportanceSamplingSize(importanceSamplingSize); logisticOpIns.IsRecordMessages = true; logisticOpIns.IsPrintTrueWhenCertain = false; /** Use mixture or not ...*/ logisticOpIns.isGaussianOp.useMixtureProposal = false; logisticOpIns.SetFeatures(new int[] { 300, 500 }); OpControl.Set(typeof(KEPOnlineLogisticOp), logisticOpIns); Type logisticOp = typeof(KEPOnlineLogisticOp); List <long> allInferTimes = new List <long>(); var allPosteriors = new List <VectorGaussian>(); var allDotNetPosteriors = new List <VectorGaussian>(); LogisticOp2.IsCollectLogisticMessages = false; LogisticOp2.IsCollectProjMsgs = false; LogisticOp2.IsCollectXMessages = false; for (int seed = seed_from; seed <= seed_to; seed++) { Rand.Restart(seed); double b = 0; // combine the bias term into W Vector[] X; bool[] Y; LogisticRegression.GenData(n, w, b, out X, out Y, seed); Console.Write("Y: "); StringUtils.PrintArray(Y); VectorGaussian wPost; LogisticOpRecords recorder = new LogisticOpRecords(); // Set a new recorder for a new problem seed logisticOpIns.SetRecorder(recorder); // Type logisticOp = typeof(LogisticOp2); // start the watch watch.Restart(); LogisticRegression.InferCoefficientsNoBias(X, Y, out wPost, epIter, logisticOp); // stop the watch long inferenceTime = watch.ElapsedMilliseconds; recorder.inferenceTimes = new List <long>(); recorder.inferenceTimes.Add(inferenceTime); allInferTimes.Add(inferenceTime); recorder.postW = MatrixUtils.ToList(wPost); allPosteriors.Add(wPost); allRecs.Add(recorder); //print Console.WriteLine("n: {0}", n); Console.WriteLine("d: {0}", d); int t = Y.Sum(o => o ? 1 : 0); Console.WriteLine("number of true: {0}", t); Console.WriteLine("True bias: {0}", b); // Vector meanW = wPost.GetMean(); Console.WriteLine("True w: {0}", w); Console.WriteLine("Inferred w: "); Console.WriteLine(wPost); // Run Infer.net's operator on the same data VectorGaussian dotNetPostW; LogisticRegression.InferCoefficientsNoBias(X, Y, out dotNetPostW, epIter, typeof(LogisticOp2)); recorder.dotNetPostW = MatrixUtils.ToList <VectorGaussian>(dotNetPostW); allDotNetPosteriors.Add(dotNetPostW); // write the records to a file string fname = string.Format("rec_onlinekep_is{0}_n{1}_logistic_iter{2}_s{3}.mat", importanceSamplingSize, n, epIter, seed); string recordPath = Config.PathToSavedFile(fname); var extra = new Dictionary <string, object>(); // MatlabWriter cannot write int extra.Add("d", (double)d); extra.Add("n", (double)n); extra.Add("epIter", (double)epIter); extra.Add("trueW", w); extra.Add("X", MatrixUtils.StackColumns(X)); extra.Add("Y", MatrixUtils.ToDouble(Y)); recorder.WriteRecords(recordPath, extra); } // merge all records and write LogisticOpRecords merged = LogisticOpRecords.Merge(allRecs.ToArray()); merged.inferenceTimes = allInferTimes; merged.dotNetPostW = allDotNetPosteriors; merged.postW = allPosteriors; string fnameM = string.Format("rec_onlinekep_is{0}_n{1}_logistic_iter{2}_sf{3}_st{4}.mat", importanceSamplingSize, n, epIter, seed_from, seed_to); string recordPathM = Config.PathToSavedFile(fnameM); merged.WriteRecords(recordPathM); }
public void RecordInferNETTime() { /**Records time by infer.net*/ /** * Only one W just like in Ali's paper. * In practice, we typically observe multiple sets of observations * where we want to do inference on the same model with the same * parameter. */ Rand.Restart(init_fixed_seed); Vector w = Vector.Zero(d); Rand.Normal(Vector.Zero(d), PositiveDefiniteMatrix.Identity(d), w); // Create the Logistic operator instance only one because we want to use the same // one after a new problem (new seed). // stopwatch for measuring inference time for each problem Stopwatch watch = new Stopwatch(); Type logisticOp = typeof(LogisticOp2); LogisticOp2.Watch = watch; List <long> allInferTimes = new List <long>(); var allPosteriors = new List <VectorGaussian>(); LogisticOp2.IsCollectLogisticMessages = false; LogisticOp2.IsCollectProjMsgs = false; LogisticOp2.IsCollectXMessages = false; for (int seed = seed_from; seed <= seed_to; seed++) { Rand.Restart(seed); double b = 0; // combine the bias term into W Vector[] X; bool[] Y; LogisticRegression.GenData(n, w, b, out X, out Y, seed); Console.Write("Y: "); StringUtils.PrintArray(Y); VectorGaussian wPost; // start the watch watch.Restart(); LogisticRegression.InferCoefficientsNoBias(X, Y, out wPost, epIter, logisticOp); // stop the watch long inferenceTime = watch.ElapsedMilliseconds; allInferTimes.Add(inferenceTime); allPosteriors.Add(wPost); //print Console.WriteLine("n: {0}", n); Console.WriteLine("d: {0}", d); int t = Y.Sum(o => o ? 1 : 0); Console.WriteLine("number of true: {0}", t); Console.WriteLine("True bias: {0}", b); // Vector meanW = wPost.GetMean(); Console.WriteLine("True w: {0}", w); Console.WriteLine("Inferred w: "); Console.WriteLine(wPost); } string fnameM = string.Format("rec_dnet_n{0}_logistic_iter{1}_sf{2}_st{3}.mat", n, epIter, seed_from, seed_to); string recordPathM = Config.PathToSavedFile(fnameM); MatlabWriter writer = new MatlabWriter(recordPathM); writer.Write("allInferTimes", MatrixUtils.ToDouble(allInferTimes)); Vector[] postMeans = allPosteriors.Select(vg => vg.GetMean()).ToArray(); Matrix[] postCovs = allPosteriors.Select(vg => vg.GetVariance()).ToArray(); writer.Write("postMeans", postMeans); writer.Write("postCovs", postCovs); writer.Write("dim", d); writer.Write("n", n); writer.Write("epIter", epIter); writer.Write("seed_from", seed_from); writer.Write("seed_to", seed_to); writer.Write("init_fixed_seed", init_fixed_seed); writer.Dispose(); }
/**Run KJIT with an importance sampler as the oracle on a number of * UCI real datasets.*/ public void RunRealOnlineKEPSampling() { Rand.Restart(1); List <LogisticOpRecords> allRecs = new List <LogisticOpRecords>(); // Create the Logistic operator instance only once because we want to use the same // one after a new problem (new seed). // stopwatch for measuring inference time for each problem Stopwatch watch = new Stopwatch(); var logisticOpIns = new KEPOnlineISLogisticOpIns( new LogisticOpRecords(), watch, -8.95); logisticOpIns.SetOnlineBatchSizeTrigger(500); logisticOpIns.SetImportanceSamplingSize(importanceSamplingSize); logisticOpIns.IsRecordMessages = true; logisticOpIns.IsPrintTrueWhenCertain = false; // See BayesLinRegFM's BatchLearn() and KEPOnlineISLogisticOpIns() // If using the sum kernel logisticOpIns.SetFeatures(new int[] { 400, 800 }); OpControl.Set(typeof(KEPOnlineLogisticOp), logisticOpIns); Type logisticOp = typeof(KEPOnlineLogisticOp); List <long> allInferTimes = new List <long>(); List <long> allOraInferTimes = new List <long>(); var allPosteriors = new List <VectorGaussian>(); var allDotNetPosteriors = new List <VectorGaussian>(); LogisticOp2.IsCollectLogisticMessages = false; LogisticOp2.IsCollectProjMsgs = false; LogisticOp2.IsCollectXMessages = false; string folder = "online_uci/"; for (int i = 0; i < dataNames.Length; i++) { Console.WriteLine(); Console.WriteLine("----------- starting problem {0} --------------", dataAbbrv[i]); Console.WriteLine(); Vector[] X; bool[] Y; LoadDataFromMat(dataPaths[i], out X, out Y); Console.Write("Y: "); StringUtils.PrintArray(Y); VectorGaussian wPost; LogisticOpRecords recorder = new LogisticOpRecords(); // Set a new recorder for a new problem seed logisticOpIns.SetRecorder(recorder); // Type logisticOp = typeof(LogisticOp2); // start the watch watch.Restart(); // We do not include the bias term. So make sure the datasets // are standardized. LogisticRegression.InferCoefficientsNoBias(X, Y, out wPost, epIter, logisticOp); // stop the watch long inferenceTime = watch.ElapsedMilliseconds; recorder.inferenceTimes = new List <long>(); recorder.inferenceTimes.Add(inferenceTime); allInferTimes.Add(inferenceTime); recorder.postW = MatrixUtils.ToList(wPost); allPosteriors.Add(wPost); allRecs.Add(recorder); //print Console.WriteLine("n: {0}", n); Console.WriteLine("d: {0}", d); int t = Y.Sum(o => o ? 1 : 0); Console.WriteLine("number of true: {0}", t); // Vector meanW = wPost.GetMean(); Console.WriteLine("Inferred w: "); Console.WriteLine(wPost); // Run Infer.net's operator on the same data VectorGaussian dotNetPostW; Stopwatch oraWatch = new Stopwatch(); oraWatch.Start(); LogisticRegression.InferCoefficientsNoBias(X, Y, out dotNetPostW, epIter, typeof(LogisticOp2)); long oraInferTime = oraWatch.ElapsedMilliseconds; allOraInferTimes.Add(oraInferTime); recorder.dotNetPostW = MatrixUtils.ToList <VectorGaussian>(dotNetPostW); allDotNetPosteriors.Add(dotNetPostW); // write the records to a file string fname = string.Format("kjit_is{0}_{1}_iter{2}.mat", importanceSamplingSize, dataAbbrv[i], epIter); string recordPath = Config.PathToSavedFile(folder + fname); var extra = new Dictionary <string, object>(); // MatlabWriter cannot write int extra.Add("d", (double)X[0].Count); extra.Add("n", (double)X.Length); extra.Add("epIter", (double)epIter); extra.Add("X", MatrixUtils.StackColumns(X)); extra.Add("Y", MatrixUtils.ToDouble(Y)); recorder.WriteRecords(recordPath, extra); } // merge all records and write LogisticOpRecords merged = LogisticOpRecords.Merge(allRecs.ToArray()); merged.inferenceTimes = allInferTimes; merged.dotNetPostW = allDotNetPosteriors; merged.postW = allPosteriors; string fnameM = string.Format("kjit_is{0}_uci{1}_iter{2}.mat", importanceSamplingSize, dataAbbrv.Length, epIter); string recordPathM = Config.PathToSavedFile(folder + fnameM); var allExtra = new Dictionary <string, object>(); double[] oraInferTimesArr = allOraInferTimes.Select(t => (double)t).ToArray(); allExtra.Add("oraInferTimes", Vector.FromArray(oraInferTimesArr)); merged.WriteRecords(recordPathM, allExtra); }
public static void RunOnlineKEPDotNet() { // KJIT with Infer.NET oracle // Compile model only once. In this model, only one message can // be collected from one EP problem. Variable <int> dataCount = Variable.Observed(-1).Named("dataCount"); Range n = new Range(dataCount).Named("N"); Variable <double> precision = Variable <double> .Factor(CGFac.FromCompoundGamma); precision.AddAttribute(new TraceMessages()); precision.AddAttribute(new MarginalPrototype(new Gamma())); VariableArray <double> x = Variable.Array <double>(n).Named("X"); x[n] = Variable.GaussianFromMeanAndPrecision(GaussMean, precision).ForEach(n); // Create the operator instance only once because we want to use the same // one after a new problem (new seed). // stopwatch for measuring inference time for each problem Stopwatch watch = new Stopwatch(); CGOpRecords record = new CGOpRecords(); var opIns = new KEP_CGFacOpIns(onlineBatchSizeTrigger, record, watch, initialThreshold); opIns.IsPrintTrueWhenCertain = true; opIns.IsRecordMessages = true; //---- records ----- List <long> allInferTimes = new List <long>(); List <long> allOracleInferTimes = new List <long>(); // List<Dictionary<string, object>> allExtras = new List<Dictionary<string, object>>(); var allPosteriors = new List <Gamma>(); var allOraclePosteriors = new List <Gamma>(); int[] Ns = new int[seed_to]; double[] trueRate2s = new double[seed_to]; double[] truePrecs = new double[seed_to]; List <double[]> allObs = new List <double[]>(); //--------------- OpControl.Set(typeof(KEP_CGFacOp), opIns); // one engine means the initial model compilation is done only once InferenceEngine ie = NewInferenceEngine("KJIT_CG"); ie.Compiler.GivePriorityTo(typeof(KEP_CGFacOp)); // engine for the oracle InferenceEngine dnetIe = NewInferenceEngine("Oracle_CG"); dnetIe.Compiler.GivePriorityTo(typeof(CGFacOp)); Random precRand = new Random(1); for (int seed = 1; seed <= seed_to; seed++) { int i = seed - 1; Rand.Restart(seed); // n is between 10 and 100 for. Could be different for each seed. int N = Rand.Int(10, 100 + 1); Console.WriteLine("\n ///// New compound Gamma problem {0} of size: {1} /////\n", seed, N); Ns[i] = N; CompoundGamma cg = new CompoundGamma(); double trueR2, truePrec; double[] obs; cg.GenData(N, seed, out obs, out trueR2, out truePrec); // Draw a number uniformly from log(1e-4) // to log(1e4) and exp(.) it to get the precision. // double logPrec = precRand.NextDouble()*(Math.Log(1e4) - Math.Log(1e-4)) + Math.Log(1e-4); // double truePrec = Math.Exp(logPrec); // double[] obs = CompoundGamma.DrawFromGaussian(GaussMean, truePrec, N); allObs.Add(obs); // trueRate2s[i] = trueR2; truePrecs[i] = truePrec; dataCount.ObservedValue = N; x.ObservedValue = obs; // Console.Write("observations: "); // StringUtils.PrintArray(obs); // ie.Compiler.UseParallelForLoops = true; watch.Restart(); Gamma postPrec = ie.Infer <Gamma>(precision); long inferenceTime = watch.ElapsedMilliseconds; allInferTimes.Add(inferenceTime); allPosteriors.Add(postPrec); // // Run Infer.net's operator on the same data Stopwatch dnetWatch = new Stopwatch(); // dataCount.ObservedValue = N; x.ObservedValue = obs; dnetWatch.Start(); Gamma dnetPost = dnetIe.Infer <Gamma>(precision); long dnetTime = dnetWatch.ElapsedMilliseconds; allOracleInferTimes.Add(dnetTime); allOraclePosteriors.Add(dnetPost); //print Console.WriteLine("seed: {0}", seed); Console.WriteLine("n: {0}", N); // Console.WriteLine("True r2: {0}", trueR2); Console.WriteLine("True precision: {0}", truePrec); Console.WriteLine("Inferred precision posterior: {0}", postPrec); Console.WriteLine("Infer.NET posterior: {0}", dnetPost); Console.WriteLine("Inference time: {0} ms", inferenceTime); Console.WriteLine("Infer.NET time: {0} ms", dnetTime); Console.WriteLine("========================="); Console.WriteLine(); } // MatlabWriter cannot write int var extra = new Dictionary <string, object>(); extra.Add("inferTimes", MatrixUtils.ToDouble(allInferTimes)); extra.Add("oraInferTimes", MatrixUtils.ToDouble(allOracleInferTimes)); double[] postShapes, postRates; double[] oraPostShapes, oraPostRates; CGOpRecords.GammaListToArrays(allPosteriors, out postShapes, out postRates); CGOpRecords.GammaListToArrays(allOraclePosteriors, out oraPostShapes, out oraPostRates); extra.Add("postShapes", postShapes); extra.Add("postRates", postRates); extra.Add("oraPostShapes", oraPostShapes); extra.Add("oraPostRates", oraPostRates); extra.Add("Ns", MatrixUtils.ToDoubleArray(Ns)); extra.Add("trueRate2s", trueRate2s); extra.Add("truePrecs", truePrecs); // MatlabWriter cannot write List<Vector> or List<double[]>. Why ? // List<Vector> allObsVec = allObs.Select(ob => Vector.FromArray(ob)).ToList(); // extra.Add("allObs", allObsVec); // write the records to a file string fname = string.Format("kjit_cg_iter{0}_bt{1}_st{2}.mat", epIter, onlineBatchSizeTrigger, seed_to); string recordPath = Config.PathToSavedFile(fname); record.WriteRecords(recordPath, extra); }