public static void TestDeterminant()
{
Matrix m1 = Matrix.Parse("6 0 \n 0 1");
Matrix m2 = Matrix.Parse("3.0 2.0 \n 1.5 3.0");
Matrix m3 = Matrix.Parse("0.7493 0.5074\n -0.004 -0.4204");
Matrix m4 = PositiveDefiniteMatrix.Identity(4);
Console.WriteLine(m1);
// expect 6
Console.WriteLine("m1 det: {0}", m1.Determinant());
// expect 6
Console.WriteLine("m2 det: {0}", m2.Determinant());
// expect -0.313
Console.WriteLine("m3 det: {0}", m3.Determinant());
Console.WriteLine("m4 det: {0}", m4.Determinant());
// expect 6
Console.WriteLine("m1 my det: {0}", MatrixUtils.Determinant(m1));
// expect 6
Console.WriteLine("m2 my det: {0}", MatrixUtils.Determinant(m2));
// expect -0.313
Console.WriteLine("m3 my det: {0}", MatrixUtils.Determinant(m3));
Console.WriteLine("m4 my det: {0}", MatrixUtils.Determinant(m4));
// Use MathNet
MNMatrix m5 = MNMatrix.Build.Dense(2, 2, new double[] { 6, 0, 0, 1 });
Console.WriteLine(m5);
Console.WriteLine("m5 det: {0}", m5.Determinant());
}
//---------
public static void TestBlkDiag()
{
Matrix m1 = Matrix.Parse("1 2\n 3 4");
Matrix m2 = Matrix.Parse("5 6 7\n 8 9 10 ");
Matrix m3 = Matrix.Parse("11\n12");
Matrix m4 = Matrix.Parse("13 14");
Matrix big = MatrixUtils.BlkDiag(m1, m2, m3, m4);
Console.WriteLine(big);
}
public static void TestMathNetInverse()
{
// http://numerics.mathdotnet.com/Matrix.html
// inverse is
//
// ans =
//
// 0.5000 -0.3333
// -0.2500 0.5000
Matrix inferMat = Matrix.Parse("3 2 \n 1.5 3");
Console.WriteLine(inferMat);
Matrix inv = MatrixUtils.Inverse(inferMat);
Console.WriteLine("Inverse: ");
Console.WriteLine(inv);
}
public void LoadDataFromMat(string filePath, out Vector[] X, out bool[] Y)
{
// Expect X (dxn) and Y (1xn) in the mat file
Dictionary <string, object> dict = MatlabReader.Read(filePath);
Matrix xmat = (Matrix)dict["X"];
X = MatrixUtils.SplitColumns(xmat);
int n = X.Length;
// Y is actually a 1xn vector
Matrix ymat = (Matrix)dict["Y"];
double[,] yarr = ymat.ToArray();
if (yarr.GetLength(1) != n)
{
throw new ArgumentException("length of X and Y do not match.");
}
Y = new bool[n];
for (int i = 0; i < n; i++)
{
// expect 0, 1
Y[i] = yarr[0, i] > 0.5;
}
}
public void RunOnlineKEPSampling()
{
// Kernel EP with importance sampling.
/**
* Only one W just like in Ali's paper.
* In practice, we typically observe multiple sets of observations
* where we want to do inference on the same model with the same
* parameter.
*/
Rand.Restart(init_fixed_seed);
Vector w = Vector.Zero(d);
Rand.Normal(Vector.Zero(d), PositiveDefiniteMatrix.Identity(d), w);
List <LogisticOpRecords> allRecs = new List <LogisticOpRecords>();
// Create the Logistic operator instance only one because we want to use the same
// one after a new problem (new seed).
// stopwatch for measuring inference time for each problem
Stopwatch watch = new Stopwatch();
var logisticOpIns = new KEPOnlineISLogisticOpIns(
new LogisticOpRecords(), watch, -8.5);
logisticOpIns.SetImportanceSamplingSize(importanceSamplingSize);
logisticOpIns.IsRecordMessages = true;
logisticOpIns.IsPrintTrueWhenCertain = false;
/** Use mixture or not ...*/
logisticOpIns.isGaussianOp.useMixtureProposal = false;
logisticOpIns.SetFeatures(new int[] { 300, 500 });
OpControl.Set(typeof(KEPOnlineLogisticOp), logisticOpIns);
Type logisticOp = typeof(KEPOnlineLogisticOp);
List <long> allInferTimes = new List <long>();
var allPosteriors = new List <VectorGaussian>();
var allDotNetPosteriors = new List <VectorGaussian>();
LogisticOp2.IsCollectLogisticMessages = false;
LogisticOp2.IsCollectProjMsgs = false;
LogisticOp2.IsCollectXMessages = false;
for (int seed = seed_from; seed <= seed_to; seed++)
{
Rand.Restart(seed);
double b = 0;
// combine the bias term into W
Vector[] X;
bool[] Y;
LogisticRegression.GenData(n, w, b, out X, out Y, seed);
Console.Write("Y: ");
StringUtils.PrintArray(Y);
VectorGaussian wPost;
LogisticOpRecords recorder = new LogisticOpRecords();
// Set a new recorder for a new problem seed
logisticOpIns.SetRecorder(recorder);
// Type logisticOp = typeof(LogisticOp2);
// start the watch
watch.Restart();
LogisticRegression.InferCoefficientsNoBias(X, Y, out wPost, epIter, logisticOp);
// stop the watch
long inferenceTime = watch.ElapsedMilliseconds;
recorder.inferenceTimes = new List <long>();
recorder.inferenceTimes.Add(inferenceTime);
allInferTimes.Add(inferenceTime);
recorder.postW = MatrixUtils.ToList(wPost);
allPosteriors.Add(wPost);
allRecs.Add(recorder);
//print
Console.WriteLine("n: {0}", n);
Console.WriteLine("d: {0}", d);
int t = Y.Sum(o => o ? 1 : 0);
Console.WriteLine("number of true: {0}", t);
Console.WriteLine("True bias: {0}", b);
// Vector meanW = wPost.GetMean();
Console.WriteLine("True w: {0}", w);
Console.WriteLine("Inferred w: ");
Console.WriteLine(wPost);
// Run Infer.net's operator on the same data
VectorGaussian dotNetPostW;
LogisticRegression.InferCoefficientsNoBias(X, Y, out dotNetPostW,
epIter, typeof(LogisticOp2));
recorder.dotNetPostW = MatrixUtils.ToList <VectorGaussian>(dotNetPostW);
allDotNetPosteriors.Add(dotNetPostW);
// write the records to a file
string fname = string.Format("rec_onlinekep_is{0}_n{1}_logistic_iter{2}_s{3}.mat",
importanceSamplingSize, n, epIter, seed);
string recordPath = Config.PathToSavedFile(fname);
var extra = new Dictionary <string, object>();
// MatlabWriter cannot write int
extra.Add("d", (double)d);
extra.Add("n", (double)n);
extra.Add("epIter", (double)epIter);
extra.Add("trueW", w);
extra.Add("X", MatrixUtils.StackColumns(X));
extra.Add("Y", MatrixUtils.ToDouble(Y));
recorder.WriteRecords(recordPath, extra);
}
// merge all records and write
LogisticOpRecords merged = LogisticOpRecords.Merge(allRecs.ToArray());
merged.inferenceTimes = allInferTimes;
merged.dotNetPostW = allDotNetPosteriors;
merged.postW = allPosteriors;
string fnameM = string.Format("rec_onlinekep_is{0}_n{1}_logistic_iter{2}_sf{3}_st{4}.mat",
importanceSamplingSize, n, epIter, seed_from, seed_to);
string recordPathM = Config.PathToSavedFile(fnameM);
merged.WriteRecords(recordPathM);
}
public void RecordInferNETTime()
{
/**Records time by infer.net*/
/**
* Only one W just like in Ali's paper.
* In practice, we typically observe multiple sets of observations
* where we want to do inference on the same model with the same
* parameter.
*/
Rand.Restart(init_fixed_seed);
Vector w = Vector.Zero(d);
Rand.Normal(Vector.Zero(d), PositiveDefiniteMatrix.Identity(d), w);
// Create the Logistic operator instance only one because we want to use the same
// one after a new problem (new seed).
// stopwatch for measuring inference time for each problem
Stopwatch watch = new Stopwatch();
Type logisticOp = typeof(LogisticOp2);
LogisticOp2.Watch = watch;
List <long> allInferTimes = new List <long>();
var allPosteriors = new List <VectorGaussian>();
LogisticOp2.IsCollectLogisticMessages = false;
LogisticOp2.IsCollectProjMsgs = false;
LogisticOp2.IsCollectXMessages = false;
for (int seed = seed_from; seed <= seed_to; seed++)
{
Rand.Restart(seed);
double b = 0;
// combine the bias term into W
Vector[] X;
bool[] Y;
LogisticRegression.GenData(n, w, b, out X, out Y, seed);
Console.Write("Y: ");
StringUtils.PrintArray(Y);
VectorGaussian wPost;
// start the watch
watch.Restart();
LogisticRegression.InferCoefficientsNoBias(X, Y, out wPost, epIter, logisticOp);
// stop the watch
long inferenceTime = watch.ElapsedMilliseconds;
allInferTimes.Add(inferenceTime);
allPosteriors.Add(wPost);
//print
Console.WriteLine("n: {0}", n);
Console.WriteLine("d: {0}", d);
int t = Y.Sum(o => o ? 1 : 0);
Console.WriteLine("number of true: {0}", t);
Console.WriteLine("True bias: {0}", b);
// Vector meanW = wPost.GetMean();
Console.WriteLine("True w: {0}", w);
Console.WriteLine("Inferred w: ");
Console.WriteLine(wPost);
}
string fnameM = string.Format("rec_dnet_n{0}_logistic_iter{1}_sf{2}_st{3}.mat",
n, epIter, seed_from, seed_to);
string recordPathM = Config.PathToSavedFile(fnameM);
MatlabWriter writer = new MatlabWriter(recordPathM);
writer.Write("allInferTimes", MatrixUtils.ToDouble(allInferTimes));
Vector[] postMeans = allPosteriors.Select(vg => vg.GetMean()).ToArray();
Matrix[] postCovs = allPosteriors.Select(vg => vg.GetVariance()).ToArray();
writer.Write("postMeans", postMeans);
writer.Write("postCovs", postCovs);
writer.Write("dim", d);
writer.Write("n", n);
writer.Write("epIter", epIter);
writer.Write("seed_from", seed_from);
writer.Write("seed_to", seed_to);
writer.Write("init_fixed_seed", init_fixed_seed);
writer.Dispose();
}
/**Run KJIT with an importance sampler as the oracle on a number of
* UCI real datasets.*/
public void RunRealOnlineKEPSampling()
{
Rand.Restart(1);
List <LogisticOpRecords> allRecs = new List <LogisticOpRecords>();
// Create the Logistic operator instance only once because we want to use the same
// one after a new problem (new seed).
// stopwatch for measuring inference time for each problem
Stopwatch watch = new Stopwatch();
var logisticOpIns = new KEPOnlineISLogisticOpIns(
new LogisticOpRecords(), watch, -8.95);
logisticOpIns.SetOnlineBatchSizeTrigger(500);
logisticOpIns.SetImportanceSamplingSize(importanceSamplingSize);
logisticOpIns.IsRecordMessages = true;
logisticOpIns.IsPrintTrueWhenCertain = false;
// See BayesLinRegFM's BatchLearn() and KEPOnlineISLogisticOpIns()
// If using the sum kernel
logisticOpIns.SetFeatures(new int[] { 400, 800 });
OpControl.Set(typeof(KEPOnlineLogisticOp), logisticOpIns);
Type logisticOp = typeof(KEPOnlineLogisticOp);
List <long> allInferTimes = new List <long>();
List <long> allOraInferTimes = new List <long>();
var allPosteriors = new List <VectorGaussian>();
var allDotNetPosteriors = new List <VectorGaussian>();
LogisticOp2.IsCollectLogisticMessages = false;
LogisticOp2.IsCollectProjMsgs = false;
LogisticOp2.IsCollectXMessages = false;
string folder = "online_uci/";
for (int i = 0; i < dataNames.Length; i++)
{
Console.WriteLine();
Console.WriteLine("----------- starting problem {0} --------------", dataAbbrv[i]);
Console.WriteLine();
Vector[] X;
bool[] Y;
LoadDataFromMat(dataPaths[i], out X, out Y);
Console.Write("Y: ");
StringUtils.PrintArray(Y);
VectorGaussian wPost;
LogisticOpRecords recorder = new LogisticOpRecords();
// Set a new recorder for a new problem seed
logisticOpIns.SetRecorder(recorder);
// Type logisticOp = typeof(LogisticOp2);
// start the watch
watch.Restart();
// We do not include the bias term. So make sure the datasets
// are standardized.
LogisticRegression.InferCoefficientsNoBias(X, Y, out wPost, epIter, logisticOp);
// stop the watch
long inferenceTime = watch.ElapsedMilliseconds;
recorder.inferenceTimes = new List <long>();
recorder.inferenceTimes.Add(inferenceTime);
allInferTimes.Add(inferenceTime);
recorder.postW = MatrixUtils.ToList(wPost);
allPosteriors.Add(wPost);
allRecs.Add(recorder);
//print
Console.WriteLine("n: {0}", n);
Console.WriteLine("d: {0}", d);
int t = Y.Sum(o => o ? 1 : 0);
Console.WriteLine("number of true: {0}", t);
// Vector meanW = wPost.GetMean();
Console.WriteLine("Inferred w: ");
Console.WriteLine(wPost);
// Run Infer.net's operator on the same data
VectorGaussian dotNetPostW;
Stopwatch oraWatch = new Stopwatch();
oraWatch.Start();
LogisticRegression.InferCoefficientsNoBias(X, Y, out dotNetPostW,
epIter, typeof(LogisticOp2));
long oraInferTime = oraWatch.ElapsedMilliseconds;
allOraInferTimes.Add(oraInferTime);
recorder.dotNetPostW = MatrixUtils.ToList <VectorGaussian>(dotNetPostW);
allDotNetPosteriors.Add(dotNetPostW);
// write the records to a file
string fname = string.Format("kjit_is{0}_{1}_iter{2}.mat",
importanceSamplingSize, dataAbbrv[i], epIter);
string recordPath = Config.PathToSavedFile(folder + fname);
var extra = new Dictionary <string, object>();
// MatlabWriter cannot write int
extra.Add("d", (double)X[0].Count);
extra.Add("n", (double)X.Length);
extra.Add("epIter", (double)epIter);
extra.Add("X", MatrixUtils.StackColumns(X));
extra.Add("Y", MatrixUtils.ToDouble(Y));
recorder.WriteRecords(recordPath, extra);
}
// merge all records and write
LogisticOpRecords merged = LogisticOpRecords.Merge(allRecs.ToArray());
merged.inferenceTimes = allInferTimes;
merged.dotNetPostW = allDotNetPosteriors;
merged.postW = allPosteriors;
string fnameM = string.Format("kjit_is{0}_uci{1}_iter{2}.mat",
importanceSamplingSize, dataAbbrv.Length, epIter);
string recordPathM = Config.PathToSavedFile(folder + fnameM);
var allExtra = new Dictionary <string, object>();
double[] oraInferTimesArr = allOraInferTimes.Select(t => (double)t).ToArray();
allExtra.Add("oraInferTimes", Vector.FromArray(oraInferTimesArr));
merged.WriteRecords(recordPathM, allExtra);
}
public static void RunOnlineKEPDotNet()
{
// KJIT with Infer.NET oracle
// Compile model only once. In this model, only one message can
// be collected from one EP problem.
Variable <int> dataCount = Variable.Observed(-1).Named("dataCount");
Range n = new Range(dataCount).Named("N");
Variable <double> precision = Variable <double> .Factor(CGFac.FromCompoundGamma);
precision.AddAttribute(new TraceMessages());
precision.AddAttribute(new MarginalPrototype(new Gamma()));
VariableArray <double> x = Variable.Array <double>(n).Named("X");
x[n] = Variable.GaussianFromMeanAndPrecision(GaussMean, precision).ForEach(n);
// Create the operator instance only once because we want to use the same
// one after a new problem (new seed).
// stopwatch for measuring inference time for each problem
Stopwatch watch = new Stopwatch();
CGOpRecords record = new CGOpRecords();
var opIns = new KEP_CGFacOpIns(onlineBatchSizeTrigger, record, watch, initialThreshold);
opIns.IsPrintTrueWhenCertain = true;
opIns.IsRecordMessages = true;
//---- records -----
List <long> allInferTimes = new List <long>();
List <long> allOracleInferTimes = new List <long>();
// List<Dictionary<string, object>> allExtras = new List<Dictionary<string, object>>();
var allPosteriors = new List <Gamma>();
var allOraclePosteriors = new List <Gamma>();
int[] Ns = new int[seed_to];
double[] trueRate2s = new double[seed_to];
double[] truePrecs = new double[seed_to];
List <double[]> allObs = new List <double[]>();
//---------------
OpControl.Set(typeof(KEP_CGFacOp), opIns);
// one engine means the initial model compilation is done only once
InferenceEngine ie = NewInferenceEngine("KJIT_CG");
ie.Compiler.GivePriorityTo(typeof(KEP_CGFacOp));
// engine for the oracle
InferenceEngine dnetIe = NewInferenceEngine("Oracle_CG");
dnetIe.Compiler.GivePriorityTo(typeof(CGFacOp));
Random precRand = new Random(1);
for (int seed = 1; seed <= seed_to; seed++)
{
int i = seed - 1;
Rand.Restart(seed);
// n is between 10 and 100 for. Could be different for each seed.
int N = Rand.Int(10, 100 + 1);
Console.WriteLine("\n ///// New compound Gamma problem {0} of size: {1} /////\n",
seed, N);
Ns[i] = N;
CompoundGamma cg = new CompoundGamma();
double trueR2, truePrec;
double[] obs;
cg.GenData(N, seed, out obs, out trueR2, out truePrec);
// Draw a number uniformly from log(1e-4)
// to log(1e4) and exp(.) it to get the precision.
// double logPrec = precRand.NextDouble()*(Math.Log(1e4) - Math.Log(1e-4)) + Math.Log(1e-4);
// double truePrec = Math.Exp(logPrec);
// double[] obs = CompoundGamma.DrawFromGaussian(GaussMean, truePrec, N);
allObs.Add(obs);
// trueRate2s[i] = trueR2;
truePrecs[i] = truePrec;
dataCount.ObservedValue = N;
x.ObservedValue = obs;
// Console.Write("observations: ");
// StringUtils.PrintArray(obs);
// ie.Compiler.UseParallelForLoops = true;
watch.Restart();
Gamma postPrec = ie.Infer <Gamma>(precision);
long inferenceTime = watch.ElapsedMilliseconds;
allInferTimes.Add(inferenceTime);
allPosteriors.Add(postPrec);
// // Run Infer.net's operator on the same data
Stopwatch dnetWatch = new Stopwatch();
//
dataCount.ObservedValue = N;
x.ObservedValue = obs;
dnetWatch.Start();
Gamma dnetPost = dnetIe.Infer <Gamma>(precision);
long dnetTime = dnetWatch.ElapsedMilliseconds;
allOracleInferTimes.Add(dnetTime);
allOraclePosteriors.Add(dnetPost);
//print
Console.WriteLine("seed: {0}", seed);
Console.WriteLine("n: {0}", N);
// Console.WriteLine("True r2: {0}", trueR2);
Console.WriteLine("True precision: {0}", truePrec);
Console.WriteLine("Inferred precision posterior: {0}", postPrec);
Console.WriteLine("Infer.NET posterior: {0}", dnetPost);
Console.WriteLine("Inference time: {0} ms", inferenceTime);
Console.WriteLine("Infer.NET time: {0} ms", dnetTime);
Console.WriteLine("=========================");
Console.WriteLine();
}
// MatlabWriter cannot write int
var extra = new Dictionary <string, object>();
extra.Add("inferTimes", MatrixUtils.ToDouble(allInferTimes));
extra.Add("oraInferTimes", MatrixUtils.ToDouble(allOracleInferTimes));
double[] postShapes, postRates;
double[] oraPostShapes, oraPostRates;
CGOpRecords.GammaListToArrays(allPosteriors, out postShapes, out postRates);
CGOpRecords.GammaListToArrays(allOraclePosteriors, out oraPostShapes, out oraPostRates);
extra.Add("postShapes", postShapes);
extra.Add("postRates", postRates);
extra.Add("oraPostShapes", oraPostShapes);
extra.Add("oraPostRates", oraPostRates);
extra.Add("Ns", MatrixUtils.ToDoubleArray(Ns));
extra.Add("trueRate2s", trueRate2s);
extra.Add("truePrecs", truePrecs);
// MatlabWriter cannot write List<Vector> or List<double[]>. Why ?
// List<Vector> allObsVec = allObs.Select(ob => Vector.FromArray(ob)).ToList();
// extra.Add("allObs", allObsVec);
// write the records to a file
string fname = string.Format("kjit_cg_iter{0}_bt{1}_st{2}.mat",
epIter, onlineBatchSizeTrigger, seed_to);
string recordPath = Config.PathToSavedFile(fname);
record.WriteRecords(recordPath, extra);
}