public static void TestLogisticRegression()
{
const int seed = 39;
Rand.Restart(seed);
const int d = 10;
const int n = 100;
const int epIter = 10;
Vector w = Vector.Zero(d);
Rand.Normal(Vector.Zero(d), PositiveDefiniteMatrix.Identity(d), w);
double b = Rand.Normal(0, 1);
// double b= 0;
Vector[] X;
bool[] Y;
GenData(n, w, b, out X, out Y);
Console.Write("Y: ");
StringUtils.PrintArray(Y);
VectorGaussian wPost;
Gaussian biasPost;
Type logisticOp = typeof(KEPLogisticOp);
// Type logisticOp = typeof(LogisticOp2);
string factorOpPath = Config.PathToFactorOperator(
// "serialFactorOp_fm_kgg_joint_irf500_orf1000_n400_iter5_sf1_st20_ntr5000.mat"
"serialFactorOp_fm_kgg_joint_irf500_orf1000_proj_n400_iter5_sf1_st20_ntr5000.mat"
);
KEPLogisticOpInstance opIns = KEPLogisticOpInstance.LoadLogisticOpInstance(factorOpPath);
opIns.SetPrintTrueMessages(true);
OpControl.Add(typeof(KEPLogisticOp), opIns);
InferCoefficients(X, Y, out wPost, out biasPost, epIter, logisticOp);
//print
Console.WriteLine("n: {0}", n);
Console.WriteLine("d: {0}", d);
int t = Y.Sum(o => o ? 1 : 0);
Console.WriteLine("number of true: {0}", t);
Console.WriteLine("True bias: {0}", b);
Console.WriteLine("Inferred bias: {0}", biasPost);
Console.WriteLine("True w: {0}", w);
Console.WriteLine("Inferred w: ");
Console.WriteLine(wPost);
}
/// <summary>
/// The constructor for a MIPS machine. Takes Memory size and Instruction ROM size.
/// </summary>
/// <param name="memSize"></param>
/// <param name="instrSize"></param>
public MIPSArch(uint memSize, uint instrSize)
{
PC = 0;
MEM = new DataMemory(memSize);
InstrMEM = new IMemory(instrSize);
IR = new IRegister();
REG = new Registry();
ALUnit = new ALU();
SGNEXT = new SignExtend();
Add = new ADD();
Control = new OpControl();
ALControl = new ALUControl();
BranchGate = new AND();
FetchBuffer = new FetchDecodeBuffer[] { new FetchDecodeBuffer(), new FetchDecodeBuffer() };
DecodeBuffer = new DecodeExecuteBuffer[] { new DecodeExecuteBuffer(), new DecodeExecuteBuffer() };
ExecuteBuffer = new ExecuteMEMBuffer[] { new ExecuteMEMBuffer(), new ExecuteMEMBuffer() };
MEMBuffer = new MEMWBBuffer[] { new MEMWBBuffer(), new MEMWBBuffer() };
MAX_PC = new MIPSLoader(MEM, InstrMEM, REG).Initialize();
CLI = new CUI(ref InstrMEM, ref MEM, ref REG);
}
public void RunOnlineKEPSampling()
{
// Kernel EP with importance sampling.
/**
* Only one W just like in Ali's paper.
* In practice, we typically observe multiple sets of observations
* where we want to do inference on the same model with the same
* parameter.
*/
Rand.Restart(init_fixed_seed);
Vector w = Vector.Zero(d);
Rand.Normal(Vector.Zero(d), PositiveDefiniteMatrix.Identity(d), w);
List <LogisticOpRecords> allRecs = new List <LogisticOpRecords>();
// Create the Logistic operator instance only one because we want to use the same
// one after a new problem (new seed).
// stopwatch for measuring inference time for each problem
Stopwatch watch = new Stopwatch();
var logisticOpIns = new KEPOnlineISLogisticOpIns(
new LogisticOpRecords(), watch, -8.5);
logisticOpIns.SetImportanceSamplingSize(importanceSamplingSize);
logisticOpIns.IsRecordMessages = true;
logisticOpIns.IsPrintTrueWhenCertain = false;
/** Use mixture or not ...*/
logisticOpIns.isGaussianOp.useMixtureProposal = false;
logisticOpIns.SetFeatures(new int[] { 300, 500 });
OpControl.Set(typeof(KEPOnlineLogisticOp), logisticOpIns);
Type logisticOp = typeof(KEPOnlineLogisticOp);
List <long> allInferTimes = new List <long>();
var allPosteriors = new List <VectorGaussian>();
var allDotNetPosteriors = new List <VectorGaussian>();
LogisticOp2.IsCollectLogisticMessages = false;
LogisticOp2.IsCollectProjMsgs = false;
LogisticOp2.IsCollectXMessages = false;
for (int seed = seed_from; seed <= seed_to; seed++)
{
Rand.Restart(seed);
double b = 0;
// combine the bias term into W
Vector[] X;
bool[] Y;
LogisticRegression.GenData(n, w, b, out X, out Y, seed);
Console.Write("Y: ");
StringUtils.PrintArray(Y);
VectorGaussian wPost;
LogisticOpRecords recorder = new LogisticOpRecords();
// Set a new recorder for a new problem seed
logisticOpIns.SetRecorder(recorder);
// Type logisticOp = typeof(LogisticOp2);
// start the watch
watch.Restart();
LogisticRegression.InferCoefficientsNoBias(X, Y, out wPost, epIter, logisticOp);
// stop the watch
long inferenceTime = watch.ElapsedMilliseconds;
recorder.inferenceTimes = new List <long>();
recorder.inferenceTimes.Add(inferenceTime);
allInferTimes.Add(inferenceTime);
recorder.postW = MatrixUtils.ToList(wPost);
allPosteriors.Add(wPost);
allRecs.Add(recorder);
//print
Console.WriteLine("n: {0}", n);
Console.WriteLine("d: {0}", d);
int t = Y.Sum(o => o ? 1 : 0);
Console.WriteLine("number of true: {0}", t);
Console.WriteLine("True bias: {0}", b);
// Vector meanW = wPost.GetMean();
Console.WriteLine("True w: {0}", w);
Console.WriteLine("Inferred w: ");
Console.WriteLine(wPost);
// Run Infer.net's operator on the same data
VectorGaussian dotNetPostW;
LogisticRegression.InferCoefficientsNoBias(X, Y, out dotNetPostW,
epIter, typeof(LogisticOp2));
recorder.dotNetPostW = MatrixUtils.ToList <VectorGaussian>(dotNetPostW);
allDotNetPosteriors.Add(dotNetPostW);
// write the records to a file
string fname = string.Format("rec_onlinekep_is{0}_n{1}_logistic_iter{2}_s{3}.mat",
importanceSamplingSize, n, epIter, seed);
string recordPath = Config.PathToSavedFile(fname);
var extra = new Dictionary <string, object>();
// MatlabWriter cannot write int
extra.Add("d", (double)d);
extra.Add("n", (double)n);
extra.Add("epIter", (double)epIter);
extra.Add("trueW", w);
extra.Add("X", MatrixUtils.StackColumns(X));
extra.Add("Y", MatrixUtils.ToDouble(Y));
recorder.WriteRecords(recordPath, extra);
}
// merge all records and write
LogisticOpRecords merged = LogisticOpRecords.Merge(allRecs.ToArray());
merged.inferenceTimes = allInferTimes;
merged.dotNetPostW = allDotNetPosteriors;
merged.postW = allPosteriors;
string fnameM = string.Format("rec_onlinekep_is{0}_n{1}_logistic_iter{2}_sf{3}_st{4}.mat",
importanceSamplingSize, n, epIter, seed_from, seed_to);
string recordPathM = Config.PathToSavedFile(fnameM);
merged.WriteRecords(recordPathM);
}
/**Run KJIT with an importance sampler as the oracle on a number of
* UCI real datasets.*/
public void RunRealOnlineKEPSampling()
{
Rand.Restart(1);
List <LogisticOpRecords> allRecs = new List <LogisticOpRecords>();
// Create the Logistic operator instance only once because we want to use the same
// one after a new problem (new seed).
// stopwatch for measuring inference time for each problem
Stopwatch watch = new Stopwatch();
var logisticOpIns = new KEPOnlineISLogisticOpIns(
new LogisticOpRecords(), watch, -8.95);
logisticOpIns.SetOnlineBatchSizeTrigger(500);
logisticOpIns.SetImportanceSamplingSize(importanceSamplingSize);
logisticOpIns.IsRecordMessages = true;
logisticOpIns.IsPrintTrueWhenCertain = false;
// See BayesLinRegFM's BatchLearn() and KEPOnlineISLogisticOpIns()
// If using the sum kernel
logisticOpIns.SetFeatures(new int[] { 400, 800 });
OpControl.Set(typeof(KEPOnlineLogisticOp), logisticOpIns);
Type logisticOp = typeof(KEPOnlineLogisticOp);
List <long> allInferTimes = new List <long>();
List <long> allOraInferTimes = new List <long>();
var allPosteriors = new List <VectorGaussian>();
var allDotNetPosteriors = new List <VectorGaussian>();
LogisticOp2.IsCollectLogisticMessages = false;
LogisticOp2.IsCollectProjMsgs = false;
LogisticOp2.IsCollectXMessages = false;
string folder = "online_uci/";
for (int i = 0; i < dataNames.Length; i++)
{
Console.WriteLine();
Console.WriteLine("----------- starting problem {0} --------------", dataAbbrv[i]);
Console.WriteLine();
Vector[] X;
bool[] Y;
LoadDataFromMat(dataPaths[i], out X, out Y);
Console.Write("Y: ");
StringUtils.PrintArray(Y);
VectorGaussian wPost;
LogisticOpRecords recorder = new LogisticOpRecords();
// Set a new recorder for a new problem seed
logisticOpIns.SetRecorder(recorder);
// Type logisticOp = typeof(LogisticOp2);
// start the watch
watch.Restart();
// We do not include the bias term. So make sure the datasets
// are standardized.
LogisticRegression.InferCoefficientsNoBias(X, Y, out wPost, epIter, logisticOp);
// stop the watch
long inferenceTime = watch.ElapsedMilliseconds;
recorder.inferenceTimes = new List <long>();
recorder.inferenceTimes.Add(inferenceTime);
allInferTimes.Add(inferenceTime);
recorder.postW = MatrixUtils.ToList(wPost);
allPosteriors.Add(wPost);
allRecs.Add(recorder);
//print
Console.WriteLine("n: {0}", n);
Console.WriteLine("d: {0}", d);
int t = Y.Sum(o => o ? 1 : 0);
Console.WriteLine("number of true: {0}", t);
// Vector meanW = wPost.GetMean();
Console.WriteLine("Inferred w: ");
Console.WriteLine(wPost);
// Run Infer.net's operator on the same data
VectorGaussian dotNetPostW;
Stopwatch oraWatch = new Stopwatch();
oraWatch.Start();
LogisticRegression.InferCoefficientsNoBias(X, Y, out dotNetPostW,
epIter, typeof(LogisticOp2));
long oraInferTime = oraWatch.ElapsedMilliseconds;
allOraInferTimes.Add(oraInferTime);
recorder.dotNetPostW = MatrixUtils.ToList <VectorGaussian>(dotNetPostW);
allDotNetPosteriors.Add(dotNetPostW);
// write the records to a file
string fname = string.Format("kjit_is{0}_{1}_iter{2}.mat",
importanceSamplingSize, dataAbbrv[i], epIter);
string recordPath = Config.PathToSavedFile(folder + fname);
var extra = new Dictionary <string, object>();
// MatlabWriter cannot write int
extra.Add("d", (double)X[0].Count);
extra.Add("n", (double)X.Length);
extra.Add("epIter", (double)epIter);
extra.Add("X", MatrixUtils.StackColumns(X));
extra.Add("Y", MatrixUtils.ToDouble(Y));
recorder.WriteRecords(recordPath, extra);
}
// merge all records and write
LogisticOpRecords merged = LogisticOpRecords.Merge(allRecs.ToArray());
merged.inferenceTimes = allInferTimes;
merged.dotNetPostW = allDotNetPosteriors;
merged.postW = allPosteriors;
string fnameM = string.Format("kjit_is{0}_uci{1}_iter{2}.mat",
importanceSamplingSize, dataAbbrv.Length, epIter);
string recordPathM = Config.PathToSavedFile(folder + fnameM);
var allExtra = new Dictionary <string, object>();
double[] oraInferTimesArr = allOraInferTimes.Select(t => (double)t).ToArray();
allExtra.Add("oraInferTimes", Vector.FromArray(oraInferTimesArr));
merged.WriteRecords(recordPathM, allExtra);
}
public static void TestLogisticRegressionNoBias()
{
const int seed = 2;
Rand.Restart(seed);
const int d = 10;
const int n = 300;
const int epIter = 10;
Vector w = Vector.Zero(d);
Rand.Normal(Vector.Zero(d), PositiveDefiniteMatrix.Identity(d), w);
double b = 0;
// combine the bias term into W
Vector[] X;
bool[] Y;
GenData(n, w, b, out X, out Y);
Console.Write("Y: ");
StringUtils.PrintArray(Y);
VectorGaussian wPost;
// string factorOpPath = Config.PathToFactorOperator(
// // "serialFactorOp_fm_kgg_joint_irf500_orf1000_n400_iter5_sf1_st20_ntr5000.mat"
// "serialFactorOp_fm_kgg_joint_irf500_orf1000_proj_n400_iter5_sf1_st20_ntr5000.mat"
// );
// KEPLogisticOpInstance opIns = KEPLogisticOpInstance.LoadLogisticOpInstance(factorOpPath);
// opIns.SetPrintTrueMessages(true);
// OpControl.Add(typeof(KEPLogisticOp), opIns);
// Type logisticOp = typeof(KEPLogisticOp);
LogisticOpRecords records = new LogisticOpRecords();
OpControl.Add(typeof(KEPOnlineLogisticOp), new KEPOnlineISLogisticOpIns(records));
Type logisticOp = typeof(KEPOnlineLogisticOp);
// Type logisticOp = typeof(LogisticOp2);
InferCoefficientsNoBias(X, Y, out wPost, epIter, logisticOp);
//print
Console.WriteLine("n: {0}", n);
Console.WriteLine("d: {0}", d);
int t = Y.Sum(o => o ? 1 : 0);
Console.WriteLine("number of true: {0}", t);
Console.WriteLine("True bias: {0}", b);
// Vector meanW = wPost.GetMean();
Console.WriteLine("True w: {0}", w);
Console.WriteLine("Inferred w: ");
Console.WriteLine(wPost);
// write the records to a file
string fname = string.Format("rec_onlinekep_is_logistic_iter{0}_n{1}.mat",
epIter, n);
string recordPath = Config.PathToSavedFile(fname);
var extra = new Dictionary <string, object>();
// MatlabWriter cannot write int
extra.Add("d", (double)d);
extra.Add("n", (double)n);
extra.Add("epIter", (double)epIter);
extra.Add("trueW", w);
extra.Add("X", MatrixUtils.StackColumns(X));
extra.Add("Y", MatrixUtils.ToDouble(Y));
records.WriteRecords(recordPath, extra);
}
public static void RunOnlineKEPDotNet()
{
// KJIT with Infer.NET oracle
// Compile model only once. In this model, only one message can
// be collected from one EP problem.
Variable <int> dataCount = Variable.Observed(-1).Named("dataCount");
Range n = new Range(dataCount).Named("N");
Variable <double> precision = Variable <double> .Factor(CGFac.FromCompoundGamma);
precision.AddAttribute(new TraceMessages());
precision.AddAttribute(new MarginalPrototype(new Gamma()));
VariableArray <double> x = Variable.Array <double>(n).Named("X");
x[n] = Variable.GaussianFromMeanAndPrecision(GaussMean, precision).ForEach(n);
// Create the operator instance only once because we want to use the same
// one after a new problem (new seed).
// stopwatch for measuring inference time for each problem
Stopwatch watch = new Stopwatch();
CGOpRecords record = new CGOpRecords();
var opIns = new KEP_CGFacOpIns(onlineBatchSizeTrigger, record, watch, initialThreshold);
opIns.IsPrintTrueWhenCertain = true;
opIns.IsRecordMessages = true;
//---- records -----
List <long> allInferTimes = new List <long>();
List <long> allOracleInferTimes = new List <long>();
// List<Dictionary<string, object>> allExtras = new List<Dictionary<string, object>>();
var allPosteriors = new List <Gamma>();
var allOraclePosteriors = new List <Gamma>();
int[] Ns = new int[seed_to];
double[] trueRate2s = new double[seed_to];
double[] truePrecs = new double[seed_to];
List <double[]> allObs = new List <double[]>();
//---------------
OpControl.Set(typeof(KEP_CGFacOp), opIns);
// one engine means the initial model compilation is done only once
InferenceEngine ie = NewInferenceEngine("KJIT_CG");
ie.Compiler.GivePriorityTo(typeof(KEP_CGFacOp));
// engine for the oracle
InferenceEngine dnetIe = NewInferenceEngine("Oracle_CG");
dnetIe.Compiler.GivePriorityTo(typeof(CGFacOp));
Random precRand = new Random(1);
for (int seed = 1; seed <= seed_to; seed++)
{
int i = seed - 1;
Rand.Restart(seed);
// n is between 10 and 100 for. Could be different for each seed.
int N = Rand.Int(10, 100 + 1);
Console.WriteLine("\n ///// New compound Gamma problem {0} of size: {1} /////\n",
seed, N);
Ns[i] = N;
CompoundGamma cg = new CompoundGamma();
double trueR2, truePrec;
double[] obs;
cg.GenData(N, seed, out obs, out trueR2, out truePrec);
// Draw a number uniformly from log(1e-4)
// to log(1e4) and exp(.) it to get the precision.
// double logPrec = precRand.NextDouble()*(Math.Log(1e4) - Math.Log(1e-4)) + Math.Log(1e-4);
// double truePrec = Math.Exp(logPrec);
// double[] obs = CompoundGamma.DrawFromGaussian(GaussMean, truePrec, N);
allObs.Add(obs);
// trueRate2s[i] = trueR2;
truePrecs[i] = truePrec;
dataCount.ObservedValue = N;
x.ObservedValue = obs;
// Console.Write("observations: ");
// StringUtils.PrintArray(obs);
// ie.Compiler.UseParallelForLoops = true;
watch.Restart();
Gamma postPrec = ie.Infer <Gamma>(precision);
long inferenceTime = watch.ElapsedMilliseconds;
allInferTimes.Add(inferenceTime);
allPosteriors.Add(postPrec);
// // Run Infer.net's operator on the same data
Stopwatch dnetWatch = new Stopwatch();
//
dataCount.ObservedValue = N;
x.ObservedValue = obs;
dnetWatch.Start();
Gamma dnetPost = dnetIe.Infer <Gamma>(precision);
long dnetTime = dnetWatch.ElapsedMilliseconds;
allOracleInferTimes.Add(dnetTime);
allOraclePosteriors.Add(dnetPost);
//print
Console.WriteLine("seed: {0}", seed);
Console.WriteLine("n: {0}", N);
// Console.WriteLine("True r2: {0}", trueR2);
Console.WriteLine("True precision: {0}", truePrec);
Console.WriteLine("Inferred precision posterior: {0}", postPrec);
Console.WriteLine("Infer.NET posterior: {0}", dnetPost);
Console.WriteLine("Inference time: {0} ms", inferenceTime);
Console.WriteLine("Infer.NET time: {0} ms", dnetTime);
Console.WriteLine("=========================");
Console.WriteLine();
}
// MatlabWriter cannot write int
var extra = new Dictionary <string, object>();
extra.Add("inferTimes", MatrixUtils.ToDouble(allInferTimes));
extra.Add("oraInferTimes", MatrixUtils.ToDouble(allOracleInferTimes));
double[] postShapes, postRates;
double[] oraPostShapes, oraPostRates;
CGOpRecords.GammaListToArrays(allPosteriors, out postShapes, out postRates);
CGOpRecords.GammaListToArrays(allOraclePosteriors, out oraPostShapes, out oraPostRates);
extra.Add("postShapes", postShapes);
extra.Add("postRates", postRates);
extra.Add("oraPostShapes", oraPostShapes);
extra.Add("oraPostRates", oraPostRates);
extra.Add("Ns", MatrixUtils.ToDoubleArray(Ns));
extra.Add("trueRate2s", trueRate2s);
extra.Add("truePrecs", truePrecs);
// MatlabWriter cannot write List<Vector> or List<double[]>. Why ?
// List<Vector> allObsVec = allObs.Select(ob => Vector.FromArray(ob)).ToList();
// extra.Add("allObs", allObsVec);
// write the records to a file
string fname = string.Format("kjit_cg_iter{0}_bt{1}_st{2}.mat",
epIter, onlineBatchSizeTrigger, seed_to);
string recordPath = Config.PathToSavedFile(fname);
record.WriteRecords(recordPath, extra);
}
