public static void TestLogisticRegression() { const int seed = 39; Rand.Restart(seed); const int d = 10; const int n = 100; const int epIter = 10; Vector w = Vector.Zero(d); Rand.Normal(Vector.Zero(d), PositiveDefiniteMatrix.Identity(d), w); double b = Rand.Normal(0, 1); // double b= 0; Vector[] X; bool[] Y; GenData(n, w, b, out X, out Y); Console.Write("Y: "); StringUtils.PrintArray(Y); VectorGaussian wPost; Gaussian biasPost; Type logisticOp = typeof(KEPLogisticOp); // Type logisticOp = typeof(LogisticOp2); string factorOpPath = Config.PathToFactorOperator( // "serialFactorOp_fm_kgg_joint_irf500_orf1000_n400_iter5_sf1_st20_ntr5000.mat" "serialFactorOp_fm_kgg_joint_irf500_orf1000_proj_n400_iter5_sf1_st20_ntr5000.mat" ); KEPLogisticOpInstance opIns = KEPLogisticOpInstance.LoadLogisticOpInstance(factorOpPath); opIns.SetPrintTrueMessages(true); OpControl.Add(typeof(KEPLogisticOp), opIns); InferCoefficients(X, Y, out wPost, out biasPost, epIter, logisticOp); //print Console.WriteLine("n: {0}", n); Console.WriteLine("d: {0}", d); int t = Y.Sum(o => o ? 1 : 0); Console.WriteLine("number of true: {0}", t); Console.WriteLine("True bias: {0}", b); Console.WriteLine("Inferred bias: {0}", biasPost); Console.WriteLine("True w: {0}", w); Console.WriteLine("Inferred w: "); Console.WriteLine(wPost); }
/// <summary> /// The constructor for a MIPS machine. Takes Memory size and Instruction ROM size. /// </summary> /// <param name="memSize"></param> /// <param name="instrSize"></param> public MIPSArch(uint memSize, uint instrSize) { PC = 0; MEM = new DataMemory(memSize); InstrMEM = new IMemory(instrSize); IR = new IRegister(); REG = new Registry(); ALUnit = new ALU(); SGNEXT = new SignExtend(); Add = new ADD(); Control = new OpControl(); ALControl = new ALUControl(); BranchGate = new AND(); FetchBuffer = new FetchDecodeBuffer[] { new FetchDecodeBuffer(), new FetchDecodeBuffer() }; DecodeBuffer = new DecodeExecuteBuffer[] { new DecodeExecuteBuffer(), new DecodeExecuteBuffer() }; ExecuteBuffer = new ExecuteMEMBuffer[] { new ExecuteMEMBuffer(), new ExecuteMEMBuffer() }; MEMBuffer = new MEMWBBuffer[] { new MEMWBBuffer(), new MEMWBBuffer() }; MAX_PC = new MIPSLoader(MEM, InstrMEM, REG).Initialize(); CLI = new CUI(ref InstrMEM, ref MEM, ref REG); }
public void RunOnlineKEPSampling() { // Kernel EP with importance sampling. /** * Only one W just like in Ali's paper. * In practice, we typically observe multiple sets of observations * where we want to do inference on the same model with the same * parameter. */ Rand.Restart(init_fixed_seed); Vector w = Vector.Zero(d); Rand.Normal(Vector.Zero(d), PositiveDefiniteMatrix.Identity(d), w); List <LogisticOpRecords> allRecs = new List <LogisticOpRecords>(); // Create the Logistic operator instance only one because we want to use the same // one after a new problem (new seed). // stopwatch for measuring inference time for each problem Stopwatch watch = new Stopwatch(); var logisticOpIns = new KEPOnlineISLogisticOpIns( new LogisticOpRecords(), watch, -8.5); logisticOpIns.SetImportanceSamplingSize(importanceSamplingSize); logisticOpIns.IsRecordMessages = true; logisticOpIns.IsPrintTrueWhenCertain = false; /** Use mixture or not ...*/ logisticOpIns.isGaussianOp.useMixtureProposal = false; logisticOpIns.SetFeatures(new int[] { 300, 500 }); OpControl.Set(typeof(KEPOnlineLogisticOp), logisticOpIns); Type logisticOp = typeof(KEPOnlineLogisticOp); List <long> allInferTimes = new List <long>(); var allPosteriors = new List <VectorGaussian>(); var allDotNetPosteriors = new List <VectorGaussian>(); LogisticOp2.IsCollectLogisticMessages = false; LogisticOp2.IsCollectProjMsgs = false; LogisticOp2.IsCollectXMessages = false; for (int seed = seed_from; seed <= seed_to; seed++) { Rand.Restart(seed); double b = 0; // combine the bias term into W Vector[] X; bool[] Y; LogisticRegression.GenData(n, w, b, out X, out Y, seed); Console.Write("Y: "); StringUtils.PrintArray(Y); VectorGaussian wPost; LogisticOpRecords recorder = new LogisticOpRecords(); // Set a new recorder for a new problem seed logisticOpIns.SetRecorder(recorder); // Type logisticOp = typeof(LogisticOp2); // start the watch watch.Restart(); LogisticRegression.InferCoefficientsNoBias(X, Y, out wPost, epIter, logisticOp); // stop the watch long inferenceTime = watch.ElapsedMilliseconds; recorder.inferenceTimes = new List <long>(); recorder.inferenceTimes.Add(inferenceTime); allInferTimes.Add(inferenceTime); recorder.postW = MatrixUtils.ToList(wPost); allPosteriors.Add(wPost); allRecs.Add(recorder); //print Console.WriteLine("n: {0}", n); Console.WriteLine("d: {0}", d); int t = Y.Sum(o => o ? 1 : 0); Console.WriteLine("number of true: {0}", t); Console.WriteLine("True bias: {0}", b); // Vector meanW = wPost.GetMean(); Console.WriteLine("True w: {0}", w); Console.WriteLine("Inferred w: "); Console.WriteLine(wPost); // Run Infer.net's operator on the same data VectorGaussian dotNetPostW; LogisticRegression.InferCoefficientsNoBias(X, Y, out dotNetPostW, epIter, typeof(LogisticOp2)); recorder.dotNetPostW = MatrixUtils.ToList <VectorGaussian>(dotNetPostW); allDotNetPosteriors.Add(dotNetPostW); // write the records to a file string fname = string.Format("rec_onlinekep_is{0}_n{1}_logistic_iter{2}_s{3}.mat", importanceSamplingSize, n, epIter, seed); string recordPath = Config.PathToSavedFile(fname); var extra = new Dictionary <string, object>(); // MatlabWriter cannot write int extra.Add("d", (double)d); extra.Add("n", (double)n); extra.Add("epIter", (double)epIter); extra.Add("trueW", w); extra.Add("X", MatrixUtils.StackColumns(X)); extra.Add("Y", MatrixUtils.ToDouble(Y)); recorder.WriteRecords(recordPath, extra); } // merge all records and write LogisticOpRecords merged = LogisticOpRecords.Merge(allRecs.ToArray()); merged.inferenceTimes = allInferTimes; merged.dotNetPostW = allDotNetPosteriors; merged.postW = allPosteriors; string fnameM = string.Format("rec_onlinekep_is{0}_n{1}_logistic_iter{2}_sf{3}_st{4}.mat", importanceSamplingSize, n, epIter, seed_from, seed_to); string recordPathM = Config.PathToSavedFile(fnameM); merged.WriteRecords(recordPathM); }
/**Run KJIT with an importance sampler as the oracle on a number of * UCI real datasets.*/ public void RunRealOnlineKEPSampling() { Rand.Restart(1); List <LogisticOpRecords> allRecs = new List <LogisticOpRecords>(); // Create the Logistic operator instance only once because we want to use the same // one after a new problem (new seed). // stopwatch for measuring inference time for each problem Stopwatch watch = new Stopwatch(); var logisticOpIns = new KEPOnlineISLogisticOpIns( new LogisticOpRecords(), watch, -8.95); logisticOpIns.SetOnlineBatchSizeTrigger(500); logisticOpIns.SetImportanceSamplingSize(importanceSamplingSize); logisticOpIns.IsRecordMessages = true; logisticOpIns.IsPrintTrueWhenCertain = false; // See BayesLinRegFM's BatchLearn() and KEPOnlineISLogisticOpIns() // If using the sum kernel logisticOpIns.SetFeatures(new int[] { 400, 800 }); OpControl.Set(typeof(KEPOnlineLogisticOp), logisticOpIns); Type logisticOp = typeof(KEPOnlineLogisticOp); List <long> allInferTimes = new List <long>(); List <long> allOraInferTimes = new List <long>(); var allPosteriors = new List <VectorGaussian>(); var allDotNetPosteriors = new List <VectorGaussian>(); LogisticOp2.IsCollectLogisticMessages = false; LogisticOp2.IsCollectProjMsgs = false; LogisticOp2.IsCollectXMessages = false; string folder = "online_uci/"; for (int i = 0; i < dataNames.Length; i++) { Console.WriteLine(); Console.WriteLine("----------- starting problem {0} --------------", dataAbbrv[i]); Console.WriteLine(); Vector[] X; bool[] Y; LoadDataFromMat(dataPaths[i], out X, out Y); Console.Write("Y: "); StringUtils.PrintArray(Y); VectorGaussian wPost; LogisticOpRecords recorder = new LogisticOpRecords(); // Set a new recorder for a new problem seed logisticOpIns.SetRecorder(recorder); // Type logisticOp = typeof(LogisticOp2); // start the watch watch.Restart(); // We do not include the bias term. So make sure the datasets // are standardized. LogisticRegression.InferCoefficientsNoBias(X, Y, out wPost, epIter, logisticOp); // stop the watch long inferenceTime = watch.ElapsedMilliseconds; recorder.inferenceTimes = new List <long>(); recorder.inferenceTimes.Add(inferenceTime); allInferTimes.Add(inferenceTime); recorder.postW = MatrixUtils.ToList(wPost); allPosteriors.Add(wPost); allRecs.Add(recorder); //print Console.WriteLine("n: {0}", n); Console.WriteLine("d: {0}", d); int t = Y.Sum(o => o ? 1 : 0); Console.WriteLine("number of true: {0}", t); // Vector meanW = wPost.GetMean(); Console.WriteLine("Inferred w: "); Console.WriteLine(wPost); // Run Infer.net's operator on the same data VectorGaussian dotNetPostW; Stopwatch oraWatch = new Stopwatch(); oraWatch.Start(); LogisticRegression.InferCoefficientsNoBias(X, Y, out dotNetPostW, epIter, typeof(LogisticOp2)); long oraInferTime = oraWatch.ElapsedMilliseconds; allOraInferTimes.Add(oraInferTime); recorder.dotNetPostW = MatrixUtils.ToList <VectorGaussian>(dotNetPostW); allDotNetPosteriors.Add(dotNetPostW); // write the records to a file string fname = string.Format("kjit_is{0}_{1}_iter{2}.mat", importanceSamplingSize, dataAbbrv[i], epIter); string recordPath = Config.PathToSavedFile(folder + fname); var extra = new Dictionary <string, object>(); // MatlabWriter cannot write int extra.Add("d", (double)X[0].Count); extra.Add("n", (double)X.Length); extra.Add("epIter", (double)epIter); extra.Add("X", MatrixUtils.StackColumns(X)); extra.Add("Y", MatrixUtils.ToDouble(Y)); recorder.WriteRecords(recordPath, extra); } // merge all records and write LogisticOpRecords merged = LogisticOpRecords.Merge(allRecs.ToArray()); merged.inferenceTimes = allInferTimes; merged.dotNetPostW = allDotNetPosteriors; merged.postW = allPosteriors; string fnameM = string.Format("kjit_is{0}_uci{1}_iter{2}.mat", importanceSamplingSize, dataAbbrv.Length, epIter); string recordPathM = Config.PathToSavedFile(folder + fnameM); var allExtra = new Dictionary <string, object>(); double[] oraInferTimesArr = allOraInferTimes.Select(t => (double)t).ToArray(); allExtra.Add("oraInferTimes", Vector.FromArray(oraInferTimesArr)); merged.WriteRecords(recordPathM, allExtra); }
public static void TestLogisticRegressionNoBias() { const int seed = 2; Rand.Restart(seed); const int d = 10; const int n = 300; const int epIter = 10; Vector w = Vector.Zero(d); Rand.Normal(Vector.Zero(d), PositiveDefiniteMatrix.Identity(d), w); double b = 0; // combine the bias term into W Vector[] X; bool[] Y; GenData(n, w, b, out X, out Y); Console.Write("Y: "); StringUtils.PrintArray(Y); VectorGaussian wPost; // string factorOpPath = Config.PathToFactorOperator( // // "serialFactorOp_fm_kgg_joint_irf500_orf1000_n400_iter5_sf1_st20_ntr5000.mat" // "serialFactorOp_fm_kgg_joint_irf500_orf1000_proj_n400_iter5_sf1_st20_ntr5000.mat" // ); // KEPLogisticOpInstance opIns = KEPLogisticOpInstance.LoadLogisticOpInstance(factorOpPath); // opIns.SetPrintTrueMessages(true); // OpControl.Add(typeof(KEPLogisticOp), opIns); // Type logisticOp = typeof(KEPLogisticOp); LogisticOpRecords records = new LogisticOpRecords(); OpControl.Add(typeof(KEPOnlineLogisticOp), new KEPOnlineISLogisticOpIns(records)); Type logisticOp = typeof(KEPOnlineLogisticOp); // Type logisticOp = typeof(LogisticOp2); InferCoefficientsNoBias(X, Y, out wPost, epIter, logisticOp); //print Console.WriteLine("n: {0}", n); Console.WriteLine("d: {0}", d); int t = Y.Sum(o => o ? 1 : 0); Console.WriteLine("number of true: {0}", t); Console.WriteLine("True bias: {0}", b); // Vector meanW = wPost.GetMean(); Console.WriteLine("True w: {0}", w); Console.WriteLine("Inferred w: "); Console.WriteLine(wPost); // write the records to a file string fname = string.Format("rec_onlinekep_is_logistic_iter{0}_n{1}.mat", epIter, n); string recordPath = Config.PathToSavedFile(fname); var extra = new Dictionary <string, object>(); // MatlabWriter cannot write int extra.Add("d", (double)d); extra.Add("n", (double)n); extra.Add("epIter", (double)epIter); extra.Add("trueW", w); extra.Add("X", MatrixUtils.StackColumns(X)); extra.Add("Y", MatrixUtils.ToDouble(Y)); records.WriteRecords(recordPath, extra); }
public static void RunOnlineKEPDotNet() { // KJIT with Infer.NET oracle // Compile model only once. In this model, only one message can // be collected from one EP problem. Variable <int> dataCount = Variable.Observed(-1).Named("dataCount"); Range n = new Range(dataCount).Named("N"); Variable <double> precision = Variable <double> .Factor(CGFac.FromCompoundGamma); precision.AddAttribute(new TraceMessages()); precision.AddAttribute(new MarginalPrototype(new Gamma())); VariableArray <double> x = Variable.Array <double>(n).Named("X"); x[n] = Variable.GaussianFromMeanAndPrecision(GaussMean, precision).ForEach(n); // Create the operator instance only once because we want to use the same // one after a new problem (new seed). // stopwatch for measuring inference time for each problem Stopwatch watch = new Stopwatch(); CGOpRecords record = new CGOpRecords(); var opIns = new KEP_CGFacOpIns(onlineBatchSizeTrigger, record, watch, initialThreshold); opIns.IsPrintTrueWhenCertain = true; opIns.IsRecordMessages = true; //---- records ----- List <long> allInferTimes = new List <long>(); List <long> allOracleInferTimes = new List <long>(); // List<Dictionary<string, object>> allExtras = new List<Dictionary<string, object>>(); var allPosteriors = new List <Gamma>(); var allOraclePosteriors = new List <Gamma>(); int[] Ns = new int[seed_to]; double[] trueRate2s = new double[seed_to]; double[] truePrecs = new double[seed_to]; List <double[]> allObs = new List <double[]>(); //--------------- OpControl.Set(typeof(KEP_CGFacOp), opIns); // one engine means the initial model compilation is done only once InferenceEngine ie = NewInferenceEngine("KJIT_CG"); ie.Compiler.GivePriorityTo(typeof(KEP_CGFacOp)); // engine for the oracle InferenceEngine dnetIe = NewInferenceEngine("Oracle_CG"); dnetIe.Compiler.GivePriorityTo(typeof(CGFacOp)); Random precRand = new Random(1); for (int seed = 1; seed <= seed_to; seed++) { int i = seed - 1; Rand.Restart(seed); // n is between 10 and 100 for. Could be different for each seed. int N = Rand.Int(10, 100 + 1); Console.WriteLine("\n ///// New compound Gamma problem {0} of size: {1} /////\n", seed, N); Ns[i] = N; CompoundGamma cg = new CompoundGamma(); double trueR2, truePrec; double[] obs; cg.GenData(N, seed, out obs, out trueR2, out truePrec); // Draw a number uniformly from log(1e-4) // to log(1e4) and exp(.) it to get the precision. // double logPrec = precRand.NextDouble()*(Math.Log(1e4) - Math.Log(1e-4)) + Math.Log(1e-4); // double truePrec = Math.Exp(logPrec); // double[] obs = CompoundGamma.DrawFromGaussian(GaussMean, truePrec, N); allObs.Add(obs); // trueRate2s[i] = trueR2; truePrecs[i] = truePrec; dataCount.ObservedValue = N; x.ObservedValue = obs; // Console.Write("observations: "); // StringUtils.PrintArray(obs); // ie.Compiler.UseParallelForLoops = true; watch.Restart(); Gamma postPrec = ie.Infer <Gamma>(precision); long inferenceTime = watch.ElapsedMilliseconds; allInferTimes.Add(inferenceTime); allPosteriors.Add(postPrec); // // Run Infer.net's operator on the same data Stopwatch dnetWatch = new Stopwatch(); // dataCount.ObservedValue = N; x.ObservedValue = obs; dnetWatch.Start(); Gamma dnetPost = dnetIe.Infer <Gamma>(precision); long dnetTime = dnetWatch.ElapsedMilliseconds; allOracleInferTimes.Add(dnetTime); allOraclePosteriors.Add(dnetPost); //print Console.WriteLine("seed: {0}", seed); Console.WriteLine("n: {0}", N); // Console.WriteLine("True r2: {0}", trueR2); Console.WriteLine("True precision: {0}", truePrec); Console.WriteLine("Inferred precision posterior: {0}", postPrec); Console.WriteLine("Infer.NET posterior: {0}", dnetPost); Console.WriteLine("Inference time: {0} ms", inferenceTime); Console.WriteLine("Infer.NET time: {0} ms", dnetTime); Console.WriteLine("========================="); Console.WriteLine(); } // MatlabWriter cannot write int var extra = new Dictionary <string, object>(); extra.Add("inferTimes", MatrixUtils.ToDouble(allInferTimes)); extra.Add("oraInferTimes", MatrixUtils.ToDouble(allOracleInferTimes)); double[] postShapes, postRates; double[] oraPostShapes, oraPostRates; CGOpRecords.GammaListToArrays(allPosteriors, out postShapes, out postRates); CGOpRecords.GammaListToArrays(allOraclePosteriors, out oraPostShapes, out oraPostRates); extra.Add("postShapes", postShapes); extra.Add("postRates", postRates); extra.Add("oraPostShapes", oraPostShapes); extra.Add("oraPostRates", oraPostRates); extra.Add("Ns", MatrixUtils.ToDoubleArray(Ns)); extra.Add("trueRate2s", trueRate2s); extra.Add("truePrecs", truePrecs); // MatlabWriter cannot write List<Vector> or List<double[]>. Why ? // List<Vector> allObsVec = allObs.Select(ob => Vector.FromArray(ob)).ToList(); // extra.Add("allObs", allObsVec); // write the records to a file string fname = string.Format("kjit_cg_iter{0}_bt{1}_st{2}.mat", epIter, onlineBatchSizeTrigger, seed_to); string recordPath = Config.PathToSavedFile(fname); record.WriteRecords(recordPath, extra); }