/// <summary> /// Active Transfer tests. /// </summary> /// <param name="trainModel">The train model.</param> /// <param name="testModel">The test model.</param> /// <param name="data">The datasets.</param> public static void ActiveTransfer(BinaryModel trainModel, BinaryModel testModel, IList <ToyData> data, string title, Marginals priors) { var learners = new Dictionary <string, IList <IActiveLearner> > { { "Random", Utils.CreateLearners <RandomLearner>(data[2].DataSet, trainModel, testModel, null) }, { "US", Utils.CreateLearners <UncertainActiveLearner>(data[2].DataSet, trainModel, testModel, null) }, { "CS", Utils.CreateLearners <UncertainActiveLearner>(data[2].DataSet, trainModel, testModel, null, true) }, { "VOI+", Utils.CreateLearners <ActiveLearner>(data[2].DataSet, trainModel, testModel, null) }, { "VOI-", Utils.CreateLearners <ActiveLearner>(data[2].DataSet, trainModel, testModel, null, true) } }; var experiments = new List <Experiment>(); foreach (var learner in learners) { Console.WriteLine("Testing {0} ({1})", title, learner.Key); var experiment = new Experiment { TrainModel = trainModel, TestModel = testModel, Name = learner.Key, ActiveLearners = learner.Value }; experiment.RunActive(data[2].DataSet, data[2].HoldoutSet, ActiveSteps, priors); experiments.Add(experiment); } Utils.PlotHoldoutMetrics(experiments, title, "", true); }
/// <summary> /// Active Transfer tests. /// </summary> /// <param name="trainModel">The train model.</param> /// <param name="testModel">The test model.</param> /// <param name="data">The datasets.</param> public static void ActiveTransfer(BinaryModel trainModel, BinaryModel testModel, IList<ToyData> data, string title, Marginals priors) { var learners = new Dictionary<string, IList<IActiveLearner>> { { "Random", Utils.CreateLearners<RandomLearner>(data[2].DataSet, trainModel, testModel, null) }, { "US", Utils.CreateLearners<UncertainActiveLearner>(data[2].DataSet, trainModel, testModel, null) }, { "CS", Utils.CreateLearners<UncertainActiveLearner>(data[2].DataSet, trainModel, testModel, null, true) }, { "VOI+", Utils.CreateLearners<ActiveLearner>(data[2].DataSet, trainModel, testModel, null) }, { "VOI-", Utils.CreateLearners<ActiveLearner>(data[2].DataSet, trainModel, testModel, null, true) } }; var experiments = new List<Experiment>(); foreach (var learner in learners) { Console.WriteLine("Testing {0} ({1})", title, learner.Key); var experiment = new Experiment { TrainModel = trainModel, TestModel = testModel, Name = learner.Key, ActiveLearners = learner.Value }; experiment.RunActive(data[2].DataSet, data[2].HoldoutSet, ActiveSteps, priors); experiments.Add(experiment); } Utils.PlotHoldoutMetrics(experiments, title, "", true); }
/// <summary> /// Initializes a new instance of the <see cref="ActiveTransfer.ToyDataRunner"/> class. /// </summary> /// <param name="trainModel">Train model.</param> /// <param name="testModel">Test model.</param> public void Run(BinaryModel trainModel, BinaryModel testModel, BinaryModel evidenceModel, bool testVOI, bool testActiveEvidence) { const int NumberOfResidents = 7; const double KeepProportion = 1.0; var selectedFeatures = new HashSet<int>(Enumerable.Range(0, 48)); var ted = Source.GetDataSet(Enumerable.Range(1, 14), AddBias, selectedFeatures, KeepProportion); var trd = Target.GetDataSet(Enumerable.Range(1, 25), AddBias, selectedFeatures, KeepProportion); // var ted = Source.GetDataSet( Enumerable.Range( 1, 1 ), AddBias, selectedFeatures, KeepProportion ); // var trd = Target.GetDataSet( Enumerable.Range( 1, 20 ), AddBias, selectedFeatures, KeepProportion ); // var hod = Target.GetDataSet( Enumerable.Range( 1 + NumberOfResidents * 1, NumberOfResidents ) ); DataSet testSet; DataSet holdoutSet; ted.SplitTrainTest(0.5, out testSet, out holdoutSet); var NumFeatures = trd.Features.First().First().Count(); var trainData = new ToyData { NumberOfResidents = trd.NumberOfResidents, NumberOfFeatures = NumFeatures, NumberOfActivities = 2, UseBias = false, DataSet = trd }; var testData = new ToyData { NumberOfResidents = NumberOfResidents, NumberOfFeatures = NumFeatures, NumberOfActivities = 2, UseBias = false, DataSet = testSet, HoldoutSet = holdoutSet }; var priors = new Marginals { WeightMeans = DistributionArrayHelpers.CreateGaussianArray(trainData.NumberOfFeatures, 0.0, 1.0).ToArray(), WeightPrecisions = DistributionArrayHelpers.CreateGammaArray(trainData.NumberOfFeatures, 1.0, 1.0).ToArray() }; // TODO: Create meta-features that allow us to do the first form of transfer learning // Train the community model var communityExperiment = new Experiment { TrainModel = trainModel, TestModel = testModel, EvidenceModel = evidenceModel, Name = "Community" }; communityExperiment.RunBatch(trainData.DataSet, priors); // communityExperiment.Posteriors.WeightPrecisions = priors.WeightPrecisions; // if (false) // { // Utils.PlotPosteriors(communityExperiment.Posteriors.WeightMeans, communityExperiment.Posteriors.WeightPrecisions, null, "Community weights", "Feature", ShowPlots); // Utils.PlotPosteriors(communityExperiment.Posteriors.WeightMeans, communityExperiment.Posteriors.WeightPrecisions, null, "Community weights (prior precision)", "Feature", ShowPlots); // } // Print top features // var topWeights = communityExperiment.Posteriors.WeightMeans.Zip(communityExperiment.Posteriors.WeightPrecisions, (m, p) => new { m, p }).Select((ia, i) => new { ia, i }) // .OrderByDescending(x => Math.Abs(x.ia.m.GetMean())).ToList(); // Console.WriteLine("Top 20 weights:\n {0}", string.Join("\n", topWeights.Take(20).Select(pair => string.Format("{0}: {1}", pair.i, new Gaussian(pair.ia.m.GetMean(), pair.ia.p.GetMean()))))); // //communityExperiment.Posteriors.WeightPrecisions = DistributionArrayHelpers.Copy( priors.WeightPrecisions ).ToArray(); var sourcePosteriors = new Marginals { WeightMeans = communityExperiment.Posteriors.WeightMeans, WeightPrecisions = priors.WeightPrecisions, //communityExperiment.Posteriors.WeightMeans, Weights = null }; // Select half the features /* trainData.DataSet.Features = trainData.DataSet.Features.Select( ia => ia.Select( ib => topWeights.Take(topWeights.Count / 2).Select(pair => ib[pair.i]).ToArray()) .ToArray()) .ToArray(); // Retrain using these weights */ // if (false) // { // // Do online learning // var onlineExperiment = new Experiment // { // TrainModel = trainModel, // TestModel = testModel, // Name = "Online" // }; // onlineExperiment.RunOnline(testData.DataSet, testData.HoldoutSet, priors); // // Do transfer learning // var personalisationExperiment = new Experiment // { // TrainModel = trainModel, // TestModel = testModel, // Name = "Community" // }; // personalisationExperiment.RunOnline(testData.DataSet, testData.HoldoutSet, communityExperiment.Posteriors); // // Plot cumulative metrics // Utils.PlotCumulativeMetrics(new [] { onlineExperiment, personalisationExperiment }, "Active", ShowPlots); // } // ACTIVE MODEL foreach (var doTransfer in new[] { false, true }) { var experiments = new List<Experiment>(); var learners = CreateLearners(trainModel, testModel, evidenceModel, testData, testVOI, testActiveEvidence); foreach (var learner in learners) { Console.WriteLine("Testing Active{0} Learning ({1})", doTransfer ? " Real Transfer" : "Real Online", learner.Key); var experiment = new Experiment { TrainModel = trainModel, TestModel = testModel, Name = learner.Key, ActiveLearners = learner.Value }; experiment.RunActive(testData.DataSet, testData.HoldoutSet, ActiveSteps, doTransfer ? sourcePosteriors : priors); experiments.Add(experiment); if (false) { Utils.PlotPosteriors( experiment.IndividualPosteriors[0].WeightMeans, experiment.IndividualPosteriors[0].WeightPrecisions, null, "Posterior weights for " + learner.Key + " " + (doTransfer ? " (transfer)" : ""), "Feature", ShowPlots); } } Utils.PlotHoldoutMetrics(experiments, doTransfer ? "Real Active Transfer" : "Real Active", "", ShowPlots); } }
/// <summary> /// Initializes a new instance of the <see cref="ActiveTransfer.ToyDataRunner"/> class. /// </summary> /// <param name="trainModel">Train model.</param> /// <param name="testModel">Test model.</param> public void Run(BinaryModel trainModel, BinaryModel testModel, BinaryModel evidenceModel, bool testVOI, bool testActiveEvidence) { const int NumberOfResidents = 7; const double KeepProportion = 1.0; var selectedFeatures = new HashSet <int>(Enumerable.Range(0, 48)); var ted = Source.GetDataSet(Enumerable.Range(1, 14), AddBias, selectedFeatures, KeepProportion); var trd = Target.GetDataSet(Enumerable.Range(1, 25), AddBias, selectedFeatures, KeepProportion); // var ted = Source.GetDataSet( Enumerable.Range( 1, 1 ), AddBias, selectedFeatures, KeepProportion ); // var trd = Target.GetDataSet( Enumerable.Range( 1, 20 ), AddBias, selectedFeatures, KeepProportion ); // var hod = Target.GetDataSet( Enumerable.Range( 1 + NumberOfResidents * 1, NumberOfResidents ) ); DataSet testSet; DataSet holdoutSet; ted.SplitTrainTest(0.5, out testSet, out holdoutSet); var NumFeatures = trd.Features.First().First().Count(); var trainData = new ToyData { NumberOfResidents = trd.NumberOfResidents, NumberOfFeatures = NumFeatures, NumberOfActivities = 2, UseBias = false, DataSet = trd }; var testData = new ToyData { NumberOfResidents = NumberOfResidents, NumberOfFeatures = NumFeatures, NumberOfActivities = 2, UseBias = false, DataSet = testSet, HoldoutSet = holdoutSet }; var priors = new Marginals { WeightMeans = DistributionArrayHelpers.CreateGaussianArray(trainData.NumberOfFeatures, 0.0, 1.0).ToArray(), WeightPrecisions = DistributionArrayHelpers.CreateGammaArray(trainData.NumberOfFeatures, 1.0, 1.0).ToArray() }; // TODO: Create meta-features that allow us to do the first form of transfer learning // Train the community model var communityExperiment = new Experiment { TrainModel = trainModel, TestModel = testModel, EvidenceModel = evidenceModel, Name = "Community" }; communityExperiment.RunBatch(trainData.DataSet, priors); // communityExperiment.Posteriors.WeightPrecisions = priors.WeightPrecisions; // if (false) // { // Utils.PlotPosteriors(communityExperiment.Posteriors.WeightMeans, communityExperiment.Posteriors.WeightPrecisions, null, "Community weights", "Feature", ShowPlots); // Utils.PlotPosteriors(communityExperiment.Posteriors.WeightMeans, communityExperiment.Posteriors.WeightPrecisions, null, "Community weights (prior precision)", "Feature", ShowPlots); // } // Print top features // var topWeights = communityExperiment.Posteriors.WeightMeans.Zip(communityExperiment.Posteriors.WeightPrecisions, (m, p) => new { m, p }).Select((ia, i) => new { ia, i }) // .OrderByDescending(x => Math.Abs(x.ia.m.GetMean())).ToList(); // Console.WriteLine("Top 20 weights:\n {0}", string.Join("\n", topWeights.Take(20).Select(pair => string.Format("{0}: {1}", pair.i, new Gaussian(pair.ia.m.GetMean(), pair.ia.p.GetMean()))))); // //communityExperiment.Posteriors.WeightPrecisions = DistributionArrayHelpers.Copy( priors.WeightPrecisions ).ToArray(); var sourcePosteriors = new Marginals { WeightMeans = communityExperiment.Posteriors.WeightMeans, WeightPrecisions = priors.WeightPrecisions, //communityExperiment.Posteriors.WeightMeans, Weights = null }; // Select half the features /* * trainData.DataSet.Features = trainData.DataSet.Features.Select( * ia => ia.Select( * ib => topWeights.Take(topWeights.Count / 2).Select(pair => ib[pair.i]).ToArray()) * .ToArray()) * .ToArray(); * * // Retrain using these weights */ // if (false) // { // // Do online learning // var onlineExperiment = new Experiment // { // TrainModel = trainModel, // TestModel = testModel, // Name = "Online" // }; // onlineExperiment.RunOnline(testData.DataSet, testData.HoldoutSet, priors); // // Do transfer learning // var personalisationExperiment = new Experiment // { // TrainModel = trainModel, // TestModel = testModel, // Name = "Community" // }; // personalisationExperiment.RunOnline(testData.DataSet, testData.HoldoutSet, communityExperiment.Posteriors); // // Plot cumulative metrics // Utils.PlotCumulativeMetrics(new [] { onlineExperiment, personalisationExperiment }, "Active", ShowPlots); // } // ACTIVE MODEL foreach (var doTransfer in new[] { false, true }) { var experiments = new List <Experiment>(); var learners = CreateLearners(trainModel, testModel, evidenceModel, testData, testVOI, testActiveEvidence); foreach (var learner in learners) { Console.WriteLine("Testing Active{0} Learning ({1})", doTransfer ? " Real Transfer" : "Real Online", learner.Key); var experiment = new Experiment { TrainModel = trainModel, TestModel = testModel, Name = learner.Key, ActiveLearners = learner.Value }; experiment.RunActive(testData.DataSet, testData.HoldoutSet, ActiveSteps, doTransfer ? sourcePosteriors : priors); experiments.Add(experiment); if (false) { Utils.PlotPosteriors( experiment.IndividualPosteriors[0].WeightMeans, experiment.IndividualPosteriors[0].WeightPrecisions, null, "Posterior weights for " + learner.Key + " " + (doTransfer ? " (transfer)" : ""), "Feature", ShowPlots); } } Utils.PlotHoldoutMetrics(experiments, doTransfer ? "Real Active Transfer" : "Real Active", "", ShowPlots); } }