示例#1
0
        /// <summary>
        /// Creates the learners
        /// </summary>
        /// <param name="trainModel">Train model.</param>
        /// <param name="testModel">Test model.</param>
        /// <param name="evidenceModel">Evidence Model.</param>
        /// <param name="testData">Test data.</param>
        /// <param name="testVOI">Whether to test VOI.</param>
        /// <param name="testActiveEvidence">Whether to test Active Evidence.</param>
        /// <returns>The learners.</returns>
        public Dictionary <string, IList <IActiveLearner> > CreateLearners(
            BinaryModel trainModel,
            BinaryModel testModel,
            BinaryModel evidenceModel,
            ToyData testData,
            bool testVOI,
            bool testActiveEvidence)
        {
            var learners = new Dictionary <string, IList <IActiveLearner> >
            {
                { "Random", Utils.CreateLearners <RandomLearner>(testData.DataSet, trainModel, testModel, evidenceModel) },
                { "US", Utils.CreateLearners <UncertainActiveLearner>(testData.DataSet, trainModel, evidenceModel, testModel) },
                // { "ActEv+", Utils.CreateLearners<ActiveEvidence>(testData.DataSet, trainModel, testModel, evidenceModel) },
                // { "ActEv-", Utils.CreateLearners<ActiveEvidence>(testData.DataSet, trainModel, testModel, evidenceModel, true) },
                // { "VOI+",   Utils.CreateLearners<ActiveLearner>(testData.DataSet, trainModel, testModel, evidenceModel) },
                // { "VOI-",   Utils.CreateLearners<ActiveLearner>(testData.DataSet, trainModel, testModel, evidenceModel, true) },
                // { "CS",     Utils.CreateLearners<UncertainActiveLearner>(testData.DataSet, trainModel, testModel, evidenceModel, true) },
            };

            if (testVOI)
            {
                learners["CS"]   = Utils.CreateLearners <UncertainActiveLearner>(testData.DataSet, trainModel, testModel, evidenceModel, true);
                learners["VOI+"] = Utils.CreateLearners <ActiveLearner>(testData.DataSet, trainModel, testModel, evidenceModel);
                learners["VOI-"] = Utils.CreateLearners <ActiveLearner>(testData.DataSet, trainModel, testModel, evidenceModel, true);
            }

            if (testActiveEvidence)
            {
                learners["ActEv+"] = Utils.CreateLearners <ActiveEvidence>(testData.DataSet, trainModel, testModel, evidenceModel);
            }

            return(learners);
        }
        /// <summary>
        /// Initializes a new instance of the <see cref="ToyDataRunner"/> class.
        /// </summary>
        /// <param name="trainModel">Train model.</param>
        /// <param name="testModel">Test model.</param>
        public static void Run(BinaryModel trainModel, BinaryModel testModel, bool testTransfer, bool testActive, bool testActiveTransfer)
        {
            var phase1PriorMean = new Gaussian(4, 1);
            var phase1PriorPrecision = new Gamma(1, 1);
            var phase2PriorMean = new Gaussian(4, 1);
            var phase2PriorPrecision = new Gamma(1, 1);

            // Generate data for 5 individuals
            var data = new List<ToyData>();
            for (int i = 0; i < 3; i++)
            {
                var toy = new ToyData
                {
                    // NumberOfInstances = 200,
                    // NumberOfHoldoutInstances = i == 0 ? 0 : 1000,
                    NumberOfResidents = 5,
                    NumberOfFeatures = NumberOfFeatures,
                    NumberOfActivities = 2,
                    UseBias = false,
                    TruePriorMean = i == 0 ? phase1PriorMean : phase2PriorMean,
                    TruePriorPrecision = i == 0 ? phase1PriorPrecision : phase2PriorPrecision
                };

                toy.Generate(i == 2 ? NoisyExampleProportion : 0.0, 200);
                if (i != 0)
                {
                    // no need for holdout data in training set
                    toy.Generate(0.0, 1000, true);
                }

                data.Add(toy);
            }

            var priors = new Marginals
            {
                WeightMeans = DistributionArrayHelpers.CreateGaussianArray(NumberOfFeatures, 0, 1).ToArray(),
                WeightPrecisions = DistributionArrayHelpers.CreateGammaArray(NumberOfFeatures, 1, 1).ToArray()
            };

            Console.WriteLine("Data Generated");

            // TODO: Create meta-features that allow us to do the first form of transfer learning

            // Train the community model
            Console.WriteLine("Training Community Model");
            var communityExperiment = new Experiment { TrainModel = trainModel, TestModel = testModel, Name = "Community" };
            communityExperiment.RunBatch(data[0].DataSet, priors);
            // PrintWeightPriors(communityExperiment.Posteriors, trainData.CommunityWeights);

            // Utils.PlotPosteriors(communityExperiment.Posteriors.Weights, data[0].Weights);
            // Utils.PlotPosteriors(communityExperiment.Posteriors.WeightMeans, communityExperiment.Posteriors.WeightPrecisions, null, "Community weights", "Feature");

            // return;

            if (testTransfer)
            {
                // Do online learning
                // Console.WriteLine("Testing Online Model");
                var onlineExperiment = new Experiment { TrainModel = trainModel, TestModel = testModel, Name = "Online" };
                onlineExperiment.RunOnline(data[1].DataSet, data[1].HoldoutSet, priors);

                // Do transfer learning
                // Console.WriteLine("Testing Community Model");
                var personalisationExperiment = new Experiment { TrainModel = trainModel, TestModel = testModel, Name = "Community" };
                personalisationExperiment.RunOnline(data[1].DataSet, data[1].HoldoutSet, communityExperiment.Posteriors);

                // Plot cumulative metrics
                Utils.PlotCumulativeMetrics(new[] { onlineExperiment, personalisationExperiment }, "Toy Transfer");
            }
            else
            {
                Console.WriteLine("Skipping Transfer Learning");
            }

            // ACTIVE MODEL
            if (testActive)
            {
                ActiveTransfer(trainModel, testModel, data, "Toy Active", priors);
            }
            else
            {
                Console.WriteLine("Skipping Active Learning");
            }

            if (testActiveTransfer)
            {
                Console.WriteLine("Note that the transfer learning is very effective here, so the active learning doesn't add much");
                ActiveTransfer(trainModel, testModel, data, "Toy Active Transfer", communityExperiment.Posteriors);
            }
            else
            {
                Console.WriteLine("Skipping Active Transfer Learning");
            }

            // Now create different costs for acquiring labels - want to demonstrate that we choose from all 3 possible labels
        }
        /// <summary>
        /// Creates the learners
        /// </summary>
        /// <param name="trainModel">Train model.</param>
        /// <param name="testModel">Test model.</param>
        /// <param name="evidenceModel">Evidence Model.</param>
        /// <param name="testData">Test data.</param>
        /// <param name="testVOI">Whether to test VOI.</param>
        /// <param name="testActiveEvidence">Whether to test Active Evidence.</param>
        /// <returns>The learners.</returns>
        public Dictionary<string, IList<IActiveLearner>> CreateLearners(
            BinaryModel trainModel,
            BinaryModel testModel,
            BinaryModel evidenceModel,
            ToyData testData,
            bool testVOI,
            bool testActiveEvidence)
        {
            var learners = new Dictionary<string, IList<IActiveLearner>>
                {
                    { "Random", Utils.CreateLearners<RandomLearner>(testData.DataSet, trainModel, testModel, evidenceModel) },
                    { "US",     Utils.CreateLearners<UncertainActiveLearner>(testData.DataSet, trainModel, evidenceModel, testModel) },
                    // { "ActEv+", Utils.CreateLearners<ActiveEvidence>(testData.DataSet, trainModel, testModel, evidenceModel) },
                    // { "ActEv-", Utils.CreateLearners<ActiveEvidence>(testData.DataSet, trainModel, testModel, evidenceModel, true) },
                    // { "VOI+",   Utils.CreateLearners<ActiveLearner>(testData.DataSet, trainModel, testModel, evidenceModel) },
                    // { "VOI-",   Utils.CreateLearners<ActiveLearner>(testData.DataSet, trainModel, testModel, evidenceModel, true) },
                    // { "CS",     Utils.CreateLearners<UncertainActiveLearner>(testData.DataSet, trainModel, testModel, evidenceModel, true) },
                };

            if (testVOI)
            {
                learners["CS"] = Utils.CreateLearners<UncertainActiveLearner>(testData.DataSet, trainModel, testModel, evidenceModel, true);
                learners["VOI+"] = Utils.CreateLearners<ActiveLearner>(testData.DataSet, trainModel, testModel, evidenceModel);
                learners["VOI-"] = Utils.CreateLearners<ActiveLearner>(testData.DataSet, trainModel, testModel, evidenceModel, true);
            }

            if (testActiveEvidence)
            {
                learners["ActEv+"] = Utils.CreateLearners<ActiveEvidence>(testData.DataSet, trainModel, testModel, evidenceModel);
            }

            return learners;
        }
        /// <summary>
        /// Initializes a new instance of the <see cref="ActiveTransfer.ToyDataRunner"/> class.
        /// </summary>
        /// <param name="trainModel">Train model.</param>
        /// <param name="testModel">Test model.</param>
        public void Run(BinaryModel trainModel, BinaryModel testModel, BinaryModel evidenceModel, bool testVOI, bool testActiveEvidence)
        {
            const int NumberOfResidents = 7;
            const double KeepProportion = 1.0;

            var selectedFeatures = new HashSet<int>(Enumerable.Range(0, 48));

            var ted = Source.GetDataSet(Enumerable.Range(1, 14), AddBias, selectedFeatures, KeepProportion);
            var trd = Target.GetDataSet(Enumerable.Range(1, 25), AddBias, selectedFeatures, KeepProportion);
            // var ted = Source.GetDataSet( Enumerable.Range( 1, 1 ), AddBias, selectedFeatures, KeepProportion );
            // var trd = Target.GetDataSet( Enumerable.Range( 1, 20 ), AddBias, selectedFeatures, KeepProportion );
            // var hod = Target.GetDataSet( Enumerable.Range( 1 + NumberOfResidents * 1, NumberOfResidents ) );

            DataSet testSet;
            DataSet holdoutSet;
            ted.SplitTrainTest(0.5, out testSet, out holdoutSet);

            var NumFeatures = trd.Features.First().First().Count();

            var trainData = new ToyData
            {
                NumberOfResidents = trd.NumberOfResidents,
                NumberOfFeatures = NumFeatures,
                NumberOfActivities = 2,
                UseBias = false,
                DataSet = trd
            };

            var testData = new ToyData
            {
                NumberOfResidents = NumberOfResidents,
                NumberOfFeatures = NumFeatures,
                NumberOfActivities = 2,
                UseBias = false,
                DataSet = testSet,
                HoldoutSet = holdoutSet
            };

            var priors = new Marginals
            {
                WeightMeans = DistributionArrayHelpers.CreateGaussianArray(trainData.NumberOfFeatures, 0.0, 1.0).ToArray(),
                WeightPrecisions = DistributionArrayHelpers.CreateGammaArray(trainData.NumberOfFeatures, 1.0, 1.0).ToArray()
            };

            // TODO: Create meta-features that allow us to do the first form of transfer learning

            // Train the community model
            var communityExperiment = new Experiment
            {
                TrainModel = trainModel,
                TestModel = testModel,
                EvidenceModel = evidenceModel,
                Name = "Community"
            };

            communityExperiment.RunBatch(trainData.DataSet, priors);
            // communityExperiment.Posteriors.WeightPrecisions = priors.WeightPrecisions;

            // if (false)
            // {
            //     Utils.PlotPosteriors(communityExperiment.Posteriors.WeightMeans, communityExperiment.Posteriors.WeightPrecisions, null, "Community weights", "Feature", ShowPlots);
            //     Utils.PlotPosteriors(communityExperiment.Posteriors.WeightMeans, communityExperiment.Posteriors.WeightPrecisions, null, "Community weights (prior precision)", "Feature", ShowPlots);
            // }

            // Print top features
            // var topWeights = communityExperiment.Posteriors.WeightMeans.Zip(communityExperiment.Posteriors.WeightPrecisions, (m, p) => new { m, p }).Select((ia, i) => new { ia, i })
            // 	.OrderByDescending(x => Math.Abs(x.ia.m.GetMean())).ToList();
            // Console.WriteLine("Top 20 weights:\n {0}", string.Join("\n", topWeights.Take(20).Select(pair => string.Format("{0}: {1}", pair.i, new Gaussian(pair.ia.m.GetMean(), pair.ia.p.GetMean())))));

            // //communityExperiment.Posteriors.WeightPrecisions = DistributionArrayHelpers.Copy( priors.WeightPrecisions ).ToArray();
            var sourcePosteriors = new Marginals
            {
                WeightMeans = communityExperiment.Posteriors.WeightMeans,
                WeightPrecisions = priors.WeightPrecisions, //communityExperiment.Posteriors.WeightMeans,
                Weights = null
            };

            // Select half the features
            /*
            trainData.DataSet.Features = trainData.DataSet.Features.Select(
                ia => ia.Select(
                    ib => topWeights.Take(topWeights.Count / 2).Select(pair => ib[pair.i]).ToArray())
                    .ToArray())
                .ToArray();

            // Retrain using these weights
            */

            // if (false)
            // {
            // 	// Do online learning
            // 	var onlineExperiment = new Experiment
            // 	{
            // 		TrainModel = trainModel,
            // 		TestModel = testModel,
            // 		Name = "Online"
            // 	};

            // 	onlineExperiment.RunOnline(testData.DataSet, testData.HoldoutSet, priors);

            // 	// Do transfer learning
            // 	var personalisationExperiment = new Experiment
            // 	{
            // 		TrainModel = trainModel,
            // 		TestModel = testModel,
            // 		Name = "Community"
            // 	};

            // 	personalisationExperiment.RunOnline(testData.DataSet, testData.HoldoutSet, communityExperiment.Posteriors);

            // 	// Plot cumulative metrics
            // 	Utils.PlotCumulativeMetrics(new [] { onlineExperiment, personalisationExperiment }, "Active", ShowPlots);
            // }

            // ACTIVE MODEL
            foreach (var doTransfer in new[] { false, true })
            {
                var experiments = new List<Experiment>();
                var learners = CreateLearners(trainModel, testModel, evidenceModel, testData, testVOI, testActiveEvidence);

                foreach (var learner in learners)
                {
                    Console.WriteLine("Testing Active{0} Learning ({1})", doTransfer ? " Real Transfer" : "Real Online", learner.Key);
                    var experiment = new Experiment
                    {
                        TrainModel = trainModel,
                        TestModel = testModel,
                        Name = learner.Key,
                        ActiveLearners = learner.Value
                    };

                    experiment.RunActive(testData.DataSet, testData.HoldoutSet, ActiveSteps, doTransfer ? sourcePosteriors : priors);
                    experiments.Add(experiment);

                    if (false)
                    {
                        Utils.PlotPosteriors(
                            experiment.IndividualPosteriors[0].WeightMeans,
                            experiment.IndividualPosteriors[0].WeightPrecisions,
                            null,
                            "Posterior weights for " + learner.Key + " " + (doTransfer ? " (transfer)" : ""),
                            "Feature",
                            ShowPlots);
                    }
                }

                Utils.PlotHoldoutMetrics(experiments, doTransfer ? "Real Active Transfer" : "Real Active", "", ShowPlots);
            }
        }
示例#5
0
        /// <summary>
        /// Initializes a new instance of the <see cref="ToyDataRunner"/> class.
        /// </summary>
        /// <param name="trainModel">Train model.</param>
        /// <param name="testModel">Test model.</param>
        public static void Run(BinaryModel trainModel, BinaryModel testModel, bool testTransfer, bool testActive, bool testActiveTransfer)
        {
            var phase1PriorMean      = new Gaussian(4, 1);
            var phase1PriorPrecision = new Gamma(1, 1);
            var phase2PriorMean      = new Gaussian(4, 1);
            var phase2PriorPrecision = new Gamma(1, 1);

            // Generate data for 5 individuals
            var data = new List <ToyData>();

            for (int i = 0; i < 3; i++)
            {
                var toy = new ToyData
                {
                    // NumberOfInstances = 200,
                    // NumberOfHoldoutInstances = i == 0 ? 0 : 1000,
                    NumberOfResidents  = 5,
                    NumberOfFeatures   = NumberOfFeatures,
                    NumberOfActivities = 2,
                    UseBias            = false,
                    TruePriorMean      = i == 0 ? phase1PriorMean : phase2PriorMean,
                    TruePriorPrecision = i == 0 ? phase1PriorPrecision : phase2PriorPrecision
                };

                toy.Generate(i == 2 ? NoisyExampleProportion : 0.0, 200);
                if (i != 0)
                {
                    // no need for holdout data in training set
                    toy.Generate(0.0, 1000, true);
                }

                data.Add(toy);
            }

            var priors = new Marginals
            {
                WeightMeans      = DistributionArrayHelpers.CreateGaussianArray(NumberOfFeatures, 0, 1).ToArray(),
                WeightPrecisions = DistributionArrayHelpers.CreateGammaArray(NumberOfFeatures, 1, 1).ToArray()
            };

            Console.WriteLine("Data Generated");

            // TODO: Create meta-features that allow us to do the first form of transfer learning

            // Train the community model
            Console.WriteLine("Training Community Model");
            var communityExperiment = new Experiment {
                TrainModel = trainModel, TestModel = testModel, Name = "Community"
            };

            communityExperiment.RunBatch(data[0].DataSet, priors);
            // PrintWeightPriors(communityExperiment.Posteriors, trainData.CommunityWeights);

            // Utils.PlotPosteriors(communityExperiment.Posteriors.Weights, data[0].Weights);
            // Utils.PlotPosteriors(communityExperiment.Posteriors.WeightMeans, communityExperiment.Posteriors.WeightPrecisions, null, "Community weights", "Feature");

            // return;

            if (testTransfer)
            {
                // Do online learning
                // Console.WriteLine("Testing Online Model");
                var onlineExperiment = new Experiment {
                    TrainModel = trainModel, TestModel = testModel, Name = "Online"
                };
                onlineExperiment.RunOnline(data[1].DataSet, data[1].HoldoutSet, priors);

                // Do transfer learning
                // Console.WriteLine("Testing Community Model");
                var personalisationExperiment = new Experiment {
                    TrainModel = trainModel, TestModel = testModel, Name = "Community"
                };
                personalisationExperiment.RunOnline(data[1].DataSet, data[1].HoldoutSet, communityExperiment.Posteriors);

                // Plot cumulative metrics
                Utils.PlotCumulativeMetrics(new[] { onlineExperiment, personalisationExperiment }, "Toy Transfer");
            }
            else
            {
                Console.WriteLine("Skipping Transfer Learning");
            }

            // ACTIVE MODEL
            if (testActive)
            {
                ActiveTransfer(trainModel, testModel, data, "Toy Active", priors);
            }
            else
            {
                Console.WriteLine("Skipping Active Learning");
            }

            if (testActiveTransfer)
            {
                Console.WriteLine("Note that the transfer learning is very effective here, so the active learning doesn't add much");
                ActiveTransfer(trainModel, testModel, data, "Toy Active Transfer", communityExperiment.Posteriors);
            }
            else
            {
                Console.WriteLine("Skipping Active Transfer Learning");
            }

            // Now create different costs for acquiring labels - want to demonstrate that we choose from all 3 possible labels
        }
示例#6
0
        /// <summary>
        /// Initializes a new instance of the <see cref="ActiveTransfer.ToyDataRunner"/> class.
        /// </summary>
        /// <param name="trainModel">Train model.</param>
        /// <param name="testModel">Test model.</param>
        public void Run(BinaryModel trainModel, BinaryModel testModel, BinaryModel evidenceModel, bool testVOI, bool testActiveEvidence)
        {
            const int    NumberOfResidents = 7;
            const double KeepProportion    = 1.0;

            var selectedFeatures = new HashSet <int>(Enumerable.Range(0, 48));

            var ted = Source.GetDataSet(Enumerable.Range(1, 14), AddBias, selectedFeatures, KeepProportion);
            var trd = Target.GetDataSet(Enumerable.Range(1, 25), AddBias, selectedFeatures, KeepProportion);
            // var ted = Source.GetDataSet( Enumerable.Range( 1, 1 ), AddBias, selectedFeatures, KeepProportion );
            // var trd = Target.GetDataSet( Enumerable.Range( 1, 20 ), AddBias, selectedFeatures, KeepProportion );
            // var hod = Target.GetDataSet( Enumerable.Range( 1 + NumberOfResidents * 1, NumberOfResidents ) );

            DataSet testSet;
            DataSet holdoutSet;

            ted.SplitTrainTest(0.5, out testSet, out holdoutSet);

            var NumFeatures = trd.Features.First().First().Count();

            var trainData = new ToyData
            {
                NumberOfResidents  = trd.NumberOfResidents,
                NumberOfFeatures   = NumFeatures,
                NumberOfActivities = 2,
                UseBias            = false,
                DataSet            = trd
            };

            var testData = new ToyData
            {
                NumberOfResidents  = NumberOfResidents,
                NumberOfFeatures   = NumFeatures,
                NumberOfActivities = 2,
                UseBias            = false,
                DataSet            = testSet,
                HoldoutSet         = holdoutSet
            };

            var priors = new Marginals
            {
                WeightMeans      = DistributionArrayHelpers.CreateGaussianArray(trainData.NumberOfFeatures, 0.0, 1.0).ToArray(),
                WeightPrecisions = DistributionArrayHelpers.CreateGammaArray(trainData.NumberOfFeatures, 1.0, 1.0).ToArray()
            };

            // TODO: Create meta-features that allow us to do the first form of transfer learning

            // Train the community model
            var communityExperiment = new Experiment
            {
                TrainModel    = trainModel,
                TestModel     = testModel,
                EvidenceModel = evidenceModel,
                Name          = "Community"
            };

            communityExperiment.RunBatch(trainData.DataSet, priors);
            // communityExperiment.Posteriors.WeightPrecisions = priors.WeightPrecisions;

            // if (false)
            // {
            //     Utils.PlotPosteriors(communityExperiment.Posteriors.WeightMeans, communityExperiment.Posteriors.WeightPrecisions, null, "Community weights", "Feature", ShowPlots);
            //     Utils.PlotPosteriors(communityExperiment.Posteriors.WeightMeans, communityExperiment.Posteriors.WeightPrecisions, null, "Community weights (prior precision)", "Feature", ShowPlots);
            // }

            // Print top features
            // var topWeights = communityExperiment.Posteriors.WeightMeans.Zip(communityExperiment.Posteriors.WeightPrecisions, (m, p) => new { m, p }).Select((ia, i) => new { ia, i })
            //  .OrderByDescending(x => Math.Abs(x.ia.m.GetMean())).ToList();
            // Console.WriteLine("Top 20 weights:\n {0}", string.Join("\n", topWeights.Take(20).Select(pair => string.Format("{0}: {1}", pair.i, new Gaussian(pair.ia.m.GetMean(), pair.ia.p.GetMean())))));

            // //communityExperiment.Posteriors.WeightPrecisions = DistributionArrayHelpers.Copy( priors.WeightPrecisions ).ToArray();
            var sourcePosteriors = new Marginals
            {
                WeightMeans      = communityExperiment.Posteriors.WeightMeans,
                WeightPrecisions = priors.WeightPrecisions, //communityExperiment.Posteriors.WeightMeans,
                Weights          = null
            };

            // Select half the features

            /*
             *          trainData.DataSet.Features = trainData.DataSet.Features.Select(
             *                  ia => ia.Select(
             *                          ib => topWeights.Take(topWeights.Count / 2).Select(pair => ib[pair.i]).ToArray())
             *                          .ToArray())
             *                  .ToArray();
             *
             *          // Retrain using these weights
             */

            // if (false)
            // {
            //  // Do online learning
            //  var onlineExperiment = new Experiment
            //  {
            //      TrainModel = trainModel,
            //      TestModel = testModel,
            //      Name = "Online"
            //  };

            //  onlineExperiment.RunOnline(testData.DataSet, testData.HoldoutSet, priors);

            //  // Do transfer learning
            //  var personalisationExperiment = new Experiment
            //  {
            //      TrainModel = trainModel,
            //      TestModel = testModel,
            //      Name = "Community"
            //  };

            //  personalisationExperiment.RunOnline(testData.DataSet, testData.HoldoutSet, communityExperiment.Posteriors);

            //  // Plot cumulative metrics
            //  Utils.PlotCumulativeMetrics(new [] { onlineExperiment, personalisationExperiment }, "Active", ShowPlots);
            // }

            // ACTIVE MODEL
            foreach (var doTransfer in new[] { false, true })
            {
                var experiments = new List <Experiment>();
                var learners    = CreateLearners(trainModel, testModel, evidenceModel, testData, testVOI, testActiveEvidence);

                foreach (var learner in learners)
                {
                    Console.WriteLine("Testing Active{0} Learning ({1})", doTransfer ? " Real Transfer" : "Real Online", learner.Key);
                    var experiment = new Experiment
                    {
                        TrainModel     = trainModel,
                        TestModel      = testModel,
                        Name           = learner.Key,
                        ActiveLearners = learner.Value
                    };

                    experiment.RunActive(testData.DataSet, testData.HoldoutSet, ActiveSteps, doTransfer ? sourcePosteriors : priors);
                    experiments.Add(experiment);

                    if (false)
                    {
                        Utils.PlotPosteriors(
                            experiment.IndividualPosteriors[0].WeightMeans,
                            experiment.IndividualPosteriors[0].WeightPrecisions,
                            null,
                            "Posterior weights for " + learner.Key + " " + (doTransfer ? " (transfer)" : ""),
                            "Feature",
                            ShowPlots);
                    }
                }

                Utils.PlotHoldoutMetrics(experiments, doTransfer ? "Real Active Transfer" : "Real Active", "", ShowPlots);
            }
        }