/// <summary>
/// Active Transfer tests.
/// </summary>
/// <param name="trainModel">The train model.</param>
/// <param name="testModel">The test model.</param>
/// <param name="data">The datasets.</param>
public static void ActiveTransfer(BinaryModel trainModel, BinaryModel testModel, IList<ToyData> data, string title, Marginals priors)
{
var learners = new Dictionary<string, IList<IActiveLearner>>
{
{ "Random", Utils.CreateLearners<RandomLearner>(data[2].DataSet, trainModel, testModel, null) },
{ "US", Utils.CreateLearners<UncertainActiveLearner>(data[2].DataSet, trainModel, testModel, null) },
{ "CS", Utils.CreateLearners<UncertainActiveLearner>(data[2].DataSet, trainModel, testModel, null, true) },
{ "VOI+", Utils.CreateLearners<ActiveLearner>(data[2].DataSet, trainModel, testModel, null) },
{ "VOI-", Utils.CreateLearners<ActiveLearner>(data[2].DataSet, trainModel, testModel, null, true) }
};
var experiments = new List<Experiment>();
foreach (var learner in learners)
{
Console.WriteLine("Testing {0} ({1})", title, learner.Key);
var experiment = new Experiment { TrainModel = trainModel, TestModel = testModel, Name = learner.Key, ActiveLearners = learner.Value };
experiment.RunActive(data[2].DataSet, data[2].HoldoutSet, ActiveSteps, priors);
experiments.Add(experiment);
}
Utils.PlotHoldoutMetrics(experiments, title, "", true);
}
/// <summary>
/// Initializes a new instance of the <see cref="ToyDataRunner"/> class.
/// </summary>
/// <param name="trainModel">Train model.</param>
/// <param name="testModel">Test model.</param>
public static void Run(BinaryModel trainModel, BinaryModel testModel, bool testTransfer, bool testActive, bool testActiveTransfer)
{
var phase1PriorMean = new Gaussian(4, 1);
var phase1PriorPrecision = new Gamma(1, 1);
var phase2PriorMean = new Gaussian(4, 1);
var phase2PriorPrecision = new Gamma(1, 1);
// Generate data for 5 individuals
var data = new List<ToyData>();
for (int i = 0; i < 3; i++)
{
var toy = new ToyData
{
// NumberOfInstances = 200,
// NumberOfHoldoutInstances = i == 0 ? 0 : 1000,
NumberOfResidents = 5,
NumberOfFeatures = NumberOfFeatures,
NumberOfActivities = 2,
UseBias = false,
TruePriorMean = i == 0 ? phase1PriorMean : phase2PriorMean,
TruePriorPrecision = i == 0 ? phase1PriorPrecision : phase2PriorPrecision
};
toy.Generate(i == 2 ? NoisyExampleProportion : 0.0, 200);
if (i != 0)
{
// no need for holdout data in training set
toy.Generate(0.0, 1000, true);
}
data.Add(toy);
}
var priors = new Marginals
{
WeightMeans = DistributionArrayHelpers.CreateGaussianArray(NumberOfFeatures, 0, 1).ToArray(),
WeightPrecisions = DistributionArrayHelpers.CreateGammaArray(NumberOfFeatures, 1, 1).ToArray()
};
Console.WriteLine("Data Generated");
// TODO: Create meta-features that allow us to do the first form of transfer learning
// Train the community model
Console.WriteLine("Training Community Model");
var communityExperiment = new Experiment { TrainModel = trainModel, TestModel = testModel, Name = "Community" };
communityExperiment.RunBatch(data[0].DataSet, priors);
// PrintWeightPriors(communityExperiment.Posteriors, trainData.CommunityWeights);
// Utils.PlotPosteriors(communityExperiment.Posteriors.Weights, data[0].Weights);
// Utils.PlotPosteriors(communityExperiment.Posteriors.WeightMeans, communityExperiment.Posteriors.WeightPrecisions, null, "Community weights", "Feature");
// return;
if (testTransfer)
{
// Do online learning
// Console.WriteLine("Testing Online Model");
var onlineExperiment = new Experiment { TrainModel = trainModel, TestModel = testModel, Name = "Online" };
onlineExperiment.RunOnline(data[1].DataSet, data[1].HoldoutSet, priors);
// Do transfer learning
// Console.WriteLine("Testing Community Model");
var personalisationExperiment = new Experiment { TrainModel = trainModel, TestModel = testModel, Name = "Community" };
personalisationExperiment.RunOnline(data[1].DataSet, data[1].HoldoutSet, communityExperiment.Posteriors);
// Plot cumulative metrics
Utils.PlotCumulativeMetrics(new[] { onlineExperiment, personalisationExperiment }, "Toy Transfer");
}
else
{
Console.WriteLine("Skipping Transfer Learning");
}
// ACTIVE MODEL
if (testActive)
{
ActiveTransfer(trainModel, testModel, data, "Toy Active", priors);
}
else
{
Console.WriteLine("Skipping Active Learning");
}
if (testActiveTransfer)
{
Console.WriteLine("Note that the transfer learning is very effective here, so the active learning doesn't add much");
ActiveTransfer(trainModel, testModel, data, "Toy Active Transfer", communityExperiment.Posteriors);
}
else
{
Console.WriteLine("Skipping Active Transfer Learning");
}
// Now create different costs for acquiring labels - want to demonstrate that we choose from all 3 possible labels
}
/// <summary>
/// Initializes a new instance of the <see cref="ToyDataRunner"/> class.
/// </summary>
/// <param name="trainModel">Train model.</param>
/// <param name="testModel">Test model.</param>
public static void Run(BinaryModel trainModel, BinaryModel testModel, bool testTransfer, bool testActive, bool testActiveTransfer)
{
var phase1PriorMean = new Gaussian(4, 1);
var phase1PriorPrecision = new Gamma(1, 1);
var phase2PriorMean = new Gaussian(4, 1);
var phase2PriorPrecision = new Gamma(1, 1);
// Generate data for 5 individuals
var data = new List <ToyData>();
for (int i = 0; i < 3; i++)
{
var toy = new ToyData
{
// NumberOfInstances = 200,
// NumberOfHoldoutInstances = i == 0 ? 0 : 1000,
NumberOfResidents = 5,
NumberOfFeatures = NumberOfFeatures,
NumberOfActivities = 2,
UseBias = false,
TruePriorMean = i == 0 ? phase1PriorMean : phase2PriorMean,
TruePriorPrecision = i == 0 ? phase1PriorPrecision : phase2PriorPrecision
};
toy.Generate(i == 2 ? NoisyExampleProportion : 0.0, 200);
if (i != 0)
{
// no need for holdout data in training set
toy.Generate(0.0, 1000, true);
}
data.Add(toy);
}
var priors = new Marginals
{
WeightMeans = DistributionArrayHelpers.CreateGaussianArray(NumberOfFeatures, 0, 1).ToArray(),
WeightPrecisions = DistributionArrayHelpers.CreateGammaArray(NumberOfFeatures, 1, 1).ToArray()
};
Console.WriteLine("Data Generated");
// TODO: Create meta-features that allow us to do the first form of transfer learning
// Train the community model
Console.WriteLine("Training Community Model");
var communityExperiment = new Experiment {
TrainModel = trainModel, TestModel = testModel, Name = "Community"
};
communityExperiment.RunBatch(data[0].DataSet, priors);
// PrintWeightPriors(communityExperiment.Posteriors, trainData.CommunityWeights);
// Utils.PlotPosteriors(communityExperiment.Posteriors.Weights, data[0].Weights);
// Utils.PlotPosteriors(communityExperiment.Posteriors.WeightMeans, communityExperiment.Posteriors.WeightPrecisions, null, "Community weights", "Feature");
// return;
if (testTransfer)
{
// Do online learning
// Console.WriteLine("Testing Online Model");
var onlineExperiment = new Experiment {
TrainModel = trainModel, TestModel = testModel, Name = "Online"
};
onlineExperiment.RunOnline(data[1].DataSet, data[1].HoldoutSet, priors);
// Do transfer learning
// Console.WriteLine("Testing Community Model");
var personalisationExperiment = new Experiment {
TrainModel = trainModel, TestModel = testModel, Name = "Community"
};
personalisationExperiment.RunOnline(data[1].DataSet, data[1].HoldoutSet, communityExperiment.Posteriors);
// Plot cumulative metrics
Utils.PlotCumulativeMetrics(new[] { onlineExperiment, personalisationExperiment }, "Toy Transfer");
}
else
{
Console.WriteLine("Skipping Transfer Learning");
}
// ACTIVE MODEL
if (testActive)
{
ActiveTransfer(trainModel, testModel, data, "Toy Active", priors);
}
else
{
Console.WriteLine("Skipping Active Learning");
}
if (testActiveTransfer)
{
Console.WriteLine("Note that the transfer learning is very effective here, so the active learning doesn't add much");
ActiveTransfer(trainModel, testModel, data, "Toy Active Transfer", communityExperiment.Posteriors);
}
else
{
Console.WriteLine("Skipping Active Transfer Learning");
}
// Now create different costs for acquiring labels - want to demonstrate that we choose from all 3 possible labels
}
/// <summary>
/// Initializes a new instance of the <see cref="ActiveTransfer.ToyDataRunner"/> class.
/// </summary>
/// <param name="trainModel">Train model.</param>
/// <param name="testModel">Test model.</param>
public void Run(BinaryModel trainModel, BinaryModel testModel, BinaryModel evidenceModel, bool testVOI, bool testActiveEvidence)
{
const int NumberOfResidents = 7;
const double KeepProportion = 1.0;
var selectedFeatures = new HashSet<int>(Enumerable.Range(0, 48));
var ted = Source.GetDataSet(Enumerable.Range(1, 14), AddBias, selectedFeatures, KeepProportion);
var trd = Target.GetDataSet(Enumerable.Range(1, 25), AddBias, selectedFeatures, KeepProportion);
// var ted = Source.GetDataSet( Enumerable.Range( 1, 1 ), AddBias, selectedFeatures, KeepProportion );
// var trd = Target.GetDataSet( Enumerable.Range( 1, 20 ), AddBias, selectedFeatures, KeepProportion );
// var hod = Target.GetDataSet( Enumerable.Range( 1 + NumberOfResidents * 1, NumberOfResidents ) );
DataSet testSet;
DataSet holdoutSet;
ted.SplitTrainTest(0.5, out testSet, out holdoutSet);
var NumFeatures = trd.Features.First().First().Count();
var trainData = new ToyData
{
NumberOfResidents = trd.NumberOfResidents,
NumberOfFeatures = NumFeatures,
NumberOfActivities = 2,
UseBias = false,
DataSet = trd
};
var testData = new ToyData
{
NumberOfResidents = NumberOfResidents,
NumberOfFeatures = NumFeatures,
NumberOfActivities = 2,
UseBias = false,
DataSet = testSet,
HoldoutSet = holdoutSet
};
var priors = new Marginals
{
WeightMeans = DistributionArrayHelpers.CreateGaussianArray(trainData.NumberOfFeatures, 0.0, 1.0).ToArray(),
WeightPrecisions = DistributionArrayHelpers.CreateGammaArray(trainData.NumberOfFeatures, 1.0, 1.0).ToArray()
};
// TODO: Create meta-features that allow us to do the first form of transfer learning
// Train the community model
var communityExperiment = new Experiment
{
TrainModel = trainModel,
TestModel = testModel,
EvidenceModel = evidenceModel,
Name = "Community"
};
communityExperiment.RunBatch(trainData.DataSet, priors);
// communityExperiment.Posteriors.WeightPrecisions = priors.WeightPrecisions;
// if (false)
// {
// Utils.PlotPosteriors(communityExperiment.Posteriors.WeightMeans, communityExperiment.Posteriors.WeightPrecisions, null, "Community weights", "Feature", ShowPlots);
// Utils.PlotPosteriors(communityExperiment.Posteriors.WeightMeans, communityExperiment.Posteriors.WeightPrecisions, null, "Community weights (prior precision)", "Feature", ShowPlots);
// }
// Print top features
// var topWeights = communityExperiment.Posteriors.WeightMeans.Zip(communityExperiment.Posteriors.WeightPrecisions, (m, p) => new { m, p }).Select((ia, i) => new { ia, i })
// .OrderByDescending(x => Math.Abs(x.ia.m.GetMean())).ToList();
// Console.WriteLine("Top 20 weights:\n {0}", string.Join("\n", topWeights.Take(20).Select(pair => string.Format("{0}: {1}", pair.i, new Gaussian(pair.ia.m.GetMean(), pair.ia.p.GetMean())))));
// //communityExperiment.Posteriors.WeightPrecisions = DistributionArrayHelpers.Copy( priors.WeightPrecisions ).ToArray();
var sourcePosteriors = new Marginals
{
WeightMeans = communityExperiment.Posteriors.WeightMeans,
WeightPrecisions = priors.WeightPrecisions, //communityExperiment.Posteriors.WeightMeans,
Weights = null
};
// Select half the features
/*
trainData.DataSet.Features = trainData.DataSet.Features.Select(
ia => ia.Select(
ib => topWeights.Take(topWeights.Count / 2).Select(pair => ib[pair.i]).ToArray())
.ToArray())
.ToArray();
// Retrain using these weights
*/
// if (false)
// {
// // Do online learning
// var onlineExperiment = new Experiment
// {
// TrainModel = trainModel,
// TestModel = testModel,
// Name = "Online"
// };
// onlineExperiment.RunOnline(testData.DataSet, testData.HoldoutSet, priors);
// // Do transfer learning
// var personalisationExperiment = new Experiment
// {
// TrainModel = trainModel,
// TestModel = testModel,
// Name = "Community"
// };
// personalisationExperiment.RunOnline(testData.DataSet, testData.HoldoutSet, communityExperiment.Posteriors);
// // Plot cumulative metrics
// Utils.PlotCumulativeMetrics(new [] { onlineExperiment, personalisationExperiment }, "Active", ShowPlots);
// }
// ACTIVE MODEL
foreach (var doTransfer in new[] { false, true })
{
var experiments = new List<Experiment>();
var learners = CreateLearners(trainModel, testModel, evidenceModel, testData, testVOI, testActiveEvidence);
foreach (var learner in learners)
{
Console.WriteLine("Testing Active{0} Learning ({1})", doTransfer ? " Real Transfer" : "Real Online", learner.Key);
var experiment = new Experiment
{
TrainModel = trainModel,
TestModel = testModel,
Name = learner.Key,
ActiveLearners = learner.Value
};
experiment.RunActive(testData.DataSet, testData.HoldoutSet, ActiveSteps, doTransfer ? sourcePosteriors : priors);
experiments.Add(experiment);
if (false)
{
Utils.PlotPosteriors(
experiment.IndividualPosteriors[0].WeightMeans,
experiment.IndividualPosteriors[0].WeightPrecisions,
null,
"Posterior weights for " + learner.Key + " " + (doTransfer ? " (transfer)" : ""),
"Feature",
ShowPlots);
}
}
Utils.PlotHoldoutMetrics(experiments, doTransfer ? "Real Active Transfer" : "Real Active", "", ShowPlots);
}
}
/// <summary>
/// Initializes a new instance of the <see cref="ActiveTransfer.ToyDataRunner"/> class.
/// </summary>
/// <param name="trainModel">Train model.</param>
/// <param name="testModel">Test model.</param>
public void Run(BinaryModel trainModel, BinaryModel testModel, BinaryModel evidenceModel, bool testVOI, bool testActiveEvidence)
{
const int NumberOfResidents = 7;
const double KeepProportion = 1.0;
var selectedFeatures = new HashSet <int>(Enumerable.Range(0, 48));
var ted = Source.GetDataSet(Enumerable.Range(1, 14), AddBias, selectedFeatures, KeepProportion);
var trd = Target.GetDataSet(Enumerable.Range(1, 25), AddBias, selectedFeatures, KeepProportion);
// var ted = Source.GetDataSet( Enumerable.Range( 1, 1 ), AddBias, selectedFeatures, KeepProportion );
// var trd = Target.GetDataSet( Enumerable.Range( 1, 20 ), AddBias, selectedFeatures, KeepProportion );
// var hod = Target.GetDataSet( Enumerable.Range( 1 + NumberOfResidents * 1, NumberOfResidents ) );
DataSet testSet;
DataSet holdoutSet;
ted.SplitTrainTest(0.5, out testSet, out holdoutSet);
var NumFeatures = trd.Features.First().First().Count();
var trainData = new ToyData
{
NumberOfResidents = trd.NumberOfResidents,
NumberOfFeatures = NumFeatures,
NumberOfActivities = 2,
UseBias = false,
DataSet = trd
};
var testData = new ToyData
{
NumberOfResidents = NumberOfResidents,
NumberOfFeatures = NumFeatures,
NumberOfActivities = 2,
UseBias = false,
DataSet = testSet,
HoldoutSet = holdoutSet
};
var priors = new Marginals
{
WeightMeans = DistributionArrayHelpers.CreateGaussianArray(trainData.NumberOfFeatures, 0.0, 1.0).ToArray(),
WeightPrecisions = DistributionArrayHelpers.CreateGammaArray(trainData.NumberOfFeatures, 1.0, 1.0).ToArray()
};
// TODO: Create meta-features that allow us to do the first form of transfer learning
// Train the community model
var communityExperiment = new Experiment
{
TrainModel = trainModel,
TestModel = testModel,
EvidenceModel = evidenceModel,
Name = "Community"
};
communityExperiment.RunBatch(trainData.DataSet, priors);
// communityExperiment.Posteriors.WeightPrecisions = priors.WeightPrecisions;
// if (false)
// {
// Utils.PlotPosteriors(communityExperiment.Posteriors.WeightMeans, communityExperiment.Posteriors.WeightPrecisions, null, "Community weights", "Feature", ShowPlots);
// Utils.PlotPosteriors(communityExperiment.Posteriors.WeightMeans, communityExperiment.Posteriors.WeightPrecisions, null, "Community weights (prior precision)", "Feature", ShowPlots);
// }
// Print top features
// var topWeights = communityExperiment.Posteriors.WeightMeans.Zip(communityExperiment.Posteriors.WeightPrecisions, (m, p) => new { m, p }).Select((ia, i) => new { ia, i })
// .OrderByDescending(x => Math.Abs(x.ia.m.GetMean())).ToList();
// Console.WriteLine("Top 20 weights:\n {0}", string.Join("\n", topWeights.Take(20).Select(pair => string.Format("{0}: {1}", pair.i, new Gaussian(pair.ia.m.GetMean(), pair.ia.p.GetMean())))));
// //communityExperiment.Posteriors.WeightPrecisions = DistributionArrayHelpers.Copy( priors.WeightPrecisions ).ToArray();
var sourcePosteriors = new Marginals
{
WeightMeans = communityExperiment.Posteriors.WeightMeans,
WeightPrecisions = priors.WeightPrecisions, //communityExperiment.Posteriors.WeightMeans,
Weights = null
};
// Select half the features
/*
* trainData.DataSet.Features = trainData.DataSet.Features.Select(
* ia => ia.Select(
* ib => topWeights.Take(topWeights.Count / 2).Select(pair => ib[pair.i]).ToArray())
* .ToArray())
* .ToArray();
*
* // Retrain using these weights
*/
// if (false)
// {
// // Do online learning
// var onlineExperiment = new Experiment
// {
// TrainModel = trainModel,
// TestModel = testModel,
// Name = "Online"
// };
// onlineExperiment.RunOnline(testData.DataSet, testData.HoldoutSet, priors);
// // Do transfer learning
// var personalisationExperiment = new Experiment
// {
// TrainModel = trainModel,
// TestModel = testModel,
// Name = "Community"
// };
// personalisationExperiment.RunOnline(testData.DataSet, testData.HoldoutSet, communityExperiment.Posteriors);
// // Plot cumulative metrics
// Utils.PlotCumulativeMetrics(new [] { onlineExperiment, personalisationExperiment }, "Active", ShowPlots);
// }
// ACTIVE MODEL
foreach (var doTransfer in new[] { false, true })
{
var experiments = new List <Experiment>();
var learners = CreateLearners(trainModel, testModel, evidenceModel, testData, testVOI, testActiveEvidence);
foreach (var learner in learners)
{
Console.WriteLine("Testing Active{0} Learning ({1})", doTransfer ? " Real Transfer" : "Real Online", learner.Key);
var experiment = new Experiment
{
TrainModel = trainModel,
TestModel = testModel,
Name = learner.Key,
ActiveLearners = learner.Value
};
experiment.RunActive(testData.DataSet, testData.HoldoutSet, ActiveSteps, doTransfer ? sourcePosteriors : priors);
experiments.Add(experiment);
if (false)
{
Utils.PlotPosteriors(
experiment.IndividualPosteriors[0].WeightMeans,
experiment.IndividualPosteriors[0].WeightPrecisions,
null,
"Posterior weights for " + learner.Key + " " + (doTransfer ? " (transfer)" : ""),
"Feature",
ShowPlots);
}
}
Utils.PlotHoldoutMetrics(experiments, doTransfer ? "Real Active Transfer" : "Real Active", "", ShowPlots);
}
}