/// <summary>
/// Runs the batch.
/// </summary>
/// <param name="dataSet">Data set.</param>
/// <param name="priors">Priors.</param>
public void RunBatch(DataSet dataSet, Marginals priors, int niter = 1)
{
// Posteriors = priors;
// for (int rr = 0; rr < dataSet.NumberOfResidents; ++rr)
// {
// Posteriors = TrainModel.Train(dataSet.GetSubSet(rr), Posteriors);
// }
Posteriors = TrainModel.Train(dataSet, priors, niter);
}
/// <summary>
/// Runs the active experiment.
/// </summary>
/// <param name="dataSet">Data set.</param>
/// <param name="holdoutSet">Holdout set.</param>
/// <param name="numberOfSelections">Number of selections.</param>
/// <param name="priors">Priors.</param>
public void RunActive(DataSet dataSet, DataSet holdoutSet, int numberOfSelections, Marginals priors)
{
if (ActiveLearners == null)
{
throw new InvalidOperationException("Active Learner not provided");
}
using (new CodeTimer("Running active experiment: " + Name))
{
Console.WriteLine();
HoldoutMetrics = new HoldoutMetricsCollection {
Metrics = new Metrics[dataSet.NumberOfResidents][]
};
// Metrics = new MetricsCollection(numberOfSelections);
PosteriorActivities = new Bernoulli[dataSet.NumberOfResidents][];
HoldoutPosteriorActivities = new Bernoulli[dataSet.NumberOfResidents][][];
IndividualPosteriors = new Marginals[dataSet.NumberOfResidents];
var accuracy = new double[dataSet.NumberOfResidents][];
for (int i = 0; i < dataSet.NumberOfResidents; i++)
{
HoldoutMetrics.Metrics[i] = new Metrics[numberOfSelections];
var collection = new List <Metrics>();
IndividualPosteriors[i] = new Marginals(priors);
// Test on holdout set
HoldoutPosteriorActivities[i] = new Bernoulli[numberOfSelections][];
accuracy[i] = new double[numberOfSelections];
var dataSetForResident = dataSet.GetSubSet(i);
var holdoutSetForResident = holdoutSet.GetSubSet(i);
// ActiveLearners[i].Transfer(i, 1);
// var individualPosteriors = new Marginals(priors);
for (int j = 0; j < numberOfSelections; j++)
{
PosteriorActivities[i] = TestModel.Test(dataSetForResident, IndividualPosteriors[i])[0];
HoldoutPosteriorActivities[i][j] = TestModel.Test(holdoutSetForResident, IndividualPosteriors[i])[0];
if (ActiveLearners[i].Unlabelled.Count == 0)
{
Console.WriteLine("Empty unlabelled set");
break;
}
// int index = ActiveLearner.GetValueOfInformation(i).ArgMax();
int index;
double val;
ActiveLearners[i].GetArgMaxVOI(PosteriorActivities[i], IndividualPosteriors[i], out index, out val);
// Console.WriteLine("Index {0,4}, VOI {1:N4}", index, value);
// Now retrain using this label
ActiveLearners[i].UpdateModel(index);
//IndividualPosteriors [i] = TrainModel.Train( dataSet.GetSubSet(i, ActiveLearners [i].Labelled.ToList()), priors, 10);
IndividualPosteriors[i] = TrainModel.Train(dataSet.GetSubSet(i, index), IndividualPosteriors[i], 50);
var metrics = new Metrics {
Name = Name, Estimates = HoldoutPosteriorActivities[i][j], TrueLabels = holdoutSet.Labels[i]
};
accuracy[i][j] = metrics.AverageAccuracy;
collection.Add(metrics);
}
// PrintPredictions(posteriorActivities.Select(ia => ia[0]).ToArray(), testLabels.Select(ia => ia[0]).ToArray());
HoldoutMetrics.Metrics[i] = collection.ToArray();
Console.WriteLine("{0,20}, Resident {1}, \n\t\tClass ratio {5}, \n\t\tHold out accuracy {2:N2}, \n\t\tAccuracies {3} \n\t\tBriers {4}\n",
Name, i, collection.Average(ia => ia.AverageAccuracy).ToString("N2"),
string.Join(", ", collection.Select(ia => ia.AverageAccuracy.ToString("N2"))),
string.Join(", ", collection.Select(ia => ia.BrierScore.ToString("N2"))),
holdoutSet.Labels[i].Average().ToString("N2")
);
}
HoldoutMetrics.RecomputeAggregateMetrics();
}
}
/// <summary>
/// Runs the online experiment.
/// </summary>
/// <param name="dataSet">Data set.</param>
/// <param name="holdoutSet">Holdout set.</param>
/// <param name="priors">Priors.</param>
public void RunOnline(DataSet dataSet, DataSet holdoutSet, Marginals priors)
{
using (new CodeTimer("Running online experiment: " + Name))
{
Console.WriteLine();
Metrics = new MetricsCollection();
HoldoutMetrics = new HoldoutMetricsCollection {
Metrics = new Metrics[dataSet.NumberOfResidents][]
};
PosteriorActivities = new Bernoulli[dataSet.NumberOfResidents][];
HoldoutPosteriorActivities = new Bernoulli[dataSet.NumberOfResidents][][];
IndividualPosteriors = new Marginals[dataSet.NumberOfResidents];
var accuracy = new double[dataSet.NumberOfResidents][];
for (int i = 0; i < dataSet.NumberOfResidents; i++)
{
var collection = new List <Metrics>();
HoldoutPosteriorActivities[i] = new Bernoulli[dataSet.NumberOfInstances[i]][];
accuracy[i] = new double[dataSet.NumberOfInstances[i]];
IndividualPosteriors[i] = new Marginals(priors);
PosteriorActivities[i] = new Bernoulli[dataSet.NumberOfInstances[i]];
for (int j = 0; j < dataSet.NumberOfInstances[i]; j++)
{
var datum = dataSet.GetSubSet(i, j);
PosteriorActivities[i][j] = TestModel.Test(datum, IndividualPosteriors[i])[0][0];
HoldoutPosteriorActivities[i][j] = TestModel.Test(holdoutSet.GetSubSet(i), IndividualPosteriors[i])[0];
// Test on holdout set
var holdoutMetrics = new Metrics {
Name = Name, Estimates = HoldoutPosteriorActivities[i][j], TrueLabels = holdoutSet.Labels[i]
};
accuracy[i][j] = holdoutMetrics.AverageAccuracy;
// PrintPrediction(i, temp[0][0], testLabels[0][i], testScores[0][i]);
// Now retrain using this label
IndividualPosteriors[i] = TrainModel.Train(datum, IndividualPosteriors[i], 10);
collection.Add(holdoutMetrics);
}
// PrintPredictions(posteriorActivities.Select(ia => ia[0]).ToArray(), testLabels.Select(ia => ia[0]).ToArray());
Metrics.Add(new Metrics {
Name = Name, Estimates = PosteriorActivities[i], TrueLabels = dataSet.Labels[i]
}, true);
HoldoutMetrics.Metrics[i] = collection.ToArray();
Console.WriteLine("{0,20}, Resident {1}, Hold out accuracy {2:N2}", Name, i, collection.Average(ia => ia.AverageAccuracy));
}
HoldoutMetrics.RecomputeAggregateMetrics();
Metrics.RecomputeAggregateMetrics();
// Console.WriteLine("Accuracies " + string.Join(", ", accuracy.ColumnAverage().Select(x => x.ToString("N2"))));
// Console.WriteLine("Std. dev. " + string.Join(", ", accuracy.ColumnStandardDeviation().Select(x => x.ToString("N2"))));
// Console.WriteLine("Accuracies " + string.Join(", ", HoldoutMetrics.AverageAccuracy.Select(x => x.ToString("N2"))));
}
}
private double ExpectedEvidence(int index, Marginals priors)
{
var niter = 1;
//var pp = hypothesisActivityPosteriors[index];
bool trueLabel = DataSet.Labels[0][index];
var labelled = new HashSet <int>(Labelled);
labelled.Add(index);
//evidenceData.Labels[0] = evidenceData.Labels[0].Select( ll => !ll ).ToArray();
// Learn as if positive
DataSet.Labels[0][index] = true;
Marginals positivePosteriors = priors;
try
{
if (Reversed)
{
positivePosteriors = priors;
}
else
{
positivePosteriors = TrainModel.Train(DataSet.GetSubSet(0, index), priors, niter);
}
}
catch (ImproperMessageException)
{
// As fallback use priors
}
var positivePriorEvidence = EvidenceModel.ComputeEvidence(DataSet.GetSubSet(0, labelled.ToList()), priors);
var positivePostrEvidence = EvidenceModel.ComputeEvidence(DataSet.GetSubSet(0, labelled.ToList()), positivePosteriors);
// Learn as if negative
DataSet.Labels[0][index] = false;
Marginals negativePosteriors = priors;
try
{
if (Reversed)
{
negativePosteriors = priors;
}
else
{
negativePosteriors = TrainModel.Train(DataSet.GetSubSet(0, index), priors, niter);
}
}
catch (ImproperMessageException)
{
// As fallback use priors
}
var negativePriorEvidence = EvidenceModel.ComputeEvidence(DataSet.GetSubSet(0, labelled.ToList()), priors);
var negativePostrEvidence = EvidenceModel.ComputeEvidence(DataSet.GetSubSet(0, labelled.ToList()), negativePosteriors);
DataSet.Labels[0][index] = trueLabel;
var returns = new List <double>();
returns.Add(
(positivePriorEvidence.LogOdds) /
(negativePriorEvidence.LogOdds)
);
//return Math.Max( returns.Last(), 1.0 / returns.Last() );
returns.Add(
(positivePostrEvidence.LogOdds) /
(negativePostrEvidence.LogOdds)
);
//return Math.Max( returns.Last(), 1.0 / returns.Last() );
returns.Add(
(positivePriorEvidence.LogOdds + positivePostrEvidence.GetLogProbTrue()) /
(negativePriorEvidence.LogOdds + negativePostrEvidence.GetLogProbTrue())
);
return(Math.Max(returns.Last(), 1.0 / returns.Last()));
}
public void Validate(int NumberofTimes)
{
double error;
Accuracy = 0; int size = 0;
MeanError = VarianceError = 0;
List <double> errors = new List <double>(NumberofTimes);
for (int h = 0; h < NumberofTimes; ++h)
{
error = 0;
// shufflling all data randomly and sperate it to list of lists each list contains (datasize/#Ks) data
Data = ShuffleList <Data>(Data);
//0 list is for training, and 1 list is for testing
List <List <Data> > ShuffledData = new List <List <Data> >(K);
//List<Data> lst = new List<Data>();
for (int i = 0; i < Kfold; i++)
{
List <Data> lst = new List <Data>();
for (int j = 0; j < chunckSize[i]; j++)
{
lst.Add(Data[i * chunckSize[0] + j]);
}
ShuffledData.Add(lst);
}
List <Data> Training = new List <Data>();
List <Data> Testing = null;
for (int i = 0; i < K; ++i)
{
for (int j = 0; j < K; ++j)
{
if (j != i)
{
Training.AddRange(ShuffledData[j]);
}
else
{
Testing = ShuffledData[j];
}
}
// train and test k-folds
TrainModel train = new TrainModel(fType, cType);
train.Train(Training);
ScoreModel score = new ScoreModel(train, K);
score.Score(Testing);
error += score.MissedRows;
Accuracy += Testing.Count - score.MissedRows;
size += Testing.Count;
}
error /= Data.Count;
errors.Add(error);
MeanError += error;
}
error = 0;
MeanError /= NumberofTimes;
for (int i = 0; i < NumberofTimes; ++i)
{
error += Math.Pow(MeanError - errors[i], 2);
}
VarianceError = error / (NumberofTimes - 1);
Accuracy /= size * 100;
}
private double JAll_j(int index, Bernoulli[] activityPosteriors, Marginals priors)
{
// var prevProbs = activityPosteriors.Select(ia => new Bernoulli(ia)).ToArray();
hypothesisActivityPosteriors = activityPosteriors.Select(ia => new Bernoulli(ia)).ToArray();
// Get datum
// var datum = DataSet.GetSubSet(0, index);
bool trueLabel = DataSet.Labels[0][index];
// Create copies of the Labelled an Unlabelled sets
var labelled = new HashSet <int>(Labelled);
var unlabelled = new HashSet <int>(Unlabelled);
labelled.Add(index);
unlabelled.Remove(index);
// datum.Labels[0][0] = true;
DataSet.Labels[0][index] = true;
// Learn as if positive
// var positivePosteriors = TrainModel.Train(datum, priors, 1);
Marginals positivePosteriors = priors;
try
{
positivePosteriors = TrainModel.Train(DataSet.GetSubSet(0, index), priors, 1);
}
catch (ImproperMessageException)
{
// As fallback use priors
}
// recompute probabilities
CalculateProbabilities(positivePosteriors);
//var jjposl = JL(labelled);
//var jjposu = JU(unlabelled);
var Jjpos = (JAll()) * (1.0 - hypothesisActivityPosteriors[index].GetMean());
// Restore posteriors
labelled.Add(index);
unlabelled.Remove(index);
// datum.Labels[0][0] = false;
DataSet.Labels[0][index] = false;
// Learn as if negative
// var negativePosteriors = TrainModel.Train(datum, priors, 1);
Marginals negativePosteriors = priors;
try
{
negativePosteriors = TrainModel.Train(DataSet.GetSubSet(0, index), priors, 1);
}
catch (ImproperMessageException)
{
// As fallback use priors
}
// recompute probabilities
CalculateProbabilities(negativePosteriors);
//var jjnegl = JL(labelled);
//var jjnegu = JU(unlabelled);
var Jjneg = (JAll()) * (hypothesisActivityPosteriors[index].GetMean());
// restore posteriors
// activityPosteriors = prevProbs;
DataSet.Labels[0][index] = trueLabel;
var voi = Jjpos + Jjneg;
return(voi);
}
