/// <summary> /// Runs the active experiment. /// </summary> /// <param name="dataSet">Data set.</param> /// <param name="holdoutSet">Holdout set.</param> /// <param name="numberOfSelections">Number of selections.</param> /// <param name="priors">Priors.</param> public void RunActive(DataSet dataSet, DataSet holdoutSet, int numberOfSelections, Marginals priors) { if (ActiveLearners == null) { throw new InvalidOperationException("Active Learner not provided"); } using (new CodeTimer("Running active experiment: " + Name)) { Console.WriteLine(); HoldoutMetrics = new HoldoutMetricsCollection { Metrics = new Metrics[dataSet.NumberOfResidents][] }; // Metrics = new MetricsCollection(numberOfSelections); PosteriorActivities = new Bernoulli[dataSet.NumberOfResidents][]; HoldoutPosteriorActivities = new Bernoulli[dataSet.NumberOfResidents][][]; IndividualPosteriors = new Marginals[dataSet.NumberOfResidents]; var accuracy = new double[dataSet.NumberOfResidents][]; for (int i = 0; i < dataSet.NumberOfResidents; i++) { HoldoutMetrics.Metrics[i] = new Metrics[numberOfSelections]; var collection = new List <Metrics>(); IndividualPosteriors[i] = new Marginals(priors); // Test on holdout set HoldoutPosteriorActivities[i] = new Bernoulli[numberOfSelections][]; accuracy[i] = new double[numberOfSelections]; var dataSetForResident = dataSet.GetSubSet(i); var holdoutSetForResident = holdoutSet.GetSubSet(i); // ActiveLearners[i].Transfer(i, 1); // var individualPosteriors = new Marginals(priors); for (int j = 0; j < numberOfSelections; j++) { PosteriorActivities[i] = TestModel.Test(dataSetForResident, IndividualPosteriors[i])[0]; HoldoutPosteriorActivities[i][j] = TestModel.Test(holdoutSetForResident, IndividualPosteriors[i])[0]; if (ActiveLearners[i].Unlabelled.Count == 0) { Console.WriteLine("Empty unlabelled set"); break; } // int index = ActiveLearner.GetValueOfInformation(i).ArgMax(); int index; double val; ActiveLearners[i].GetArgMaxVOI(PosteriorActivities[i], IndividualPosteriors[i], out index, out val); // Console.WriteLine("Index {0,4}, VOI {1:N4}", index, value); // Now retrain using this label ActiveLearners[i].UpdateModel(index); //IndividualPosteriors [i] = TrainModel.Train( dataSet.GetSubSet(i, ActiveLearners [i].Labelled.ToList()), priors, 10); IndividualPosteriors[i] = TrainModel.Train(dataSet.GetSubSet(i, index), IndividualPosteriors[i], 50); var metrics = new Metrics { Name = Name, Estimates = HoldoutPosteriorActivities[i][j], TrueLabels = holdoutSet.Labels[i] }; accuracy[i][j] = metrics.AverageAccuracy; collection.Add(metrics); } // PrintPredictions(posteriorActivities.Select(ia => ia[0]).ToArray(), testLabels.Select(ia => ia[0]).ToArray()); HoldoutMetrics.Metrics[i] = collection.ToArray(); Console.WriteLine("{0,20}, Resident {1}, \n\t\tClass ratio {5}, \n\t\tHold out accuracy {2:N2}, \n\t\tAccuracies {3} \n\t\tBriers {4}\n", Name, i, collection.Average(ia => ia.AverageAccuracy).ToString("N2"), string.Join(", ", collection.Select(ia => ia.AverageAccuracy.ToString("N2"))), string.Join(", ", collection.Select(ia => ia.BrierScore.ToString("N2"))), holdoutSet.Labels[i].Average().ToString("N2") ); } HoldoutMetrics.RecomputeAggregateMetrics(); } }
/// <summary> /// Runs the online experiment. /// </summary> /// <param name="dataSet">Data set.</param> /// <param name="holdoutSet">Holdout set.</param> /// <param name="priors">Priors.</param> public void RunOnline(DataSet dataSet, DataSet holdoutSet, Marginals priors) { using (new CodeTimer("Running online experiment: " + Name)) { Console.WriteLine(); Metrics = new MetricsCollection(); HoldoutMetrics = new HoldoutMetricsCollection { Metrics = new Metrics[dataSet.NumberOfResidents][] }; PosteriorActivities = new Bernoulli[dataSet.NumberOfResidents][]; HoldoutPosteriorActivities = new Bernoulli[dataSet.NumberOfResidents][][]; IndividualPosteriors = new Marginals[dataSet.NumberOfResidents]; var accuracy = new double[dataSet.NumberOfResidents][]; for (int i = 0; i < dataSet.NumberOfResidents; i++) { var collection = new List<Metrics>(); HoldoutPosteriorActivities[i] = new Bernoulli[dataSet.NumberOfInstances[i]][]; accuracy[i] = new double[dataSet.NumberOfInstances[i]]; IndividualPosteriors[i] = new Marginals(priors); PosteriorActivities[i] = new Bernoulli[dataSet.NumberOfInstances[i]]; for (int j = 0; j < dataSet.NumberOfInstances[i]; j++) { var datum = dataSet.GetSubSet(i, j); PosteriorActivities[i][j] = TestModel.Test(datum, IndividualPosteriors[i])[0][0]; HoldoutPosteriorActivities[i][j] = TestModel.Test(holdoutSet.GetSubSet(i), IndividualPosteriors[i])[0]; // Test on holdout set var holdoutMetrics = new Metrics { Name = Name, Estimates = HoldoutPosteriorActivities[i][j], TrueLabels = holdoutSet.Labels[i] }; accuracy[i][j] = holdoutMetrics.AverageAccuracy; // PrintPrediction(i, temp[0][0], testLabels[0][i], testScores[0][i]); // Now retrain using this label IndividualPosteriors[i] = TrainModel.Train(datum, IndividualPosteriors[i], 10); collection.Add(holdoutMetrics); } // PrintPredictions(posteriorActivities.Select(ia => ia[0]).ToArray(), testLabels.Select(ia => ia[0]).ToArray()); Metrics.Add(new Metrics { Name = Name, Estimates = PosteriorActivities[i], TrueLabels = dataSet.Labels[i] }, true); HoldoutMetrics.Metrics[i] = collection.ToArray(); Console.WriteLine("{0,20}, Resident {1}, Hold out accuracy {2:N2}", Name, i, collection.Average(ia => ia.AverageAccuracy)); } HoldoutMetrics.RecomputeAggregateMetrics(); Metrics.RecomputeAggregateMetrics(); // Console.WriteLine("Accuracies " + string.Join(", ", accuracy.ColumnAverage().Select(x => x.ToString("N2")))); // Console.WriteLine("Std. dev. " + string.Join(", ", accuracy.ColumnStandardDeviation().Select(x => x.ToString("N2")))); // Console.WriteLine("Accuracies " + string.Join(", ", HoldoutMetrics.AverageAccuracy.Select(x => x.ToString("N2")))); } }
/// <summary> /// Runs the online experiment. /// </summary> /// <param name="dataSet">Data set.</param> /// <param name="holdoutSet">Holdout set.</param> /// <param name="priors">Priors.</param> public void RunOnline(DataSet dataSet, DataSet holdoutSet, Marginals priors) { using (new CodeTimer("Running online experiment: " + Name)) { Console.WriteLine(); Metrics = new MetricsCollection(); HoldoutMetrics = new HoldoutMetricsCollection { Metrics = new Metrics[dataSet.NumberOfResidents][] }; PosteriorActivities = new Bernoulli[dataSet.NumberOfResidents][]; HoldoutPosteriorActivities = new Bernoulli[dataSet.NumberOfResidents][][]; IndividualPosteriors = new Marginals[dataSet.NumberOfResidents]; var accuracy = new double[dataSet.NumberOfResidents][]; for (int i = 0; i < dataSet.NumberOfResidents; i++) { var collection = new List <Metrics>(); HoldoutPosteriorActivities[i] = new Bernoulli[dataSet.NumberOfInstances[i]][]; accuracy[i] = new double[dataSet.NumberOfInstances[i]]; IndividualPosteriors[i] = new Marginals(priors); PosteriorActivities[i] = new Bernoulli[dataSet.NumberOfInstances[i]]; for (int j = 0; j < dataSet.NumberOfInstances[i]; j++) { var datum = dataSet.GetSubSet(i, j); PosteriorActivities[i][j] = TestModel.Test(datum, IndividualPosteriors[i])[0][0]; HoldoutPosteriorActivities[i][j] = TestModel.Test(holdoutSet.GetSubSet(i), IndividualPosteriors[i])[0]; // Test on holdout set var holdoutMetrics = new Metrics { Name = Name, Estimates = HoldoutPosteriorActivities[i][j], TrueLabels = holdoutSet.Labels[i] }; accuracy[i][j] = holdoutMetrics.AverageAccuracy; // PrintPrediction(i, temp[0][0], testLabels[0][i], testScores[0][i]); // Now retrain using this label IndividualPosteriors[i] = TrainModel.Train(datum, IndividualPosteriors[i], 10); collection.Add(holdoutMetrics); } // PrintPredictions(posteriorActivities.Select(ia => ia[0]).ToArray(), testLabels.Select(ia => ia[0]).ToArray()); Metrics.Add(new Metrics { Name = Name, Estimates = PosteriorActivities[i], TrueLabels = dataSet.Labels[i] }, true); HoldoutMetrics.Metrics[i] = collection.ToArray(); Console.WriteLine("{0,20}, Resident {1}, Hold out accuracy {2:N2}", Name, i, collection.Average(ia => ia.AverageAccuracy)); } HoldoutMetrics.RecomputeAggregateMetrics(); Metrics.RecomputeAggregateMetrics(); // Console.WriteLine("Accuracies " + string.Join(", ", accuracy.ColumnAverage().Select(x => x.ToString("N2")))); // Console.WriteLine("Std. dev. " + string.Join(", ", accuracy.ColumnStandardDeviation().Select(x => x.ToString("N2")))); // Console.WriteLine("Accuracies " + string.Join(", ", HoldoutMetrics.AverageAccuracy.Select(x => x.ToString("N2")))); } }
/// <summary> /// Runs the active experiment. /// </summary> /// <param name="dataSet">Data set.</param> /// <param name="holdoutSet">Holdout set.</param> /// <param name="numberOfSelections">Number of selections.</param> /// <param name="priors">Priors.</param> public void RunActive(DataSet dataSet, DataSet holdoutSet, int numberOfSelections, Marginals priors) { if (ActiveLearners == null) { throw new InvalidOperationException("Active Learner not provided"); } using (new CodeTimer("Running active experiment: " + Name)) { Console.WriteLine(); HoldoutMetrics = new HoldoutMetricsCollection { Metrics = new Metrics[dataSet.NumberOfResidents][] }; // Metrics = new MetricsCollection(numberOfSelections); PosteriorActivities = new Bernoulli[dataSet.NumberOfResidents][]; HoldoutPosteriorActivities = new Bernoulli[dataSet.NumberOfResidents][][]; IndividualPosteriors = new Marginals[dataSet.NumberOfResidents]; var accuracy = new double[dataSet.NumberOfResidents][]; for (int i = 0; i < dataSet.NumberOfResidents; i++) { HoldoutMetrics.Metrics[i] = new Metrics[numberOfSelections]; var collection = new List<Metrics>(); IndividualPosteriors[i] = new Marginals(priors); // Test on holdout set HoldoutPosteriorActivities[i] = new Bernoulli[numberOfSelections][]; accuracy[i] = new double[numberOfSelections]; var dataSetForResident = dataSet.GetSubSet(i); var holdoutSetForResident = holdoutSet.GetSubSet(i); // ActiveLearners[i].Transfer(i, 1); // var individualPosteriors = new Marginals(priors); for (int j = 0; j < numberOfSelections; j++) { PosteriorActivities[i] = TestModel.Test(dataSetForResident, IndividualPosteriors[i])[0]; HoldoutPosteriorActivities[i][j] = TestModel.Test(holdoutSetForResident, IndividualPosteriors[i])[0]; if (ActiveLearners[i].Unlabelled.Count == 0) { Console.WriteLine("Empty unlabelled set"); break; } // int index = ActiveLearner.GetValueOfInformation(i).ArgMax(); int index; double val; ActiveLearners[i].GetArgMaxVOI(PosteriorActivities[i], IndividualPosteriors[i], out index, out val); // Console.WriteLine("Index {0,4}, VOI {1:N4}", index, value); // Now retrain using this label ActiveLearners[i].UpdateModel(index); //IndividualPosteriors [i] = TrainModel.Train( dataSet.GetSubSet(i, ActiveLearners [i].Labelled.ToList()), priors, 10); IndividualPosteriors[i] = TrainModel.Train(dataSet.GetSubSet(i, index), IndividualPosteriors[i], 50); var metrics = new Metrics { Name = Name, Estimates = HoldoutPosteriorActivities[i][j], TrueLabels = holdoutSet.Labels[i] }; accuracy[i][j] = metrics.AverageAccuracy; collection.Add(metrics); } // PrintPredictions(posteriorActivities.Select(ia => ia[0]).ToArray(), testLabels.Select(ia => ia[0]).ToArray()); HoldoutMetrics.Metrics[i] = collection.ToArray(); Console.WriteLine("{0,20}, Resident {1}, \n\t\tClass ratio {5}, \n\t\tHold out accuracy {2:N2}, \n\t\tAccuracies {3} \n\t\tBriers {4}\n", Name, i, collection.Average(ia => ia.AverageAccuracy).ToString("N2"), string.Join(", ", collection.Select(ia => ia.AverageAccuracy.ToString("N2"))), string.Join(", ", collection.Select(ia => ia.BrierScore.ToString("N2"))), holdoutSet.Labels[i].Average().ToString("N2") ); } HoldoutMetrics.RecomputeAggregateMetrics(); } }