/// <summary> /// Runs the active experiment. /// </summary> /// <param name="dataSet">Data set.</param> /// <param name="holdoutSet">Holdout set.</param> /// <param name="numberOfSelections">Number of selections.</param> /// <param name="priors">Priors.</param> public void RunActive(DataSet dataSet, DataSet holdoutSet, int numberOfSelections, Marginals priors) { if (ActiveLearners == null) { throw new InvalidOperationException("Active Learner not provided"); } using (new CodeTimer("Running active experiment: " + Name)) { Console.WriteLine(); HoldoutMetrics = new HoldoutMetricsCollection { Metrics = new Metrics[dataSet.NumberOfResidents][] }; // Metrics = new MetricsCollection(numberOfSelections); PosteriorActivities = new Bernoulli[dataSet.NumberOfResidents][]; HoldoutPosteriorActivities = new Bernoulli[dataSet.NumberOfResidents][][]; IndividualPosteriors = new Marginals[dataSet.NumberOfResidents]; var accuracy = new double[dataSet.NumberOfResidents][]; for (int i = 0; i < dataSet.NumberOfResidents; i++) { HoldoutMetrics.Metrics[i] = new Metrics[numberOfSelections]; var collection = new List<Metrics>(); IndividualPosteriors[i] = new Marginals(priors); // Test on holdout set HoldoutPosteriorActivities[i] = new Bernoulli[numberOfSelections][]; accuracy[i] = new double[numberOfSelections]; var dataSetForResident = dataSet.GetSubSet(i); var holdoutSetForResident = holdoutSet.GetSubSet(i); // ActiveLearners[i].Transfer(i, 1); // var individualPosteriors = new Marginals(priors); for (int j = 0; j < numberOfSelections; j++) { PosteriorActivities[i] = TestModel.Test(dataSetForResident, IndividualPosteriors[i])[0]; HoldoutPosteriorActivities[i][j] = TestModel.Test(holdoutSetForResident, IndividualPosteriors[i])[0]; if (ActiveLearners[i].Unlabelled.Count == 0) { Console.WriteLine("Empty unlabelled set"); break; } // int index = ActiveLearner.GetValueOfInformation(i).ArgMax(); int index; double val; ActiveLearners[i].GetArgMaxVOI(PosteriorActivities[i], IndividualPosteriors[i], out index, out val); // Console.WriteLine("Index {0,4}, VOI {1:N4}", index, value); // Now retrain using this label ActiveLearners[i].UpdateModel(index); //IndividualPosteriors [i] = TrainModel.Train( dataSet.GetSubSet(i, ActiveLearners [i].Labelled.ToList()), priors, 10); IndividualPosteriors[i] = TrainModel.Train(dataSet.GetSubSet(i, index), IndividualPosteriors[i], 50); var metrics = new Metrics { Name = Name, Estimates = HoldoutPosteriorActivities[i][j], TrueLabels = holdoutSet.Labels[i] }; accuracy[i][j] = metrics.AverageAccuracy; collection.Add(metrics); } // PrintPredictions(posteriorActivities.Select(ia => ia[0]).ToArray(), testLabels.Select(ia => ia[0]).ToArray()); HoldoutMetrics.Metrics[i] = collection.ToArray(); Console.WriteLine("{0,20}, Resident {1}, \n\t\tClass ratio {5}, \n\t\tHold out accuracy {2:N2}, \n\t\tAccuracies {3} \n\t\tBriers {4}\n", Name, i, collection.Average(ia => ia.AverageAccuracy).ToString("N2"), string.Join(", ", collection.Select(ia => ia.AverageAccuracy.ToString("N2"))), string.Join(", ", collection.Select(ia => ia.BrierScore.ToString("N2"))), holdoutSet.Labels[i].Average().ToString("N2") ); } HoldoutMetrics.RecomputeAggregateMetrics(); } }
/// <summary> /// Runs the online experiment. /// </summary> /// <param name="dataSet">Data set.</param> /// <param name="holdoutSet">Holdout set.</param> /// <param name="priors">Priors.</param> public void RunOnline(DataSet dataSet, DataSet holdoutSet, Marginals priors) { using (new CodeTimer("Running online experiment: " + Name)) { Console.WriteLine(); Metrics = new MetricsCollection(); HoldoutMetrics = new HoldoutMetricsCollection { Metrics = new Metrics[dataSet.NumberOfResidents][] }; PosteriorActivities = new Bernoulli[dataSet.NumberOfResidents][]; HoldoutPosteriorActivities = new Bernoulli[dataSet.NumberOfResidents][][]; IndividualPosteriors = new Marginals[dataSet.NumberOfResidents]; var accuracy = new double[dataSet.NumberOfResidents][]; for (int i = 0; i < dataSet.NumberOfResidents; i++) { var collection = new List<Metrics>(); HoldoutPosteriorActivities[i] = new Bernoulli[dataSet.NumberOfInstances[i]][]; accuracy[i] = new double[dataSet.NumberOfInstances[i]]; IndividualPosteriors[i] = new Marginals(priors); PosteriorActivities[i] = new Bernoulli[dataSet.NumberOfInstances[i]]; for (int j = 0; j < dataSet.NumberOfInstances[i]; j++) { var datum = dataSet.GetSubSet(i, j); PosteriorActivities[i][j] = TestModel.Test(datum, IndividualPosteriors[i])[0][0]; HoldoutPosteriorActivities[i][j] = TestModel.Test(holdoutSet.GetSubSet(i), IndividualPosteriors[i])[0]; // Test on holdout set var holdoutMetrics = new Metrics { Name = Name, Estimates = HoldoutPosteriorActivities[i][j], TrueLabels = holdoutSet.Labels[i] }; accuracy[i][j] = holdoutMetrics.AverageAccuracy; // PrintPrediction(i, temp[0][0], testLabels[0][i], testScores[0][i]); // Now retrain using this label IndividualPosteriors[i] = TrainModel.Train(datum, IndividualPosteriors[i], 10); collection.Add(holdoutMetrics); } // PrintPredictions(posteriorActivities.Select(ia => ia[0]).ToArray(), testLabels.Select(ia => ia[0]).ToArray()); Metrics.Add(new Metrics { Name = Name, Estimates = PosteriorActivities[i], TrueLabels = dataSet.Labels[i] }, true); HoldoutMetrics.Metrics[i] = collection.ToArray(); Console.WriteLine("{0,20}, Resident {1}, Hold out accuracy {2:N2}", Name, i, collection.Average(ia => ia.AverageAccuracy)); } HoldoutMetrics.RecomputeAggregateMetrics(); Metrics.RecomputeAggregateMetrics(); // Console.WriteLine("Accuracies " + string.Join(", ", accuracy.ColumnAverage().Select(x => x.ToString("N2")))); // Console.WriteLine("Std. dev. " + string.Join(", ", accuracy.ColumnStandardDeviation().Select(x => x.ToString("N2")))); // Console.WriteLine("Accuracies " + string.Join(", ", HoldoutMetrics.AverageAccuracy.Select(x => x.ToString("N2")))); } }
public void VOITest(int numActivelySelected, Marginals priors) { var onlineEstimates = new List<Bernoulli>(); var onlineTargets = new List<bool>(); Metrics metrics = null; for (int jj = 0; jj < numActivelySelected; ++jj) { CalculateProbabilities(priors); //Console.WriteLine( "\nJL: {0}", JL() ); //Console.WriteLine( "JU: {0}", JU() ); int argMax; double maxVal; GetArgMaxVOI(hypothesisActivityPosteriors, priors, out argMax, out maxVal); Unlabelled.Remove(argMax); Labelled.Add(argMax); UpdateModel(argMax); onlineEstimates.Add(GetProbabilityOf(argMax, priors)); onlineTargets.Add(DataSet.Labels[0][argMax]); metrics = new Metrics { Name = "active", Estimates = onlineEstimates.Select(ia => new Bernoulli(ia)).ToArray(), TrueLabels = onlineTargets.ToArray() }; // metrics.PrintSummary(); } if (Unlabelled.Any()) { CalculateProbabilities(priors); foreach (var index in Unlabelled) { onlineEstimates.Add(hypothesisActivityPosteriors[index]); onlineTargets.Add(DataSet.Labels[0][index]); } metrics = new Metrics { Name = "active", Estimates = onlineEstimates.Select(ia => new Bernoulli(ia)).ToArray(), TrueLabels = onlineTargets.ToArray() }; } if (metrics != null) { metrics.PrintSummary(); } }
/// <summary> /// Add the specified metrics. /// </summary> /// <param name="metrics">Metrics.</param> /// <param name="printSummary">If set to <c>true</c> print summary.</param> public void Add(Metrics metrics, bool printSummary = false) { this.metrics.Add(metrics); if (printSummary) { metrics.PrintSummary(); } }
public void VOITest( int numActivelySelected ) { var onlineEstimates = new List<double>(); var onlineTargets = new List<bool>(); for ( int jj = 0; jj < numActivelySelected; ++jj ) { CalculateProbabilities(); //Console.WriteLine( "\nJL: {0}", JL() ); //Console.WriteLine( "JU: {0}", JU() ); double jall = JAll(); var inds = U.Select( uu => uu ).ToArray(); int maxind = -1; double maxval = -1.0; var signs = new List<double>(); foreach ( var ind in inds ) { var voi = VOI( jall, ind ); signs.Add( Math.Sign( voi ) / 2.0 + 0.5 ); //Console.Write( "." ); //Console.WriteLine( "y_true: {0}", labels[0][ind] ); //Console.WriteLine( "y_hat: {0}", probs[ind] ); //Console.WriteLine( "VOJ_{0}: {1}", ind, voi ); //Console.WriteLine(); if ( voi > maxval || maxind < 0) { maxval = voi; maxind = ind; } } //Console.WriteLine(); //Console.WriteLine( "\n+ivity: {0}", signs.Average() ); U.Remove( maxind ); L.Add( maxind ); UpdateModel( maxind ); onlineEstimates.Add( probof( maxind ) ); onlineTargets.Add( labels[0][maxind] ); metrics = new Metrics { Name = "active", Estimates = onlineEstimates.Select( oo => new Bernoulli( oo ) ).ToArray(), TrueLabels = onlineTargets.ToArray() }; //metrics.PrintSummary(); } if ( U.Count() > 0 ) { CalculateProbabilities(); foreach ( var ind in U ) { onlineEstimates.Add( probs[ind] ); onlineTargets.Add( labels[0][ind] ); } metrics = new Metrics { Name = "active", Estimates = onlineEstimates.Select( oo => new Bernoulli( oo ) ).ToArray(), TrueLabels = onlineTargets.ToArray() }; } metrics.PrintSummary(); }
public void asdfasdf() { Bernoulli [] [] activeActivities = { new Bernoulli [N] }; for ( int i = 0; i < N; i++ ) { Bernoulli [] [] temp; testModel.Test( new [] { new [] { features [0] [i] } }, activePosteriorWeightMeans, activePosteriorWeightPrecisions, out temp ); activeActivities [0] [i] = temp [0] [0]; // Now retrain using this label trainModel.Train( new [] { new [] { features [0] [i] } }, new [] { new [] { labels [0] [i] } }, activePosteriorWeightMeans, activePosteriorWeightPrecisions, out activePosteriorWeights, out activePosteriorWeightMeans, out activePosteriorWeightPrecisions ); } var activeMetrics = new Metrics { Estimates = activeActivities [0], TrueLabels = labels [0] }; activeMetrics.PrintSummary(); }
public void VOITest(int numActivelySelected) { var onlineEstimates = new List <double>(); var onlineTargets = new List <bool>(); for (int jj = 0; jj < numActivelySelected; ++jj) { CalculateProbabilities(); //Console.WriteLine( "\nJL: {0}", JL() ); //Console.WriteLine( "JU: {0}", JU() ); double jall = JAll(); var inds = U.Select(uu => uu).ToArray(); int maxind = -1; double maxval = -1.0; var signs = new List <double>(); foreach (var ind in inds) { var voi = VOI(jall, ind); signs.Add(Math.Sign(voi) / 2.0 + 0.5); //Console.Write( "." ); //Console.WriteLine( "y_true: {0}", labels[0][ind] ); //Console.WriteLine( "y_hat: {0}", probs[ind] ); //Console.WriteLine( "VOJ_{0}: {1}", ind, voi ); //Console.WriteLine(); if (voi > maxval || maxind < 0) { maxval = voi; maxind = ind; } } //Console.WriteLine(); //Console.WriteLine( "\n+ivity: {0}", signs.Average() ); U.Remove(maxind); L.Add(maxind); UpdateModel(maxind); onlineEstimates.Add(probof(maxind)); onlineTargets.Add(labels[0][maxind]); metrics = new Metrics { Name = "active", Estimates = onlineEstimates.Select(oo => new Bernoulli(oo)).ToArray(), TrueLabels = onlineTargets.ToArray() }; //metrics.PrintSummary(); } if (U.Count() > 0) { CalculateProbabilities(); foreach (var ind in U) { onlineEstimates.Add(probs[ind]); onlineTargets.Add(labels[0][ind]); } metrics = new Metrics { Name = "active", Estimates = onlineEstimates.Select(oo => new Bernoulli(oo)).ToArray(), TrueLabels = onlineTargets.ToArray() }; } metrics.PrintSummary(); }