Example #1
0
        /// <summary>
        /// Runs the active experiment.
        /// </summary>
        /// <param name="dataSet">Data set.</param>
        /// <param name="holdoutSet">Holdout set.</param>
        /// <param name="numberOfSelections">Number of selections.</param>
        /// <param name="priors">Priors.</param>
        public void RunActive(DataSet dataSet, DataSet holdoutSet, int numberOfSelections, Marginals priors)
        {
            if (ActiveLearners == null)
            {
                throw new InvalidOperationException("Active Learner not provided");
            }

            using (new CodeTimer("Running active experiment: " + Name))
            {
                Console.WriteLine();

                HoldoutMetrics = new HoldoutMetricsCollection { Metrics = new Metrics[dataSet.NumberOfResidents][] };

                // Metrics = new MetricsCollection(numberOfSelections);
                PosteriorActivities = new Bernoulli[dataSet.NumberOfResidents][];
                HoldoutPosteriorActivities = new Bernoulli[dataSet.NumberOfResidents][][];
                IndividualPosteriors = new Marginals[dataSet.NumberOfResidents];

                var accuracy = new double[dataSet.NumberOfResidents][];

                for (int i = 0; i < dataSet.NumberOfResidents; i++)
                {
                    HoldoutMetrics.Metrics[i] = new Metrics[numberOfSelections];

                    var collection = new List<Metrics>();
                    IndividualPosteriors[i] = new Marginals(priors);

                    // Test on holdout set
                    HoldoutPosteriorActivities[i] = new Bernoulli[numberOfSelections][];
                    accuracy[i] = new double[numberOfSelections];

                    var dataSetForResident = dataSet.GetSubSet(i);
                    var holdoutSetForResident = holdoutSet.GetSubSet(i);
                    // ActiveLearners[i].Transfer(i, 1);

                    // var individualPosteriors = new Marginals(priors);
                    for (int j = 0; j < numberOfSelections; j++)
                    {
                        PosteriorActivities[i] = TestModel.Test(dataSetForResident, IndividualPosteriors[i])[0];
                        HoldoutPosteriorActivities[i][j] = TestModel.Test(holdoutSetForResident, IndividualPosteriors[i])[0];

                        if (ActiveLearners[i].Unlabelled.Count == 0)
                        {
                            Console.WriteLine("Empty unlabelled set");
                            break;
                        }

                        // int index = ActiveLearner.GetValueOfInformation(i).ArgMax();
                        int index;
                        double val;
                        ActiveLearners[i].GetArgMaxVOI(PosteriorActivities[i], IndividualPosteriors[i], out index, out val);

                        // Console.WriteLine("Index {0,4}, VOI {1:N4}", index, value);

                        // Now retrain using this label
                        ActiveLearners[i].UpdateModel(index);
                        //IndividualPosteriors [i] = TrainModel.Train( dataSet.GetSubSet(i, ActiveLearners [i].Labelled.ToList()), priors, 10);
                        IndividualPosteriors[i] = TrainModel.Train(dataSet.GetSubSet(i, index), IndividualPosteriors[i], 50);

                        var metrics = new Metrics { Name = Name, Estimates = HoldoutPosteriorActivities[i][j], TrueLabels = holdoutSet.Labels[i] };
                        accuracy[i][j] = metrics.AverageAccuracy;

                        collection.Add(metrics);
                    }

                    // PrintPredictions(posteriorActivities.Select(ia => ia[0]).ToArray(), testLabels.Select(ia => ia[0]).ToArray());
                    HoldoutMetrics.Metrics[i] = collection.ToArray();

                    Console.WriteLine("{0,20}, Resident {1}, \n\t\tClass ratio       {5}, \n\t\tHold out accuracy {2:N2}, \n\t\tAccuracies {3} \n\t\tBriers     {4}\n",
                        Name, i, collection.Average(ia => ia.AverageAccuracy).ToString("N2"),
                        string.Join(", ", collection.Select(ia => ia.AverageAccuracy.ToString("N2"))),
                        string.Join(", ", collection.Select(ia => ia.BrierScore.ToString("N2"))),
                        holdoutSet.Labels[i].Average().ToString("N2")
                        );
                }

                HoldoutMetrics.RecomputeAggregateMetrics();

            }
        }
Example #2
0
        /// <summary>
        /// Runs the online experiment.
        /// </summary>
        /// <param name="dataSet">Data set.</param>
        /// <param name="holdoutSet">Holdout set.</param>
        /// <param name="priors">Priors.</param>
        public void RunOnline(DataSet dataSet, DataSet holdoutSet, Marginals priors)
        {
            using (new CodeTimer("Running online experiment: " + Name))
            {
                Console.WriteLine();

                Metrics = new MetricsCollection();
                HoldoutMetrics = new HoldoutMetricsCollection { Metrics = new Metrics[dataSet.NumberOfResidents][] };

                PosteriorActivities = new Bernoulli[dataSet.NumberOfResidents][];
                HoldoutPosteriorActivities = new Bernoulli[dataSet.NumberOfResidents][][];
                IndividualPosteriors = new Marginals[dataSet.NumberOfResidents];

                var accuracy = new double[dataSet.NumberOfResidents][];

                for (int i = 0; i < dataSet.NumberOfResidents; i++)
                {
                    var collection = new List<Metrics>();
                    HoldoutPosteriorActivities[i] = new Bernoulli[dataSet.NumberOfInstances[i]][];
                    accuracy[i] = new double[dataSet.NumberOfInstances[i]];

                    IndividualPosteriors[i] = new Marginals(priors);
                    PosteriorActivities[i] = new Bernoulli[dataSet.NumberOfInstances[i]];

                    for (int j = 0; j < dataSet.NumberOfInstances[i]; j++)
                    {
                        var datum = dataSet.GetSubSet(i, j);
                        PosteriorActivities[i][j] = TestModel.Test(datum, IndividualPosteriors[i])[0][0];
                        HoldoutPosteriorActivities[i][j] = TestModel.Test(holdoutSet.GetSubSet(i), IndividualPosteriors[i])[0];

                        // Test on holdout set
                        var holdoutMetrics = new Metrics { Name = Name, Estimates = HoldoutPosteriorActivities[i][j], TrueLabels = holdoutSet.Labels[i] };
                        accuracy[i][j] = holdoutMetrics.AverageAccuracy;

                        // PrintPrediction(i, temp[0][0], testLabels[0][i], testScores[0][i]);

                        // Now retrain using this label
                        IndividualPosteriors[i] = TrainModel.Train(datum, IndividualPosteriors[i], 10);

                        collection.Add(holdoutMetrics);
                    }

                    // PrintPredictions(posteriorActivities.Select(ia => ia[0]).ToArray(), testLabels.Select(ia => ia[0]).ToArray());
                    Metrics.Add(new Metrics { Name = Name, Estimates = PosteriorActivities[i], TrueLabels = dataSet.Labels[i] }, true);

                    HoldoutMetrics.Metrics[i] = collection.ToArray();

                    Console.WriteLine("{0,20}, Resident {1}, Hold out accuracy {2:N2}", Name, i, collection.Average(ia => ia.AverageAccuracy));
                }

                HoldoutMetrics.RecomputeAggregateMetrics();
                Metrics.RecomputeAggregateMetrics();

                // Console.WriteLine("Accuracies " + string.Join(", ", accuracy.ColumnAverage().Select(x => x.ToString("N2"))));
                // Console.WriteLine("Std. dev.  " + string.Join(", ", accuracy.ColumnStandardDeviation().Select(x => x.ToString("N2"))));
                // Console.WriteLine("Accuracies " + string.Join(", ", HoldoutMetrics.AverageAccuracy.Select(x => x.ToString("N2"))));
            }
        }
        public void VOITest(int numActivelySelected, Marginals priors)
        {
            var onlineEstimates = new List<Bernoulli>();
            var onlineTargets = new List<bool>();

            Metrics metrics = null;

            for (int jj = 0; jj < numActivelySelected; ++jj)
            {
                CalculateProbabilities(priors);

                //Console.WriteLine( "\nJL: {0}", JL() );
                //Console.WriteLine( "JU: {0}", JU() );

                int argMax;
                double maxVal;
                GetArgMaxVOI(hypothesisActivityPosteriors, priors, out argMax, out maxVal);

                Unlabelled.Remove(argMax);
                Labelled.Add(argMax);

                UpdateModel(argMax);

                onlineEstimates.Add(GetProbabilityOf(argMax, priors));
                onlineTargets.Add(DataSet.Labels[0][argMax]);

                metrics = new Metrics
                {
                    Name = "active",
                    Estimates = onlineEstimates.Select(ia => new Bernoulli(ia)).ToArray(),
                    TrueLabels = onlineTargets.ToArray()
                };

                // metrics.PrintSummary();
            }

            if (Unlabelled.Any())
            {
                CalculateProbabilities(priors);
                foreach (var index in Unlabelled)
                {
                    onlineEstimates.Add(hypothesisActivityPosteriors[index]);
                    onlineTargets.Add(DataSet.Labels[0][index]);
                }

                metrics = new Metrics
                {
                    Name = "active",
                    Estimates = onlineEstimates.Select(ia => new Bernoulli(ia)).ToArray(),
                    TrueLabels = onlineTargets.ToArray()
                };
            }

            if (metrics != null)
            {
                metrics.PrintSummary();
            }
        }
 /// <summary>
 /// Add the specified metrics.
 /// </summary>
 /// <param name="metrics">Metrics.</param>
 /// <param name="printSummary">If set to <c>true</c> print summary.</param>
 public void Add(Metrics metrics, bool printSummary = false)
 {
     this.metrics.Add(metrics);
     if (printSummary)
     {
         metrics.PrintSummary();
     }
 }
Example #5
0
        public void VOITest( int numActivelySelected )
        {
            var onlineEstimates = new List<double>();
              var onlineTargets = new List<bool>();

              for ( int jj = 0; jj < numActivelySelected; ++jj ) {
            CalculateProbabilities();

            //Console.WriteLine( "\nJL: {0}", JL() );
            //Console.WriteLine( "JU: {0}", JU() );

            double jall = JAll();

            var inds = U.Select( uu => uu ).ToArray();
            int maxind = -1;
            double maxval = -1.0;

            var signs = new List<double>();

            foreach ( var ind in inds ) {
              var voi = VOI( jall, ind );
              signs.Add( Math.Sign( voi ) / 2.0 + 0.5 );

              //Console.Write( "." );
              //Console.WriteLine( "y_true: {0}", labels[0][ind] );
              //Console.WriteLine( "y_hat: {0}", probs[ind] );
              //Console.WriteLine( "VOJ_{0}: {1}", ind, voi );
              //Console.WriteLine();

              if ( voi > maxval || maxind < 0) {
            maxval = voi;
            maxind = ind;
              }
            }
            //Console.WriteLine();

            //Console.WriteLine( "\n+ivity: {0}", signs.Average() );

            U.Remove( maxind );
            L.Add( maxind );

            UpdateModel( maxind );

            onlineEstimates.Add( probof( maxind ) );
            onlineTargets.Add( labels[0][maxind] );

            metrics = new Metrics {
              Name = "active",
              Estimates = onlineEstimates.Select( oo => new Bernoulli( oo ) ).ToArray(),
              TrueLabels = onlineTargets.ToArray()
            };

            //metrics.PrintSummary();

              }

              if ( U.Count() > 0 ) {
            CalculateProbabilities();
            foreach ( var ind in U ) {
              onlineEstimates.Add( probs[ind] );
              onlineTargets.Add( labels[0][ind] );
            }

            metrics = new Metrics {
              Name = "active",
              Estimates = onlineEstimates.Select( oo => new Bernoulli( oo ) ).ToArray(),
              TrueLabels = onlineTargets.ToArray()
            };
              }

              metrics.PrintSummary();
        }
Example #6
0
        public void asdfasdf()
        {
            Bernoulli [] [] activeActivities = { new Bernoulli [N] };

              for ( int i = 0; i < N; i++ ) {
            Bernoulli [] [] temp;
            testModel.Test(
              new [] { new [] { features [0] [i] } },
              activePosteriorWeightMeans,
              activePosteriorWeightPrecisions,
              out temp );
            activeActivities [0] [i] = temp [0] [0];

            // Now retrain using this label
            trainModel.Train( new [] { new [] { features [0] [i] } }, new [] { new [] { labels [0] [i] } },
              activePosteriorWeightMeans,
              activePosteriorWeightPrecisions,
              out activePosteriorWeights,
              out activePosteriorWeightMeans,
              out activePosteriorWeightPrecisions );
              }

              var activeMetrics = new Metrics {
            Estimates = activeActivities [0],
            TrueLabels = labels [0]
              };

              activeMetrics.PrintSummary();
        }
Example #7
0
        public void VOITest(int numActivelySelected)
        {
            var onlineEstimates = new List <double>();
            var onlineTargets   = new List <bool>();

            for (int jj = 0; jj < numActivelySelected; ++jj)
            {
                CalculateProbabilities();

                //Console.WriteLine( "\nJL: {0}", JL() );
                //Console.WriteLine( "JU: {0}", JU() );

                double jall = JAll();

                var    inds   = U.Select(uu => uu).ToArray();
                int    maxind = -1;
                double maxval = -1.0;

                var signs = new List <double>();

                foreach (var ind in inds)
                {
                    var voi = VOI(jall, ind);
                    signs.Add(Math.Sign(voi) / 2.0 + 0.5);

                    //Console.Write( "." );
                    //Console.WriteLine( "y_true: {0}", labels[0][ind] );
                    //Console.WriteLine( "y_hat: {0}", probs[ind] );
                    //Console.WriteLine( "VOJ_{0}: {1}", ind, voi );
                    //Console.WriteLine();

                    if (voi > maxval || maxind < 0)
                    {
                        maxval = voi;
                        maxind = ind;
                    }
                }
                //Console.WriteLine();

                //Console.WriteLine( "\n+ivity: {0}", signs.Average() );

                U.Remove(maxind);
                L.Add(maxind);

                UpdateModel(maxind);


                onlineEstimates.Add(probof(maxind));
                onlineTargets.Add(labels[0][maxind]);

                metrics = new Metrics {
                    Name       = "active",
                    Estimates  = onlineEstimates.Select(oo => new Bernoulli(oo)).ToArray(),
                    TrueLabels = onlineTargets.ToArray()
                };

                //metrics.PrintSummary();
            }

            if (U.Count() > 0)
            {
                CalculateProbabilities();
                foreach (var ind in U)
                {
                    onlineEstimates.Add(probs[ind]);
                    onlineTargets.Add(labels[0][ind]);
                }

                metrics = new Metrics {
                    Name       = "active",
                    Estimates  = onlineEstimates.Select(oo => new Bernoulli(oo)).ToArray(),
                    TrueLabels = onlineTargets.ToArray()
                };
            }

            metrics.PrintSummary();
        }