/// <summary> /// Runs the online experiment. /// </summary> /// <param name="dataSet">Data set.</param> /// <param name="holdoutSet">Holdout set.</param> /// <param name="priors">Priors.</param> public void RunOnline(DataSet dataSet, DataSet holdoutSet, Marginals priors) { using (new CodeTimer("Running online experiment: " + Name)) { Console.WriteLine(); Metrics = new MetricsCollection(); HoldoutMetrics = new HoldoutMetricsCollection { Metrics = new Metrics[dataSet.NumberOfResidents][] }; PosteriorActivities = new Bernoulli[dataSet.NumberOfResidents][]; HoldoutPosteriorActivities = new Bernoulli[dataSet.NumberOfResidents][][]; IndividualPosteriors = new Marginals[dataSet.NumberOfResidents]; var accuracy = new double[dataSet.NumberOfResidents][]; for (int i = 0; i < dataSet.NumberOfResidents; i++) { var collection = new List <Metrics>(); HoldoutPosteriorActivities[i] = new Bernoulli[dataSet.NumberOfInstances[i]][]; accuracy[i] = new double[dataSet.NumberOfInstances[i]]; IndividualPosteriors[i] = new Marginals(priors); PosteriorActivities[i] = new Bernoulli[dataSet.NumberOfInstances[i]]; for (int j = 0; j < dataSet.NumberOfInstances[i]; j++) { var datum = dataSet.GetSubSet(i, j); PosteriorActivities[i][j] = TestModel.Test(datum, IndividualPosteriors[i])[0][0]; HoldoutPosteriorActivities[i][j] = TestModel.Test(holdoutSet.GetSubSet(i), IndividualPosteriors[i])[0]; // Test on holdout set var holdoutMetrics = new Metrics { Name = Name, Estimates = HoldoutPosteriorActivities[i][j], TrueLabels = holdoutSet.Labels[i] }; accuracy[i][j] = holdoutMetrics.AverageAccuracy; // PrintPrediction(i, temp[0][0], testLabels[0][i], testScores[0][i]); // Now retrain using this label IndividualPosteriors[i] = TrainModel.Train(datum, IndividualPosteriors[i], 10); collection.Add(holdoutMetrics); } // PrintPredictions(posteriorActivities.Select(ia => ia[0]).ToArray(), testLabels.Select(ia => ia[0]).ToArray()); Metrics.Add(new Metrics { Name = Name, Estimates = PosteriorActivities[i], TrueLabels = dataSet.Labels[i] }, true); HoldoutMetrics.Metrics[i] = collection.ToArray(); Console.WriteLine("{0,20}, Resident {1}, Hold out accuracy {2:N2}", Name, i, collection.Average(ia => ia.AverageAccuracy)); } HoldoutMetrics.RecomputeAggregateMetrics(); Metrics.RecomputeAggregateMetrics(); // Console.WriteLine("Accuracies " + string.Join(", ", accuracy.ColumnAverage().Select(x => x.ToString("N2")))); // Console.WriteLine("Std. dev. " + string.Join(", ", accuracy.ColumnStandardDeviation().Select(x => x.ToString("N2")))); // Console.WriteLine("Accuracies " + string.Join(", ", HoldoutMetrics.AverageAccuracy.Select(x => x.ToString("N2")))); } }
/// <summary> /// Runs the online experiment. /// </summary> /// <param name="dataSet">Data set.</param> /// <param name="holdoutSet">Holdout set.</param> /// <param name="priors">Priors.</param> public void RunOnline(DataSet dataSet, DataSet holdoutSet, Marginals priors) { using (new CodeTimer("Running online experiment: " + Name)) { Console.WriteLine(); Metrics = new MetricsCollection(); HoldoutMetrics = new HoldoutMetricsCollection { Metrics = new Metrics[dataSet.NumberOfResidents][] }; PosteriorActivities = new Bernoulli[dataSet.NumberOfResidents][]; HoldoutPosteriorActivities = new Bernoulli[dataSet.NumberOfResidents][][]; IndividualPosteriors = new Marginals[dataSet.NumberOfResidents]; var accuracy = new double[dataSet.NumberOfResidents][]; for (int i = 0; i < dataSet.NumberOfResidents; i++) { var collection = new List<Metrics>(); HoldoutPosteriorActivities[i] = new Bernoulli[dataSet.NumberOfInstances[i]][]; accuracy[i] = new double[dataSet.NumberOfInstances[i]]; IndividualPosteriors[i] = new Marginals(priors); PosteriorActivities[i] = new Bernoulli[dataSet.NumberOfInstances[i]]; for (int j = 0; j < dataSet.NumberOfInstances[i]; j++) { var datum = dataSet.GetSubSet(i, j); PosteriorActivities[i][j] = TestModel.Test(datum, IndividualPosteriors[i])[0][0]; HoldoutPosteriorActivities[i][j] = TestModel.Test(holdoutSet.GetSubSet(i), IndividualPosteriors[i])[0]; // Test on holdout set var holdoutMetrics = new Metrics { Name = Name, Estimates = HoldoutPosteriorActivities[i][j], TrueLabels = holdoutSet.Labels[i] }; accuracy[i][j] = holdoutMetrics.AverageAccuracy; // PrintPrediction(i, temp[0][0], testLabels[0][i], testScores[0][i]); // Now retrain using this label IndividualPosteriors[i] = TrainModel.Train(datum, IndividualPosteriors[i], 10); collection.Add(holdoutMetrics); } // PrintPredictions(posteriorActivities.Select(ia => ia[0]).ToArray(), testLabels.Select(ia => ia[0]).ToArray()); Metrics.Add(new Metrics { Name = Name, Estimates = PosteriorActivities[i], TrueLabels = dataSet.Labels[i] }, true); HoldoutMetrics.Metrics[i] = collection.ToArray(); Console.WriteLine("{0,20}, Resident {1}, Hold out accuracy {2:N2}", Name, i, collection.Average(ia => ia.AverageAccuracy)); } HoldoutMetrics.RecomputeAggregateMetrics(); Metrics.RecomputeAggregateMetrics(); // Console.WriteLine("Accuracies " + string.Join(", ", accuracy.ColumnAverage().Select(x => x.ToString("N2")))); // Console.WriteLine("Std. dev. " + string.Join(", ", accuracy.ColumnStandardDeviation().Select(x => x.ToString("N2")))); // Console.WriteLine("Accuracies " + string.Join(", ", HoldoutMetrics.AverageAccuracy.Select(x => x.ToString("N2")))); } }