/// <summary> /// Runs the active learning experiment presented in Venanzi et.al (WWW14) on a single data set. /// </summary> /// <param name="dataSet">The data.</param> /// <param name="runType">The model run type.</param> /// <param name="taskSelectionMethod">The method for selecting tasks (Random / Entropy).</param> /// <param name="model">The model instance.</param> /// <param name="communityCount">The number of communities (only for CBCC).</param> static void RunWWWActiveLearning(string dataSet, RunType runType, TaskSelectionMethod taskSelectionMethod, BCC model, int communityCount = 4) { // Reset the random seed so results can be duplicated for the paper Rand.Restart(12347); var workerSelectionMethod = WorkerSelectionMethod.RandomWorker; var data = Datum.LoadData(@"Data\" + dataSet + ".csv"); string modelName = GetModelName(dataSet, runType, taskSelectionMethod, workerSelectionMethod, communityCount); ActiveLearning.RunActiveLearning(data, modelName, runType, model, taskSelectionMethod, workerSelectionMethod, ResultsDir, communityCount); }
/// <summary> /// Runs the standard active learning procedure on a model instance and an input data set. /// </summary> /// <param name="data">The data.</param> /// <param name="modelName">The model name.</param> /// <param name="runType">The model run type.</param> /// <param name="model">The model instance.</param> /// <param name="taskSelectionMethod">The method for selecting tasks (Random / Entropy).</param> /// <param name="workerSelectionMethod">The method for selecting workers (only Random is implemented).</param> /// <param name="resultsDir">The directory to save the log files.</param> /// <param name="communityCount">The number of communities (only for CBCC).</param> /// <param name="initialNumLabelsPerTask">The initial number of exploratory labels that are randomly selected for each task.</param> public static void RunActiveLearning(IList <Datum> data, string modelName, RunType runType, BCC model, TaskSelectionMethod taskSelectionMethod, WorkerSelectionMethod workerSelectionMethod, string resultsDir, int communityCount = -1, int initialNumLabelsPerTask = 1) { //Count elapsed time Stopwatch stopWatchTotal = new Stopwatch(); stopWatchTotal.Start(); int totalLabels = data.Count(); // Dictionary keyed by task Id, with randomly order labelings var groupedRandomisedData = data.GroupBy(d => d.TaskId). Select(g => { var arr = g.ToArray(); int cnt = arr.Length; var perm = Rand.Perm(cnt); return(new { key = g.Key, arr = g.Select((t, i) => arr[perm[i]]).ToArray() }); }).ToDictionary(a => a.key, a => a.arr); // Dictionary keyed by task Id, with label counts Dictionary <string, int> totalCounts = groupedRandomisedData.ToDictionary(kvp => kvp.Key, kvp => kvp.Value.Length); Dictionary <string, int> currentCounts = groupedRandomisedData.ToDictionary(kvp => kvp.Key, kvp => initialNumLabelsPerTask); // Keyed by task, value is a HashSet containing all the remaining workers with a label - workers are removed after adding a new datum Dictionary <string, HashSet <string> > remainingWorkersPerTask = groupedRandomisedData.ToDictionary(kvp => kvp.Key, kvp => new HashSet <string>(kvp.Value.Select(dat => dat.WorkerId))); int numTaskIds = totalCounts.Count(); int totalInstances = data.Count - initialNumLabelsPerTask * numTaskIds; string[] WorkerIds = data.Select(d => d.WorkerId).Distinct().ToArray(); // Log structures List <double> accuracy = new List <double>(); List <double> nlpd = new List <double>(); List <double> avgRecall = new List <double>(); List <ActiveLearningResult> taskValueList = new List <ActiveLearningResult>(); int index = 0; Console.WriteLine("Active Learning: {0}", modelName); Console.WriteLine("\t\tAcc\tAvgRec"); // Get initial data Results results = new Results(); List <Datum> subData = null; subData = GetSubdata(groupedRandomisedData, currentCounts, remainingWorkersPerTask); var s = remainingWorkersPerTask.Select(w => w.Value.Count).Sum(); List <Datum> nextData = null; int numIncremData = 3; ActiveLearning activeLearning = null; for (int iter = 0; iter < 500; iter++) { bool calculateAccuracy = true; ////bool doSnapShot = iter % 100 == 0; // Frequency of snapshots bool doSnapShot = true; if (subData != null || nextData != null) { switch (runType) { case RunType.VoteDistribution: results.RunMajorityVote(subData, calculateAccuracy, true); break; case RunType.MajorityVote: results.RunMajorityVote(subData, calculateAccuracy, false); break; case RunType.DawidSkene: results.RunDawidSkene(subData, calculateAccuracy); break; default: // Run BCC models results.RunBCC(modelName, subData, data, model, Results.RunMode.ClearResults, calculateAccuracy, communityCount, false); break; } } if (activeLearning == null) { activeLearning = new ActiveLearning(data, model, results, communityCount); } else { activeLearning.UpdateActiveLearningResults(results); } // Select next task Dictionary <string, ActiveLearningResult> TaskValue = null; List <Tuple <string, string, ActiveLearningResult> > LabelValue = null; switch (taskSelectionMethod) { case TaskSelectionMethod.EntropyTask: TaskValue = activeLearning.EntropyTrueLabelPosterior(); break; case TaskSelectionMethod.RandomTask: TaskValue = data.GroupBy(d => d.TaskId).ToDictionary(a => a.Key, a => new ActiveLearningResult { TaskValue = Rand.Double() }); break; default: // Entropy task selection TaskValue = activeLearning.EntropyTrueLabelPosterior(); break; } nextData = GetNextData(groupedRandomisedData, TaskValue, currentCounts, totalCounts, numIncremData); if (nextData == null || nextData.Count == 0) { break; } index += nextData.Count; subData.AddRange(nextData); // Logs if (calculateAccuracy) { accuracy.Add(results.Accuracy); nlpd.Add(results.NegativeLogProb); avgRecall.Add(results.AvgRecall); if (TaskValue == null) { var sortedLabelValue = LabelValue.OrderByDescending(kvp => kvp.Item3.TaskValue).ToArray(); taskValueList.Add(sortedLabelValue.First().Item3); } else { taskValueList.Add(TaskValue[nextData.First().TaskId]); } if (doSnapShot) { Console.WriteLine("{0} of {1}:\t{2:0.000}\t{3:0.0000}", index, totalInstances, accuracy.Last(), avgRecall.Last()); DoSnapshot(accuracy, nlpd, avgRecall, taskValueList, results, modelName, "interim", resultsDir); } } } stopWatchTotal.Stop(); DoSnapshot(accuracy, nlpd, avgRecall, taskValueList, results, modelName, "final", resultsDir); Console.WriteLine("Elapsed time: {0}\n", stopWatchTotal.Elapsed); }