/// <summary>
/// Runs the active learning experiment presented in Venanzi et.al (WWW14) on a single data set.
/// </summary>
/// <param name="dataSet">The data.</param>
/// <param name="runType">The model run type.</param>
/// <param name="taskSelectionMethod">The method for selecting tasks (Random / Entropy).</param>
/// <param name="model">The model instance.</param>
/// <param name="communityCount">The number of communities (only for CBCC).</param>
static void RunWWWActiveLearning(string dataSet, RunType runType, TaskSelectionMethod taskSelectionMethod, BCC model, int communityCount = 4)
{
// Reset the random seed so results can be duplicated for the paper
Rand.Restart(12347);
var workerSelectionMethod = WorkerSelectionMethod.RandomWorker;
var data = Datum.LoadData(@"Data\" + dataSet + ".csv");
string modelName = GetModelName(dataSet, runType, taskSelectionMethod, workerSelectionMethod, communityCount);
ActiveLearning.RunActiveLearning(data, modelName, runType, model, taskSelectionMethod, workerSelectionMethod, ResultsDir, communityCount);
}
/// <summary>
/// Runs the standard active learning procedure on a model instance and an input data set.
/// </summary>
/// <param name="data">The data.</param>
/// <param name="modelName">The model name.</param>
/// <param name="runType">The model run type.</param>
/// <param name="model">The model instance.</param>
/// <param name="taskSelectionMethod">The method for selecting tasks (Random / Entropy).</param>
/// <param name="workerSelectionMethod">The method for selecting workers (only Random is implemented).</param>
/// <param name="resultsDir">The directory to save the log files.</param>
/// <param name="communityCount">The number of communities (only for CBCC).</param>
/// <param name="initialNumLabelsPerTask">The initial number of exploratory labels that are randomly selected for each task.</param>
public static void RunActiveLearning(IList <Datum> data, string modelName, RunType runType, BCC model, TaskSelectionMethod taskSelectionMethod, WorkerSelectionMethod workerSelectionMethod, string resultsDir, int communityCount = -1, int initialNumLabelsPerTask = 1)
{
//Count elapsed time
Stopwatch stopWatchTotal = new Stopwatch();
stopWatchTotal.Start();
int totalLabels = data.Count();
// Dictionary keyed by task Id, with randomly order labelings
var groupedRandomisedData =
data.GroupBy(d => d.TaskId).
Select(g =>
{
var arr = g.ToArray();
int cnt = arr.Length;
var perm = Rand.Perm(cnt);
return(new
{
key = g.Key,
arr = g.Select((t, i) => arr[perm[i]]).ToArray()
});
}).ToDictionary(a => a.key, a => a.arr);
// Dictionary keyed by task Id, with label counts
Dictionary <string, int> totalCounts = groupedRandomisedData.ToDictionary(kvp => kvp.Key, kvp => kvp.Value.Length);
Dictionary <string, int> currentCounts = groupedRandomisedData.ToDictionary(kvp => kvp.Key, kvp => initialNumLabelsPerTask);
// Keyed by task, value is a HashSet containing all the remaining workers with a label - workers are removed after adding a new datum
Dictionary <string, HashSet <string> > remainingWorkersPerTask = groupedRandomisedData.ToDictionary(kvp => kvp.Key, kvp => new HashSet <string>(kvp.Value.Select(dat => dat.WorkerId)));
int numTaskIds = totalCounts.Count();
int totalInstances = data.Count - initialNumLabelsPerTask * numTaskIds;
string[] WorkerIds = data.Select(d => d.WorkerId).Distinct().ToArray();
// Log structures
List <double> accuracy = new List <double>();
List <double> nlpd = new List <double>();
List <double> avgRecall = new List <double>();
List <ActiveLearningResult> taskValueList = new List <ActiveLearningResult>();
int index = 0;
Console.WriteLine("Active Learning: {0}", modelName);
Console.WriteLine("\t\tAcc\tAvgRec");
// Get initial data
Results results = new Results();
List <Datum> subData = null;
subData = GetSubdata(groupedRandomisedData, currentCounts, remainingWorkersPerTask);
var s = remainingWorkersPerTask.Select(w => w.Value.Count).Sum();
List <Datum> nextData = null;
int numIncremData = 3;
ActiveLearning activeLearning = null;
for (int iter = 0; iter < 500; iter++)
{
bool calculateAccuracy = true;
////bool doSnapShot = iter % 100 == 0; // Frequency of snapshots
bool doSnapShot = true;
if (subData != null || nextData != null)
{
switch (runType)
{
case RunType.VoteDistribution:
results.RunMajorityVote(subData, calculateAccuracy, true);
break;
case RunType.MajorityVote:
results.RunMajorityVote(subData, calculateAccuracy, false);
break;
case RunType.DawidSkene:
results.RunDawidSkene(subData, calculateAccuracy);
break;
default: // Run BCC models
results.RunBCC(modelName, subData, data, model, Results.RunMode.ClearResults, calculateAccuracy, communityCount, false);
break;
}
}
if (activeLearning == null)
{
activeLearning = new ActiveLearning(data, model, results, communityCount);
}
else
{
activeLearning.UpdateActiveLearningResults(results);
}
// Select next task
Dictionary <string, ActiveLearningResult> TaskValue = null;
List <Tuple <string, string, ActiveLearningResult> > LabelValue = null;
switch (taskSelectionMethod)
{
case TaskSelectionMethod.EntropyTask:
TaskValue = activeLearning.EntropyTrueLabelPosterior();
break;
case TaskSelectionMethod.RandomTask:
TaskValue = data.GroupBy(d => d.TaskId).ToDictionary(a => a.Key, a => new ActiveLearningResult
{
TaskValue = Rand.Double()
});
break;
default: // Entropy task selection
TaskValue = activeLearning.EntropyTrueLabelPosterior();
break;
}
nextData = GetNextData(groupedRandomisedData, TaskValue, currentCounts, totalCounts, numIncremData);
if (nextData == null || nextData.Count == 0)
{
break;
}
index += nextData.Count;
subData.AddRange(nextData);
// Logs
if (calculateAccuracy)
{
accuracy.Add(results.Accuracy);
nlpd.Add(results.NegativeLogProb);
avgRecall.Add(results.AvgRecall);
if (TaskValue == null)
{
var sortedLabelValue = LabelValue.OrderByDescending(kvp => kvp.Item3.TaskValue).ToArray();
taskValueList.Add(sortedLabelValue.First().Item3);
}
else
{
taskValueList.Add(TaskValue[nextData.First().TaskId]);
}
if (doSnapShot)
{
Console.WriteLine("{0} of {1}:\t{2:0.000}\t{3:0.0000}", index, totalInstances, accuracy.Last(), avgRecall.Last());
DoSnapshot(accuracy, nlpd, avgRecall, taskValueList, results, modelName, "interim", resultsDir);
}
}
}
stopWatchTotal.Stop();
DoSnapshot(accuracy, nlpd, avgRecall, taskValueList, results, modelName, "final", resultsDir);
Console.WriteLine("Elapsed time: {0}\n", stopWatchTotal.Elapsed);
}
