/// <summary>
/// Constructs an active learning instance with a specified data set and model instance.
/// </summary>
/// <param name="data">The data.</param>
/// <param name="model">The model instance.</param>
/// <param name="results">The results instance.</param>
/// <param name="numCommunities">The number of communities (only for CBCC).</param>
public ActiveLearning(IList<Datum> data, BCC model, Results results, int numCommunities)
{
this.bcc = model;
CBCC communityModel = model as CBCC;
IsCommunityModel = (communityModel != null);
ActiveLearningResults = results;
BatchResults = results;
isExperimentCompleted = false;
// Builds the full matrix of data from every task and worker
PredictionData = new List<Datum>();
}
/// <summary>
/// Background Thread for running the active learning experiment
/// <param name="worker"></param>
/// <param name="e"></param>
public void RunParallelActiveLearning(
System.ComponentModel.BackgroundWorker worker,
System.ComponentModel.DoWorkEventArgs e)
{
//Create a state of the Thread
CurrentParallelState currentState = new CurrentParallelState();
//Set setting in the experimentSetting Class
int totalNumberOfModels = GetNumberOfExperiemntModels();
//Clear previous results
ActiveLearning.ResetParallelAccuracyList(totalNumberOfModels);
//obtain the accuracy list reference
accuracyArrayOfAllExperimentModels = ActiveLearning.accuracyArray;
//The RunTypes that have Worker Confusion Matrices
RunType[] runTypesHaveWorkerMatrices = { RunType.DawidSkene, RunType.BCC, RunType.CBCC };
//Set the models selected in the setting pane
string[] currentModelNames = new string[totalNumberOfModels];
RunType[] currentRunTypes = new RunType[totalNumberOfModels];
TaskSelectionMethod[] currentTaskSelectionMethods = new TaskSelectionMethod[totalNumberOfModels];
WorkerSelectionMethod[] currentWorkerSelectionMethods = new WorkerSelectionMethod[totalNumberOfModels];
BCC[] currentBCCModels = new BCC[totalNumberOfModels];
//for each ExperimentModel, set runTypeArray, taskSelectionMethodArray, workerSelectionMethodArray...
for (int i = 0; i < totalNumberOfModels; i++)
{
ExperimentModel currentExperimentModel = GetExperimentModel(i);
RunType currentRunType = currentExperimentModel.runType;
currentRunTypes[i] = currentRunType;
//set the task selection method
currentTaskSelectionMethods[i] = currentExperimentModel.taskSelectionMethod;
//Add into worker selection method array if the runType can have worker selection
if (runTypesHaveWorkerMatrices.Contains(currentRunType))
{
currentWorkerSelectionMethods[i] = currentExperimentModel.WorkerSelectionMethod;
//Add corresponding model
//if the RunType is BCC, add into BCC model array
if (currentRunType == RunType.BCC)
{
currentBCCModels[i] = new BCC();
}//CBCC Model
else if(currentRunType == RunType.CBCC)
{
CBCC currentBCCmodel = new CBCC();
currentBCCModels[i] = currentBCCmodel;
}
} //end if the runType has worker confusion matrices
} //end for
currentModelNames = currentModelNames.Select((s, i) => CrowdsourcingModels.Program.GetModelName(currentDataset.GetDataSetNameWithoutExtension(), currentRunTypes[i])).ToArray();
//run RunParallelActiveLearning in the ActiveLearning
ActiveLearning.RunParallelActiveLearning(currentDataset.LoadData(), currentModelNames, currentRunTypes,
currentBCCModels, currentTaskSelectionMethods, currentWorkerSelectionMethods,
communityCount, numberOfLabellingRound);
currentState.isRunningComplete = true;
Debug.WriteLine("RunParallelActiveLearning Complete");
//isSimulationComplete = true;
//worker.ReportProgress(0, currentState);
}//end function RunParallelActiveLearning
}//end function RunParallelActiveLearning
/// <summary>
/// RunBatchRunning experiment in background thread
/// </summary>
/// <param name="worker"></param>
/// <param name="e"></param>
public void RunBatchRunningExperiment(
System.ComponentModel.BackgroundWorker worker,
System.ComponentModel.DoWorkEventArgs e)
{
CurrentParallelState currentState;
//Get the number of the total experiment Items
int totalNumberOfModels = GetNumberOfExperiemntModels();
//A List of Results array ofreport all experimentItems
results = new List<Results>();
currentState = new CurrentParallelState();
//Running the current experimentSetting lists and runGold accordinglyb
foreach (ExperimentModel currentExpItem in experimentModels)
{
//currentState.currentExperimentModel = currentExpItem;
if (MainPage.mainPageForm.isExperimentComplete)
{
return;
}
currentState = new CurrentParallelState();
currentState.currentExperimentModelIndex = GetExperimenModelIndex(currentExpItem);
currentState.isCurrentModelCompleted = false;
//Pass the started currentIndex to the mainpage, such that this currentExpItem is started
worker.ReportProgress(0, currentState);
//Create a BCC/CBCC model of the Batch Running Experiment
BCC currentModel = null;
if( currentExpItem.runType == RunType.BCC)
{
currentModel = new BCC();
}
else if(currentExpItem.runType == RunType.CBCC)
{
currentModel = new CBCC();
((CBCC)currentModel).SetCommunityCount(MainPage.mainPageForm.currentExperimentSetting.communityCount);
}
//When the experiment is not running
while (!MainPage.mainPageForm.isExperimentRunning )
{
}
if (MainPage.mainPageForm.isExperimentComplete)
{
return;
}
results.Add(CrowdsourcingModels.Program.RunBatchLearning(currentDataset.DatasetPath, currentExpItem.runType, currentModel, MainPage.mainPageForm.currentExperimentSetting.communityCount));
//When the experiment is not running
while (!MainPage.mainPageForm.isExperimentRunning)
{
}
if (MainPage.mainPageForm.isExperimentComplete)
{
return;
}
//add the results into the List<Results[]>
//convert the lists into a single array of results (using LINQ)
//notify the mainPage UI while it is completed
currentState.isCurrentModelCompleted = true;
worker.ReportProgress(0, currentState);
} // For each experimentItem
//The Batch Running is completed
currentState.isRunningComplete = true;
}
/// <summary>
/// Run the BCC models
/// </summary>
/// <param name="modelName"></param>
/// <param name="data"></param>
/// <param name="fullData"></param>
/// <param name="model"></param>
/// <param name="mode"></param>
/// <param name="calculateAccuracy"></param>
/// <param name="numCommunities"></param>
/// <param name="serialize"></param>
/// <param name="serializeCommunityPosteriors"></param>
public void RunBCC(string modelName,
IList <Datum> data,
IList <Datum> fullData,
BCC model,
RunMode mode,
bool calculateAccuracy,
int numCommunities = -1,
bool serialize = false,
bool serializeCommunityPosteriors = false)
{
CBCC communityModel = model as CBCC;
IsCommunityModel = communityModel != null;
bool IsBCC = !(IsCommunityModel);
if (this.Mapping == null)
{
this.Mapping = new DataMapping(fullData, numCommunities);
this.FullMapping = Mapping;
this.GoldLabels = this.Mapping.GetGoldLabelsPerTaskId();
}
bool createModel = (Mapping.LabelCount != model.LabelCount) || (Mapping.TaskCount != model.TaskCount);
if (IsCommunityModel)
{
//Console.WriteLine("--- CBCC ---");
CommunityCount = numCommunities;
createModel = createModel || (numCommunities != communityModel.CommunityCount);
if (createModel)
{
communityModel.CreateModel(Mapping.TaskCount, Mapping.LabelCount, numCommunities);
}
}
else if (createModel)
{
model.CreateModel(Mapping.TaskCount, Mapping.LabelCount);
}
BCCPosteriors priors = null;
switch (mode)
{
case RunMode.Prediction:
priors = ToPriors();
break;
default:
ClearResults();
if (mode == RunMode.LoadAndUseCommunityPriors && IsCommunityModel)
{
priors = DeserializeCommunityPosteriors(modelName, numCommunities);
}
break;
}
// Get data structures
int[][] taskIndices = Mapping.GetTaskIndicesPerWorkerIndex(data);
int[][] workerLabels = Mapping.GetLabelsPerWorkerIndex(data);
if (mode == RunMode.Prediction)
{
// Signal prediction mode by setting all labels to null
workerLabels = workerLabels.Select(arr => (int[])null).ToArray();
}
// Call inference
BCCPosteriors posteriors = model.Infer(
taskIndices,
workerLabels,
priors);
UpdateResults(posteriors, mode);
if (calculateAccuracy)
{
UpdateAccuracy();
}
if (serialize)
{
using (FileStream stream = new FileStream(modelName + ".xml", FileMode.Create))
{
var serializer = new System.Xml.Serialization.XmlSerializer(IsCommunityModel ? typeof(CBCCPosteriors) : typeof(BCCPosteriors));
serializer.Serialize(stream, posteriors);
}
}
if (serializeCommunityPosteriors && IsCommunityModel)
{
SerializeCommunityPosteriors(modelName);
}
}