/// <inheritdoc />
public override CrowdData LimitData(
int maxJudgments = int.MaxValue,
int maxNumTweets = int.MaxValue,
int maxNumWorkers = int.MaxValue,
int maxJudgmentsPerWorker = int.MaxValue,
int maxJudgmentsPerTweet = int.MaxValue,
bool balanceTweetsByLabel = false,
int randomSeed = 12347)
{
var baseResult = base.LimitData(maxJudgments, maxNumTweets, maxNumWorkers, maxJudgmentsPerWorker, maxJudgmentsPerTweet, balanceTweetsByLabel, randomSeed);
var tweetIds = new HashSet <string>(baseResult.TweetIds);
var result = new CrowdDataWithText
{
CrowdLabels = baseResult.CrowdLabels,
GoldLabels = baseResult.GoldLabels,
TweetTexts =
this.TweetTexts.Where(kvp => tweetIds.Contains(kvp.Key)).ToDictionary(
kvp => kvp.Key,
kvp => kvp.Value)
};
result.Tweets = Tweet.FromCrowdData(result);
result.Workers = Worker.FromCrowdData(result);
return(result);
}
/// <summary>
/// Creates tweet info from crowd data.
/// </summary>
/// <param name="crowdData">
/// The crowd data.
/// </param>
/// <returns>
/// The tweet information.
/// </returns>
public static Dictionary <string, Tweet> FromCrowdData(CrowdDataWithText crowdData)
{
var tweets = crowdData.TweetTexts;
return(crowdData.CrowdLabels.GroupBy(cd => cd.TweetId).ToDictionary(
grp => grp.Key,
grp =>
{
return new Tweet
{
TweetId = grp.Key,
TweetText =
tweets == null || !tweets.ContainsKey(grp.Key)
? string.Empty
: tweets[grp.Key],
GoldLabel =
crowdData.GoldLabels == null ||
!crowdData.GoldLabels.ContainsKey(grp.Key)
? (int?)null
: crowdData.GoldLabels[grp.Key],
// Occasionally a worker labels the same tweet more than once, so just take the first.
WorkerLabels =
grp.GroupBy(cd => cd.WorkerId).Select(grp1 => grp1.First())
.ToDictionary(cd => cd.WorkerId, cd => cd.WorkerLabel)
};
}));
}
/// <summary>
/// Creates worker info from a crowd datum.
/// </summary>
/// <param name="crowdData">
/// The crowd data.
/// </param>
/// <returns>
/// The <see cref="Worker"/>.
/// </returns>
public static Dictionary <string, Worker> FromCrowdData(CrowdDataWithText crowdData)
{
return(crowdData.CrowdLabels.GroupBy(cd => cd.WorkerId).ToDictionary(
grp => grp.Key,
grp => new Worker
{
WorkerId = grp.Key,
JudgedTweets = grp.Select(cd => crowdData.Tweets?[cd.TweetId]).ToList()
}));
}
/// <summary>
/// Run the random model.
/// </summary>
/// <param name="validationDataSet">
/// The validation data set.
/// </param>
/// <param name="modelName">
/// The model name.
/// </param>
/// <returns>
/// The metrics for the validation set.
/// </returns>
public static Dictionary <string, object> RunRandomModel(
CrowdDataWithText validationDataSet,
string modelName)
{
var validationRunner = (RunnerBase) new RandomModelRunner(
new CrowdDataMapping(validationDataSet, LabelValuesToString));
var result = RunModel(validationRunner, $"{modelName}Validation");
return(result);
}
/// <summary>
/// Run the majority vote model.
/// </summary>
/// <param name="validationDataSet">
/// The validation data set.
/// </param>
/// <param name="modelName">
/// The model name.
/// </param>
/// <returns>
/// The metrics for the validation set.
/// </returns>
public static Dictionary <string, object> RunMajorityVoteModel(
CrowdDataWithText validationDataSet,
string modelName)
{
var validationRunner = (RunnerBase) new MajorityVoteRunner(
new CrowdDataMapping(validationDataSet, LabelValuesToString));
var result = RunModel(validationRunner, modelName);
return(result);
}
/// <inheritdoc />
/// <summary>
/// Initializes a new instance of the <see cref="T:CrowdSourcing.WordsDataMapping" /> class.
/// </summary>
/// <param name="data">
/// The data.
/// </param>
/// <param name="labelValueToString">
/// The mapping from label values in file to label strings.
/// </param>
/// <param name="corpusInfo">
/// The corpus information.
/// </param>
/// <param name="maxWordsPerTweet">The maximum number of words considered per tweet.</param>
public CrowdDataWithTextMapping(
CrowdDataWithText data,
Dictionary <int, string> labelValueToString,
CorpusInformation corpusInfo)
: base(data, labelValueToString)
{
this.CorpusInfo = corpusInfo;
var docs = this.TweetIds.Select(tid => data.TweetTexts[tid]).ToArray();
this.WordIndicesPerTweetIndex = docs.Select(
doc =>
{
var indices = corpusInfo.GetWordIndices(doc).ToArray();
return(indices);
}).ToArray();
this.WordCountsPerTweetIndex = this.WordIndicesPerTweetIndex.Select(arr => arr.Length).ToArray();
}
/// <inheritdoc />
public override CrowdData RestrictDataToSingleTweet(string tweetId)
{
var baseResult = base.RestrictDataToSingleTweet(tweetId);
var result = new CrowdDataWithText
{
CrowdLabels = baseResult.CrowdLabels,
GoldLabels = baseResult.GoldLabels,
TweetTexts =
this.TweetTexts.Where(kvp => kvp.Key == tweetId).ToDictionary(
kvp => kvp.Key,
kvp => kvp.Value)
};
result.Tweets = Tweet.FromCrowdData(result);
result.Workers = Worker.FromCrowdData(result);
return(result);
}
/// <summary>
/// Split data. The validation data just consists of the specified fraction of tweets that contain gold labels,
/// along with all crowd labels for those tweets.
/// The training data consists of the labels (including gold labels), of all other tweets.
/// </summary>
/// <param name="fractionGoldLabelsForTraining">
/// The fraction gold label tweets for training.
/// </param>
/// <param name="randomSeed">
/// The random seed.
/// </param>
/// <returns>
/// The crowd data for the split.
/// </returns>
public new Dictionary <Mode, CrowdDataWithText> SplitData(double fractionGoldLabelsForTraining, int randomSeed = 12347)
{
var baseResult = base.SplitData(fractionGoldLabelsForTraining, randomSeed);
var trainingCrowdData = baseResult[Mode.Training];
var trainingIds = new HashSet <string>(trainingCrowdData.TweetIds);
var validationCrowdData = baseResult[Mode.Validation];
var validationIds = new HashSet <string>(validationCrowdData.TweetIds);
var trainingResult = new CrowdDataWithText
{
CrowdLabels = trainingCrowdData.CrowdLabels,
GoldLabels = trainingCrowdData.GoldLabels,
TweetTexts =
this.TweetTexts
.Where(kvp => trainingIds.Contains(kvp.Key))
.ToDictionary(kvp => kvp.Key, kvp => kvp.Value)
};
trainingResult.Tweets = Tweet.FromCrowdData(trainingResult);
trainingResult.Workers = Worker.FromCrowdData(trainingResult);
var validationResult = new CrowdDataWithText
{
CrowdLabels = validationCrowdData.CrowdLabels,
GoldLabels = validationCrowdData.GoldLabels,
TweetTexts =
this.TweetTexts
.Where(kvp => validationIds.Contains(kvp.Key))
.ToDictionary(kvp => kvp.Key, kvp => kvp.Value)
};
validationResult.Tweets = Tweet.FromCrowdData(validationResult);
validationResult.Workers = Worker.FromCrowdData(validationResult);
var result = new Dictionary <Mode, CrowdDataWithText>
{
[Mode.Training] = trainingResult,
[Mode.Validation] = validationResult
};
return(result);
}
/// <summary>
/// Loads the data.
/// </summary>
/// <param name="crowdLabelsFileName">
/// The crowd labels file name, format: (tweet id, worker id, worker label).
/// </param>
/// <param name="goldLabelsFileName">
/// The gold labels file name, format: (tweet id, gold label).
/// </param>
/// <param name="textsFileName">
/// The texts File Name.
/// </param>
/// <param name="allowedLabels">
/// The allowed labels.
/// </param>
/// <returns>
/// The crowd data.
/// </returns>
public static CrowdDataWithText LoadData(
string crowdLabelsFileName,
string goldLabelsFileName,
string textsFileName,
HashSet <int> allowedLabels)
{
var crowdData = CrowdData.LoadData(crowdLabelsFileName, goldLabelsFileName, allowedLabels);
var texts = File.ReadLines(textsFileName).Select(line => line.Split('\t'))
.ToDictionary(strarr => strarr[0], strarr => strarr[1]);
var tweetSet = new HashSet <string>(crowdData.CrowdLabels.Select(cd => cd.TweetId).Distinct());
var tweetsForData = texts.Where(kvp => tweetSet.Contains(kvp.Key))
.ToDictionary(kvp => kvp.Key, kvp => kvp.Value);
var result = new CrowdDataWithText {
CrowdLabels = crowdData.CrowdLabels, GoldLabels = crowdData.GoldLabels, TweetTexts = tweetsForData
};
result.Tweets = Tweet.FromCrowdData(result);
result.Workers = Worker.FromCrowdData(result);
return(result);
}
RunSweep(
CrowdDataWithText[] trainingDataSets,
CrowdDataWithText validationDataSet,
ModelBase model,
CorpusInformation corpusInformation,
Func <CrowdDataMapping, ModelBase, ModelRunnerBase, ModelRunnerBase> runnerCreator,
Func <ModelRunnerBase, Dictionary <string, object> > trainingResultGetter,
Func <ModelRunnerBase, Dictionary <string, object>, Dictionary <string, object> > validationResultGetter,
ExperimentParameters experimentParameters,
CrowdDataWithText validationDataSetNoLabels = null,
List <ModelRunnerBase> previousTrainingRunners = null,
int numIterations = 200,
Dictionary <string, Dictionary <string, object> > resultStorage = null)
{
var allResultsForThisModel = resultStorage ?? new Dictionary <string, Dictionary <string, object> >();
var trainingRunners = new List <ModelRunnerBase>();
const int NumIterationsForValidation = 5;
for (var i = 0; i < trainingDataSets.Length; i++)
{
var currentModelName =
$"{model.Name}_{i}"; // Chart code assumes suffix of dash then number for ordering chart points
var trainingData = trainingDataSets[i];
var trainingMapping = new CrowdDataWithTextMapping(
trainingData,
LabelValuesToString,
corpusInformation);
var trainingRunner = runnerCreator.Invoke(trainingMapping, model, previousTrainingRunners?[i]);
trainingRunners.Add(trainingRunner);
Rand.Restart(experimentParameters.RandomSeed);
RunModel(trainingRunner, currentModelName + "_Training", numIterations,
experimentParameters.UseGoldLabelsInTraining);
var trainingResults = trainingResultGetter?.Invoke(trainingRunner) ?? new Dictionary <string, object>();
trainingResults[ErrorsKey] = trainingRunner.GetErrors();
trainingResults[WorkerMetricsKey] = GetWorkerMetrics(trainingRunner.DataMapping.Data, trainingRunner, experimentParameters.MaximumNumberWorkers);
var currentResults =
new Dictionary <string, object> {
[TrainingKey] = trainingResults
};
var validationMapping = new CrowdDataWithTextMapping(
validationDataSet,
LabelValuesToString,
corpusInformation);
var validationRunner = runnerCreator.Invoke(validationMapping, model, trainingRunner);
Rand.Restart(experimentParameters.RandomSeed);
var validationMetrics = RunModel(validationRunner, currentModelName + "_Validation",
NumIterationsForValidation, false);
foreach (var prediction in validationRunner.Posteriors.TrueLabel)
{
Console.WriteLine(prediction);
}
var validationResults = validationResultGetter.Invoke(validationRunner, validationMetrics);
validationResults[ErrorsKey] = validationRunner.GetErrors();
validationResults[WorkerMetricsKey] = GetWorkerMetrics(validationRunner.DataMapping.Data,
validationRunner, experimentParameters.MaximumNumberWorkers);
currentResults[ValidationKey] = validationResults;
if (validationDataSetNoLabels != null)
{
var validationMappingNoLabels = new CrowdDataWithTextMapping(
validationDataSetNoLabels,
LabelValuesToString,
corpusInformation);
var validationNoLabelsRunner =
runnerCreator.Invoke(validationMappingNoLabels, model, trainingRunner);
Rand.Restart(experimentParameters.RandomSeed);
var validationMetricsNoLabels = RunModel(validationNoLabelsRunner,
currentModelName + "_ValidationNoLabels", NumIterationsForValidation, false);
currentResults[ValidationNoLabelsKey] =
validationResultGetter.Invoke(validationNoLabelsRunner, validationMetricsNoLabels);
}
allResultsForThisModel[$"TrainingPercent_{Math.Min((i + 1) * 100 / trainingDataSets.Length, 100)}"] =
currentResults;
}
return(allResultsForThisModel, trainingRunners);
}
public static void RunExperiments(Outputter outputter, ExperimentParameters experimentParameters, string dataPath)
{
var engine = new InferenceEngine(new ExpectationPropagation())
{
ShowFactorGraph = false,
ShowWarnings = true,
ShowProgress = false
};
// Set engine flags
engine.Compiler.WriteSourceFiles = true;
engine.Compiler.UseParallelForLoops = false;
string labelsFileName = Path.Combine(dataPath, "HarnessingTheCrowdLabels.tsv");
string goldLabelsFileName = Path.Combine(dataPath, "HarnessingTheCrowdGoldLabels.tsv");
string textsFileName = Path.Combine(dataPath, "HarnessingTheCrowdTexts.tsv");
if (!File.Exists(labelsFileName) ||
!File.Exists(goldLabelsFileName) ||
!File.Exists(textsFileName))
{
throw new FileNotFoundException("Unfortunately, we were not able to ship the data necessary to run the code for this chapter.");
}
var crowdData = CrowdDataWithText.LoadData(
labelsFileName,
goldLabelsFileName,
textsFileName,
new HashSet <int>(LabelValuesToString.Keys));
Rand.Restart(experimentParameters.RandomSeed);
// Preparing the input data
var split = crowdData.SplitData(experimentParameters.FractionGoldLabelsReservedForTraining);
var fullTrainingData = split[CrowdData.Mode.Training];
var fullValidationData = split[CrowdData.Mode.Validation];
var trainingData = (CrowdDataWithText)fullTrainingData.LimitData(
maxNumTweets: experimentParameters.UseOnlyTweetsWithGoldLabels
? fullTrainingData.NumGoldTweets
: experimentParameters.NumberOfTrainingTweets,
randomSeed: experimentParameters.RandomSeed);
if (experimentParameters.UseBalancedTrainingSets)
{
trainingData = (CrowdDataWithText)fullTrainingData.LimitData(
maxNumTweets: experimentParameters.UseOnlyTweetsWithGoldLabels
? fullTrainingData.NumGoldTweets
: experimentParameters.NumberOfTrainingTweets,
balanceTweetsByLabel: true,
randomSeed: experimentParameters.RandomSeed);
}
var validationData = (CrowdDataWithText)fullValidationData.LimitData(
experimentParameters.NumberOfValidationJudgments,
randomSeed: experimentParameters.RandomSeed);
var validationDataWithNoWorkerLabels = (CrowdDataWithText)validationData.LimitData(maxNumWorkers: 0);
var trainingWorkerMetrics = GetWorkerMetrics(trainingData, null, experimentParameters.MaximumNumberWorkers);
var validationWorkerMetrics =
GetWorkerMetrics(validationData, null, experimentParameters.MaximumNumberWorkers);
var corpus = trainingData.TweetTexts.Values.Distinct().ToArray();
var corpusInformation = CorpusInformation.BuildCorpusInformation(
corpus,
experimentParameters.VocabularyThreshold);
var totalNumTrainingJudgments = trainingData.CrowdLabels.Count;
var numTrainingJudgments = Util.ArrayInit(
experimentParameters.NumDataSizes,
i => Math.Round(totalNumTrainingJudgments * (i + 1) / (double)experimentParameters.NumDataSizes));
var trainingDataSets = Util.ArrayInit(
experimentParameters.NumDataSizes,
i => (CrowdDataWithText)trainingData.LimitData((int)numTrainingJudgments[i],
randomSeed: experimentParameters.RandomSeed));
var trainingTweets = trainingData.Tweets;
var trainingCrowdWorkers = trainingData.Workers;
var validationTweets = validationData.Tweets;
var validationCrowdWorkers = validationData.Workers;
var allDataInfo = new Dictionary <string, object>
{
["Full training data"] = fullTrainingData,
["Full validation data"] = fullValidationData,
["Validation data"] = validationData,
["Training data"] = trainingData,
["Training data sets"] = trainingDataSets,
["Validation data set"] = validationData,
["Training set tweets"] = trainingTweets,
["Training set crowd workers"] = trainingCrowdWorkers,
["Validation set tweets"] = validationTweets,
["Validation set crowd workers"] = validationCrowdWorkers,
["CorpusInformation"] = corpusInformation
};
outputter.Out(allDataInfo, "Inputs");
WriteCrowdDataDetailsToConsole(fullTrainingData, "Full training data");
WriteCrowdDataDetailsToConsole(fullValidationData, "Full validation data");
WriteCrowdDataDetailsToConsole(trainingData, "Training set");
WriteCrowdDataDetailsToConsole(validationData, "Validation set");
var now = DateTime.Now;
var nowStr = $"{now.Year:D4}{now.Month:D2}{now.Day:D2}T{now.Hour:D2}{now.Minute:D2}{now.Second:D2}";
var experimentInfo = new Dictionary <string, object>
{
["Date"] = nowStr,
["Parameters"] = experimentParameters,
[$"Training {WorkerMetricsKey}"] = trainingWorkerMetrics,
[$"Validation {WorkerMetricsKey}"] = validationWorkerMetrics,
[$"Training {LabelCountHistogramKey}"] = GetWorkerLabelCountHistogram(trainingData),
[$"Validation {LabelCountHistogramKey}"] = GetWorkerLabelCountHistogram(validationData)
};
outputter.Out(experimentInfo, "Experiment");
// Data structures for comparing results
var metricsForPlots = new List <Metric> {
Metric.Accuracy, Metric.AverageLogProb
};
var metricsStripCharts = new Dictionary <string, Dictionary <string, PointWithBounds[]> >();
var metricsBarCharts = new Dictionary <string, Dictionary <string, double> >();
foreach (var metric in metricsForPlots)
{
var metricString = metric.ToString();
metricsStripCharts[metricString] = new Dictionary <string, PointWithBounds[]>();
metricsBarCharts[metricString] = new Dictionary <string, double>();
}
// Creates a snapshot of currently accumulated comparison metrics
Dictionary <string, object> getCurrentComparisonSnapshot()
{
var metricsStripChartsSnapshot =
new Dictionary <string, Dictionary <string, PointWithBounds[]> >();
var metricsBarChartsSnapshot =
new Dictionary <string, Dictionary <string, double> >();
foreach (var metric in metricsForPlots)
{
var metricString = metric.ToString();
metricsStripChartsSnapshot[metricString] = new Dictionary <string, PointWithBounds[]>(metricsStripCharts[metricString]);
metricsBarChartsSnapshot[metricString] = new Dictionary <string, double>(metricsBarCharts[metricString]);
}
return(new Dictionary <string, object>
{
["Strip Charts"] = metricsStripChartsSnapshot,
["Bar Charts"] = metricsBarChartsSnapshot
});
}
// Running models as specified in experiment parameters
bool section2Started = false;
Dictionary <string, object> section2Comparison = null;
if ((experimentParameters.ModelTypes & ModelTypes.MajorityVote) != 0)
{
Console.WriteLine($"\n{Contents.S2TryingOutTheWorkerModel.NumberedName}\n");
section2Started = true;
var accuracyString = Metric.Accuracy.ToString();
var majorityVoteString = "Majority Vote";
var majorityVoteMetrics = RunMajorityVoteModel(validationData, majorityVoteString);
outputter.Out(majorityVoteMetrics, Contents.S2TryingOutTheWorkerModel.NumberedName, majorityVoteString);
metricsStripCharts[accuracyString][majorityVoteString] = GetChartPointsForMetric(
Enumerable.Repeat(majorityVoteMetrics, trainingDataSets.Length).ToList(),
Metric.Accuracy.ToString(),
numTrainingJudgments);
metricsBarCharts[accuracyString][majorityVoteString] =
metricsStripCharts[accuracyString][majorityVoteString].Last().Y;
section2Comparison = getCurrentComparisonSnapshot();
outputter.Out(section2Comparison, Contents.S2TryingOutTheWorkerModel.NumberedName, "Comparison");
}
if ((experimentParameters.ModelTypes & ModelTypes.HonestWorker) != 0)
{
if (!section2Started)
{
Console.WriteLine($"\n{Contents.S2TryingOutTheWorkerModel.NumberedName}\n");
}
var currentModel = new HonestWorkerModel(engine);
var honestWorkerMetrics = new Dictionary <string, Dictionary <string, object> >();
outputter.Out(honestWorkerMetrics, Contents.S2TryingOutTheWorkerModel.NumberedName, currentModel.Name);
honestWorkerMetrics = RunSweep(
trainingDataSets,
validationData,
currentModel,
corpusInformation,
(map, model, runner) => new HonestWorkerRunner(map, (HonestWorkerModel)model, runner),
GetHonestWorkerTrainingResults,
(runner, metrics) =>
GetHonestWorkerValidationResults(runner, metrics, experimentParameters.MaximumNumberWorkers),
experimentParameters,
resultStorage: honestWorkerMetrics).Results;
foreach (var metric in metricsForPlots)
{
metricsStripCharts[metric.ToString()][currentModel.Name] =
GetChartPointsForMetric(honestWorkerMetrics, metric, numTrainingJudgments);
metricsBarCharts[metric.ToString()][currentModel.Name] =
metricsStripCharts[metric.ToString()][currentModel.Name].Last().Y;
}
section2Comparison = getCurrentComparisonSnapshot();
outputter.Out(section2Comparison, Contents.S2TryingOutTheWorkerModel.NumberedName, "Comparison");
}
if ((experimentParameters.ModelTypes & ModelTypes.BiasedWorker) != 0)
{
Console.WriteLine($"\n{Contents.S3CorrectingForWorkerBiases.NumberedName}\n");
var currentModel = new BiasedWorkerModel(engine);
var biasedWorkerMetrics = new Dictionary <string, Dictionary <string, object> >();
outputter.Out(biasedWorkerMetrics, Contents.S3CorrectingForWorkerBiases.NumberedName, currentModel.Name);
biasedWorkerMetrics = RunSweep(
trainingDataSets,
validationData,
currentModel,
corpusInformation,
(map, model, runner) => new BiasedWorkerModelRunner(map, (BiasedWorkerModel)model, runner),
runner => GetBiasedWorkerTrainingResults(runner, experimentParameters.MaximumNumberWorkers),
GetValidationResults,
experimentParameters,
resultStorage: biasedWorkerMetrics).Results;
foreach (var metric in metricsForPlots)
{
metricsStripCharts[metric.ToString()][currentModel.Name] =
GetChartPointsForMetric(biasedWorkerMetrics, metric, numTrainingJudgments);
metricsBarCharts[metric.ToString()][currentModel.Name] =
metricsStripCharts[metric.ToString()][currentModel.Name].Last().Y;
}
outputter.Out(getCurrentComparisonSnapshot(), Contents.S3CorrectingForWorkerBiases.NumberedName, "Comparison");
}
if ((experimentParameters.ModelTypes & ModelTypes.BiasedCommunity) != 0)
{
Console.WriteLine($"\n{Contents.S4CommunitiesOfWorkers.NumberedName}\n");
engine.Algorithm = new VariationalMessagePassing();
foreach (int numCommunities in experimentParameters.NumberOfCommunitiesSweep)
{
var currentModel = new BiasedCommunityModel(engine)
{
NumberOfCommunities = numCommunities
};
var result = new Dictionary <string, Dictionary <string, object> >();
outputter.Out(result, Contents.S4CommunitiesOfWorkers.NumberedName, currentModel.Name);
result = RunSweep(
trainingDataSets,
validationData,
currentModel,
corpusInformation,
(map, model, runner) => new BiasedCommunityModelRunner(
map,
(BiasedCommunityModel)model,
runner),
GetBiasedCommunityTrainingResults,
GetValidationResults,
experimentParameters,
resultStorage: result).Results;
foreach (var metric in metricsForPlots)
{
metricsStripCharts[metric.ToString()][currentModel.Name] =
GetChartPointsForMetric(result, metric, numTrainingJudgments);
metricsBarCharts[metric.ToString()][currentModel.Name] =
metricsStripCharts[metric.ToString()][currentModel.Name].Last().Y;
}
}
outputter.Out(getCurrentComparisonSnapshot(), Contents.S4CommunitiesOfWorkers.NumberedName, "Comparison");
}
if ((experimentParameters.ModelTypes & ModelTypes.BiasedCommunityWords) != 0)
{
Console.WriteLine($"\n{Contents.S5MakingUseOfTheTweets.NumberedName}\n");
// VMP is an order of magnitude faster than EP here, while producing almost the same result
engine.Algorithm = new VariationalMessagePassing();
foreach (int numCommunities in experimentParameters.NumberOfCommunitiesSweep)
{
var currentModel = new BiasedCommunityWordsModel(engine)
{
NumberOfCommunities = numCommunities
};
var result = new Dictionary <string, Dictionary <string, object> >();
outputter.Out(result, Contents.S5MakingUseOfTheTweets.NumberedName, currentModel.Name);
result = RunSweep(
trainingDataSets,
validationData,
currentModel,
corpusInformation,
(map, model, runner) => new BiasedCommunityWordsRunner(
(CrowdDataWithTextMapping)map,
(BiasedCommunityWordsModel)model,
runner),
GetBiasedCommunityWordsRunnerResults,
GetValidationResults,
experimentParameters,
validationDataWithNoWorkerLabels,
null,
50,
resultStorage: result).Results;
foreach (var metric in metricsForPlots)
{
metricsStripCharts[metric.ToString()][currentModel.Name] =
GetChartPointsForMetric(result, metric, numTrainingJudgments);
metricsBarCharts[metric.ToString()][currentModel.Name] =
metricsStripCharts[metric.ToString()][currentModel.Name].Last().Y;
}
}
outputter.Out(getCurrentComparisonSnapshot(), Contents.S5MakingUseOfTheTweets.NumberedName, "Comparison");
}
Console.WriteLine("\nCompleted all experiments.");
}