/// <summary>
/// Loads the data file in the format (worker id, task id, worker label, ?gold label).
/// </summary>
/// <param name="filename">The data file.</param>
/// <returns>The list of parsed data.</returns>
public static IList<Datum> LoadData(string filename)
{
var result = new List<Datum>();
using (var reader = new StreamReader(filename))
{
string line;
while ((line = reader.ReadLine()) != null)
{
var strarr = line.Split(',');
int length = strarr.Length;
int workerLabel = int.Parse(strarr[2]);
var datum = new Datum()
{
WorkerId = strarr[0],
TaskId = strarr[1],
WorkerLabel = workerLabel,
};
if (length >= 4 && !strarr[3].Equals("NaN"))
datum.GoldLabel = int.Parse(strarr[3]);
else
datum.GoldLabel = null;
if (length == 5)
datum.TimeSpent = double.Parse(strarr[4]);
else
datum.TimeSpent = null;
result.Add(datum);
}
}
return result;
}
private List<Datum> BuildDataFromAssignedLabels(Dictionary<string, int?> AssignedLabels, IList<Datum> OriginalData)
{
List<Datum> data = new List<Datum>();
string firstWorkerId =WorkerIndexToId[0];
foreach (var entry in AssignedLabels)
{
var datum = new Datum();
datum.TaskId = entry.Key;
datum.GoldLabel = entry.Value;
datum.WorkerLabel = (int) entry.Value;
datum.WorkerId = firstWorkerId;
datum.BodyText = OriginalData.Where(d => d.TaskId == entry.Key).First().BodyText;
data.Add(datum);
}
if (data.Count == 0)
Console.WriteLine("*** Warning: There are no gold labels in the dataset ***");
return data;
}
