public static void ValidateSameTaskSameWorkerMultipleResultsEffect(string jobGUID, string confusingImageListFilePath = null)
{
SatyamResultsTableAccess resultsDB = new SatyamResultsTableAccess();
List <SatyamResultsTableEntry> entries = resultsDB.getEntriesByGUIDOrderByID(jobGUID);
resultsDB.close();
Dictionary <int, int> noResultsPerTask = new Dictionary <int, int>();
Dictionary <int, int> noAcceptedResultsPerTask = new Dictionary <int, int>();
Dictionary <int, int> noPaidResultsPerTask = new Dictionary <int, int>();
Dictionary <int, double> moneyPaidPerTask = new Dictionary <int, double>();
SortedDictionary <int, Dictionary <string, List <string> > > ResultsPerWorkerPerTask = new SortedDictionary <int, Dictionary <string, List <string> > >();
SortedDictionary <int, Dictionary <string, int> > noDifferentDuplicateResultsPerWorkerPerTask = new SortedDictionary <int, Dictionary <string, int> >();
Dictionary <int, string> taskSatyamUri = new Dictionary <int, string>();
Dictionary <int, double> timeTakenTillAggregationPerTask = new Dictionary <int, double>();
Dictionary <int, DateTime> finalTaskEndTime = new Dictionary <int, DateTime>();
Dictionary <string, double> finalHITEndTime = new Dictionary <string, double>();
int totalResults = 0;
int noTasksWithDuplicateResultsFromSameWorker = 0;
int noTasksWithMixedResultsFromSameWorker = 0;
int noTaskWithDuplicateResultsAsMajority = 0;
int noTaskWhoseDuplicateResultsChangedAggregation = 0;
int noTaskWhoseDuplicateResultsChangedAggregationIncorrectly = 0;
int noCorrectDecisionsAmongDuplicates = 0;
//noCorrectDuplicateResultsOfSameWorkerSameTask = 0;
int noTasksWithCorrectDuplicateResultsOfSameWorkerSameTask = 0;
int noWorkerSwitchedToCorrect = 0;
int noWorkerSwitchedToCorrectAndMaintained = 0;
int noTasksWithWorkerSwitchedToCorrect = 0;
int noTasksWithWorkerSwitchedToCorrectAndMaintained = 0;
List <double> timeTakenPerResult = new List <double>();
List <double> acceptedTimeTakenPerResult = new List <double>();
List <int> SequenceNumberOfResultPerTask = new List <int>();
List <int> SequenceNumberOfResultPerWorker = new List <int>();
string JobGUID = entries[0].JobGUID;
Dictionary <string, int> noJobsPerWorker = new Dictionary <string, int>();
Dictionary <string, int> noAcceptedJobsPerWorker = new Dictionary <string, int>();
Dictionary <string, int> noPaidJobsPerWorker = new Dictionary <string, int>();
List <double> resultArrivalTimes = new List <double>();
totalResults = entries.Count;
///imagenet only
Dictionary <string, ConfusingReason> imageBlackListReason = new Dictionary <string, ConfusingReason>();
if (confusingImageListFilePath != null)
{
imageBlackListReason = getConfusingImageList(confusingImageListFilePath);
}
for (int i = 0; i < entries.Count; i++)
{
SatyamResultsTableEntry entry = entries[i];
SatyamResult satyamResult = JSonUtils.ConvertJSonToObject <SatyamResult>(entry.ResultString);
SatyamTask task = JSonUtils.ConvertJSonToObject <SatyamTask>(satyamResult.TaskParametersString);
SatyamJob job = task.jobEntry;
if (IsBlackListed(task.SatyamURI, imageBlackListReason))
{
continue;
}
int taskEntryID = entry.SatyamTaskTableEntryID;
if (!noResultsPerTask.ContainsKey(taskEntryID))
{
noResultsPerTask.Add(taskEntryID, 0);
noAcceptedResultsPerTask.Add(taskEntryID, 0);
noPaidResultsPerTask.Add(taskEntryID, 0);
moneyPaidPerTask.Add(taskEntryID, 0);
finalTaskEndTime.Add(taskEntryID, entry.SubmitTime);
ResultsPerWorkerPerTask.Add(taskEntryID, new Dictionary <string, List <string> >());
noDifferentDuplicateResultsPerWorkerPerTask.Add(taskEntryID, new Dictionary <string, int>());
taskSatyamUri.Add(taskEntryID, task.SatyamURI);
}
if (!ResultsPerWorkerPerTask[taskEntryID].ContainsKey(satyamResult.amazonInfo.WorkerID))
{
ResultsPerWorkerPerTask[taskEntryID].Add(satyamResult.amazonInfo.WorkerID, new List <string>());
noDifferentDuplicateResultsPerWorkerPerTask[taskEntryID].Add(satyamResult.amazonInfo.WorkerID, 0);
}
if (!ResultsPerWorkerPerTask[taskEntryID][satyamResult.amazonInfo.WorkerID].Contains(satyamResult.TaskResult))
{
noDifferentDuplicateResultsPerWorkerPerTask[taskEntryID][satyamResult.amazonInfo.WorkerID]++;
}
ResultsPerWorkerPerTask[taskEntryID][satyamResult.amazonInfo.WorkerID].Add(satyamResult.TaskResult);
}
///////////////////////////////////// Per Task Analysis //////////////////////////////////////
SortedDictionary <int, int> resultsPerTaskHistogram = new SortedDictionary <int, int>();
SortedDictionary <int, int> resultsAcceptedPerTaskHistogram = new SortedDictionary <int, int>();
SortedDictionary <int, int> resultsPaidPerTaskHistogram = new SortedDictionary <int, int>();
SortedDictionary <int, int> moneyPaidPerTaskHistogram = new SortedDictionary <int, int>(); //cents
SortedDictionary <int, int> ResultsPerWorkerPerTaskHistogram = new SortedDictionary <int, int>();
List <int> taskIDs = noResultsPerTask.Keys.ToList();
foreach (int taskID in taskIDs)
{
if (ResultsPerWorkerPerTask[taskID].Count != noResultsPerTask[taskID])
{
//has multiple results from same turker
noTasksWithDuplicateResultsFromSameWorker++;
// the aggregation result
List <SingleObjectLabelingResult> allResultsPerTask = new List <SingleObjectLabelingResult>();
SingleObjectLabelingAggregatedResult aggregatedResultPerTask = new SingleObjectLabelingAggregatedResult();
// the aggregation if without duplicate results
List <SingleObjectLabelingResult> OnlyFirstResultOfEachTurkerPerTask = new List <SingleObjectLabelingResult>();
SingleObjectLabelingAggregatedResult aggregatedOnlyFirstResultOfTurkerPerTask = new SingleObjectLabelingAggregatedResult();
foreach (List <string> ResultsStringsPerWorkerPerTask in ResultsPerWorkerPerTask[taskID].Values)
{
foreach (string resultString in ResultsStringsPerWorkerPerTask)
{
allResultsPerTask.Add(JSonUtils.ConvertJSonToObject <SingleObjectLabelingResult>(resultString));
}
OnlyFirstResultOfEachTurkerPerTask.Add(JSonUtils.ConvertJSonToObject <SingleObjectLabelingResult>(ResultsStringsPerWorkerPerTask[0]));
}
aggregatedResultPerTask = SingleObjectLabelingAggregator.getAggregatedResult(allResultsPerTask);
aggregatedOnlyFirstResultOfTurkerPerTask = SingleObjectLabelingAggregator.getAggregatedResult(OnlyFirstResultOfEachTurkerPerTask);
if (aggregatedResultPerTask != null)
{
if (aggregatedOnlyFirstResultOfTurkerPerTask == null ||
aggregatedResultPerTask.Category != aggregatedOnlyFirstResultOfTurkerPerTask.Category)
{
noTaskWhoseDuplicateResultsChangedAggregation++;
if (AggregatedResultEqualsGroundTruth(taskSatyamUri[taskID], JSonUtils.ConvertObjectToJSon(aggregatedResultPerTask)))
{
noTaskWhoseDuplicateResultsChangedAggregationIncorrectly++;
}
}
}
bool hasMixedResults = false;
bool hasCorrectResult = false;
bool atLeastOneShiftedFromIncorrectToCorrect = false;
bool atLeastOneShiftedFromIncorrectToCorrectAndMaintained = false;
foreach (List <string> ResultsStringsPerWorkerPerTask in ResultsPerWorkerPerTask[taskID].Values)
{
if (!ResultsPerWorkerPerTaskHistogram.ContainsKey(ResultsStringsPerWorkerPerTask.Count))
{
ResultsPerWorkerPerTaskHistogram.Add(ResultsStringsPerWorkerPerTask.Count, 0);
}
ResultsPerWorkerPerTaskHistogram[ResultsStringsPerWorkerPerTask.Count]++;
double superMajority = 0.6; // tunable
if (((double)ResultsStringsPerWorkerPerTask.Count + 1) / ((double)noResultsPerTask[taskID] + 2) > superMajority)
{
noTaskWithDuplicateResultsAsMajority++;
}
///////////// imagenet only //////////////////
if (ResultsStringsPerWorkerPerTask.Distinct().Count() > 1)
{
// multiple choices given
hasMixedResults = true;
bool incorrect = false;
bool switchToCorrect = false;
foreach (string resultString in ResultsStringsPerWorkerPerTask)
{
if (SingleObjectLabelingResultEqualsGroundTruth(taskSatyamUri[taskID], resultString))
{
hasCorrectResult = true;
noCorrectDecisionsAmongDuplicates++;
if (incorrect)
{
switchToCorrect = true;
}
incorrect = false;
}
else
{
incorrect = true;
}
}
if (switchToCorrect)
{
// been incorrect and swithed to correct
noWorkerSwitchedToCorrect++;
atLeastOneShiftedFromIncorrectToCorrect = true;
if (!incorrect)
{
//switch to correct and maintain till the end
noWorkerSwitchedToCorrectAndMaintained++;
atLeastOneShiftedFromIncorrectToCorrectAndMaintained = true;
}
}
}
////////////////////////////////////////////////
}
if (hasMixedResults)
{
noTasksWithMixedResultsFromSameWorker++;
}
if (hasCorrectResult)
{
noTasksWithCorrectDuplicateResultsOfSameWorkerSameTask++;
}
if (atLeastOneShiftedFromIncorrectToCorrect)
{
noTasksWithWorkerSwitchedToCorrect++;
}
if (atLeastOneShiftedFromIncorrectToCorrectAndMaintained)
{
noTasksWithWorkerSwitchedToCorrectAndMaintained++;
}
}
}
Console.WriteLine("DuplicateResultsHistogram");
foreach (int no in ResultsPerWorkerPerTaskHistogram.Keys)
{
Console.WriteLine("{0}, {1}", no, ResultsPerWorkerPerTaskHistogram[no]);
}
Console.WriteLine(
//"{0} images (in total {1}({2} of which are correct)) has duplicate results from same turker, \n" +
//"\t{3} images have >=1 workers making more than majority number of results.\n" +
//"\t{11} images aggregation is changed ({12} of which incorrectly) by duplicate results from same turker.\n" +
//"\t{4} images(In total {5} times) with duplicate results include mixed(>= 2) decisions from same worker.\n" +
"\t\t{6} images has correct decision among duplicate decisions.\n" +
"\t\t{7} images({8} times) a worker has switched from incorrect to correct choice\n" +
"\t\t\t{9} images({10} times) of which maintained the correct choice till the last time of their job on that image.",
//noTasksWithDuplicateResultsFromSameWorker, noDuplicateResultsFromSameWorker, noCorrectDecisionsAmongDuplicates,
//noTaskWithDuplicateResultsAsMajority,
//noTasksWithMixedResultsFromSameWorker, noDecisionChangeAmongDuplicateResultsOfSameWorkerSameTask,
//noTasksWithCorrectDuplicateResultsOfSameWorkerSameTask,
noTasksWithWorkerSwitchedToCorrect, noWorkerSwitchedToCorrect,
noTasksWithWorkerSwitchedToCorrectAndMaintained, noWorkerSwitchedToCorrectAndMaintained,
noTaskWhoseDuplicateResultsChangedAggregation, noTaskWhoseDuplicateResultsChangedAggregationIncorrectly
);
}
public static void AggregateWithParameterAndValidateSatyamVideoClassificationResultByGUID(string guid,
int MinResults = TaskConstants.SINGLE_OBJECT_LABLING_MTURK_MIN_RESULTS_TO_AGGREGATE,
int MaxResults = TaskConstants.SINGLE_OBJECT_LABLING_MTURK_MAX_RESULTS_TO_AGGREGATE,
double MajorityThreshold = TaskConstants.SINGLE_OBJECT_LABLING_MTURK_MAJORITY_THRESHOLD,
bool prepareDataForTraining = false, string outputDirectory = null,
string confusingVideoListFilePath = null)
{
string configString = "Min_" + MinResults + "_Max_" + MaxResults + "_Thresh_" + MajorityThreshold;
Console.WriteLine("Aggregating for param set " + configString);
int noTerminatedTasks = 0;
SatyamResultsTableAccess resultsDB = new SatyamResultsTableAccess();
List <SatyamResultsTableEntry> entries = resultsDB.getEntriesByGUIDOrderByID(guid);
resultsDB.close();
SortedDictionary <DateTime, List <SatyamResultsTableEntry> > entriesBySubmitTime = SatyamResultValidationToolKit.SortResultsBySubmitTime(entries);
int noTotalConverged = 0;
int noCorrect = 0;
Dictionary <int, List <SatyamResultsTableEntry> > ResultsPerTask = new Dictionary <int, List <SatyamResultsTableEntry> >();
List <int> aggregatedTasks = new List <int>();
SortedDictionary <string, Dictionary <string, int> > confusionMatrix_res_groundtruth = new SortedDictionary <string, Dictionary <string, int> >();
SortedDictionary <string, Dictionary <string, int> > confusionMatrix_groundtruth_res = new SortedDictionary <string, Dictionary <string, int> >();
List <SatyamAggregatedResultsTableEntry> aggEntries = new List <SatyamAggregatedResultsTableEntry>();
Dictionary <int, int> noResultsNeededForAggregation = SatyamResultsAnalysis.getNoResultsNeededForAggregationFromLog(configString, guid);
Dictionary <int, int> noResultsNeededForAggregation_new = new Dictionary <int, int>();
foreach (DateTime t in entriesBySubmitTime.Keys)
{
//Console.WriteLine("Processing Results of time: {0}", t);
foreach (SatyamResultsTableEntry entry in entriesBySubmitTime[t])
{
SatyamResult satyamResult = JSonUtils.ConvertJSonToObject <SatyamResult>(entry.ResultString);
SatyamTask task = JSonUtils.ConvertJSonToObject <SatyamTask>(satyamResult.TaskParametersString);
SatyamJob job = task.jobEntry;
string fileName = URIUtilities.filenameFromURI(task.SatyamURI);
string VideoCategoryGroundTruth = getVideoCategoryFromFileName(fileName);
int taskEntryID = entry.SatyamTaskTableEntryID;
if (aggregatedTasks.Contains(taskEntryID))
{
continue;
}
if (!ResultsPerTask.ContainsKey(taskEntryID))
{
ResultsPerTask.Add(taskEntryID, new List <SatyamResultsTableEntry>());
}
ResultsPerTask[taskEntryID].Add(entry);
// check log if enough results are collected
if (noResultsNeededForAggregation != null && noResultsNeededForAggregation.ContainsKey(taskEntryID) &&
ResultsPerTask[taskEntryID].Count < noResultsNeededForAggregation[taskEntryID])
{
continue;
}
//SingleObjectLabelingAggregatedResult aggResult = SingleObjectLabelingAggregator.getAggregatedResult(ResultsPerTask[taskEntryID], MinResults, MaxResult, MajorityThreshold);
string aggResultString = SingleObjectLabelingAggregator.GetAggregatedResultString(ResultsPerTask[taskEntryID], MinResults, MaxResults, MajorityThreshold);
if (aggResultString == null)
{
continue;
}
SatyamAggregatedResultsTableEntry aggEntry = new SatyamAggregatedResultsTableEntry();
aggEntry.JobGUID = ResultsPerTask[taskEntryID][0].JobGUID;
aggEntry.JobTemplateType = ResultsPerTask[taskEntryID][0].JobTemplateType;
aggEntry.SatyamTaskTableEntryID = taskEntryID;
aggEntry.UserID = ResultsPerTask[taskEntryID][0].UserID;
aggEntry.ResultString = aggResultString;
/// aggregation happen
// record logs
if (noResultsNeededForAggregation == null || !noResultsNeededForAggregation.ContainsKey(taskEntryID))
{
noResultsNeededForAggregation_new.Add(taskEntryID, ResultsPerTask[taskEntryID].Count);
}
aggEntries.Add(aggEntry);
noTotalConverged++;
if (ResultsPerTask[taskEntryID].Count >= MaxResults)
{
noTerminatedTasks++;
}
aggregatedTasks.Add(taskEntryID);
}
}
SatyamResultsAnalysis.RecordAggregationLog(noResultsNeededForAggregation_new, configString, guid);
string outputmatFile = DirectoryConstants.defaultTempDirectory + guid + "\\" + configString + "_mat.txt";
EvaluateAndPrintConfusionMatrixOfAggregatedResultEntries(aggEntries, outputmatFile, out noCorrect, prepareDataForTraining, outputDirectory);
Console.WriteLine("noTotalConverged {0}", noTotalConverged);
Console.WriteLine("noTerminatedTasks {0}", noTerminatedTasks);
Console.WriteLine("Result: {0}/{1}, precision: {2}", noCorrect, noTotalConverged, (double)noCorrect / noTotalConverged);
/// local file
string outputString = String.Format("{0} {1} {2} {3} {4} {5}\n", configString, noCorrect, noTotalConverged, (double)noCorrect / noTotalConverged, noTerminatedTasks, ResultsPerTask.Count - noTotalConverged);
string outputfile = DirectoryConstants.defaultTempDirectory + guid + "\\resultSummary.txt";
File.AppendAllText(outputfile, outputString);
//for (double prob = 0; prob < 1;prob +=0.2)
//{
// SatyamResultsAnalysis.AnalyzeApprovalRate(aggEntries, entries, guid, configString, anotherChanceProbablity: prob);
//}
//for (double ratio = 0; ratio < 1; ratio += 0.2)
//{
// SatyamResultsAnalysis.AnalyzeApprovalRate(aggEntries, entriesBySubmitTime, noResultsNeededForAggregation, noResultsNeededForAggregation_new, guid, configString, approvalRatioThreshold: ratio);
//}
SatyamResultsAnalysis.AggregationAnalysis(aggEntries, entriesBySubmitTime, noResultsNeededForAggregation, noResultsNeededForAggregation_new, guid, configString);
}
public static void AggregateWithParameterAndValidateSatyamImageNetClassificationResultByGUID(string guid,
int MinResults = TaskConstants.SINGLE_OBJECT_LABLING_MTURK_MIN_RESULTS_TO_AGGREGATE,
int MaxResults = TaskConstants.SINGLE_OBJECT_LABLING_MTURK_MAX_RESULTS_TO_AGGREGATE,
double MajorityThreshold = TaskConstants.SINGLE_OBJECT_LABLING_MTURK_MAJORITY_THRESHOLD,
string confusingImageListFilePath = null)
{
string configString = "Min_" + MinResults + "_Max_" + MaxResults + "_Thresh_" + MajorityThreshold;
Console.WriteLine("Aggregating for param set " + configString);
int noTerminatedTasks = 0;
SatyamResultsTableAccess resultsDB = new SatyamResultsTableAccess();
List <SatyamResultsTableEntry> entries = resultsDB.getEntriesByGUIDOrderByID(guid);
resultsDB.close();
//SortedDictionary<DateTime, List<SatyamResultsTableEntry>> entriesBySubmitTime =
// SatyamResultValidation.SortResultsBySubmitTime_OneResultPerTurkerPerTask(entries);
SortedDictionary <DateTime, List <SatyamResultsTableEntry> > entriesBySubmitTime =
SatyamResultValidationToolKit.SortResultsBySubmitTime(entries);
Dictionary <string, ConfusingReason> imageBlackListReason = new Dictionary <string, ConfusingReason>();
if (confusingImageListFilePath != null)
{
imageBlackListReason = getConfusingImageList(confusingImageListFilePath);
}
//Dictionary<int, List<SingleObjectLabelingResult>> ResultsPerTask = new Dictionary<int, List<SingleObjectLabelingResult>>();
Dictionary <int, List <SatyamResultsTableEntry> > ResultsPerTask = new Dictionary <int, List <SatyamResultsTableEntry> >();
List <int> aggregatedTasks = new List <int>();
int noTotalConverged = 0;
int noCorrect = 0;
SortedDictionary <string, Dictionary <string, int> > confusionMatrix_res_groundtruth = new SortedDictionary <string, Dictionary <string, int> >();
SortedDictionary <string, Dictionary <string, int> > confusionMatrix_groundtruth_res = new SortedDictionary <string, Dictionary <string, int> >();
StringBuilder s = new StringBuilder();
s.Append("<!DOCTYPE html>\n");
s.Append("<html>\n");
s.Append("<body>\n");
String title = String.Format("<h1>Job GUID {0} Incorrect Result Summary</h1>\n", guid);
s.Append(title);
Dictionary <int, int> noResultsNeededForAggregation = SatyamResultsAnalysis.getNoResultsNeededForAggregationFromLog(configString, guid);
Dictionary <int, int> noResultsNeededForAggregation_new = new Dictionary <int, int>();
List <SatyamAggregatedResultsTableEntry> aggEntries = new List <SatyamAggregatedResultsTableEntry>();
foreach (DateTime t in entriesBySubmitTime.Keys)
{
foreach (SatyamResultsTableEntry entry in entriesBySubmitTime[t])
{
//SatyamResultsTableEntry entry = entries[i];
SatyamResult satyamResult = JSonUtils.ConvertJSonToObject <SatyamResult>(entry.ResultString);
SatyamTask task = JSonUtils.ConvertJSonToObject <SatyamTask>(satyamResult.TaskParametersString);
SatyamJob job = task.jobEntry;
string fileName = URIUtilities.filenameFromURI(task.SatyamURI);
string imageCategoryName = fileName.Split('_')[0];
if (IsBlackListed(task.SatyamURI, imageBlackListReason))
{
continue;
}
int taskEntryID = entry.SatyamTaskTableEntryID;
if (aggregatedTasks.Contains(taskEntryID))
{
continue;
}
if (!ResultsPerTask.ContainsKey(taskEntryID))
{
ResultsPerTask.Add(taskEntryID, new List <SatyamResultsTableEntry>());
}
//ResultEntriesPerTask[taskEntryID].Add(JSonUtils.ConvertJSonToObject<SingleObjectLabelingResult>(satyamResult.TaskResult));
ResultsPerTask[taskEntryID].Add(entry);
// check log if enough results are collected
if (noResultsNeededForAggregation != null && noResultsNeededForAggregation.ContainsKey(taskEntryID) &&
ResultsPerTask[taskEntryID].Count < noResultsNeededForAggregation[taskEntryID])
{
continue;
}
string aggResultString = SingleObjectLabelingAggregator.GetAggregatedResultString(ResultsPerTask[taskEntryID], MinResults, MaxResults, MajorityThreshold);
//SingleObjectLabelingAggregatedResult aggResult = SingleObjectLabelingAggregator.getAggregatedResult(ResultEntriesPerTask[taskEntryID], MinResults, MaxResults, MajorityThreshold);
if (aggResultString == null)
{
continue;
}
SatyamAggregatedResultsTableEntry aggEntry = new SatyamAggregatedResultsTableEntry();
aggEntry.JobGUID = ResultsPerTask[taskEntryID][0].JobGUID;
aggEntry.JobTemplateType = ResultsPerTask[taskEntryID][0].JobTemplateType;
aggEntry.SatyamTaskTableEntryID = ResultsPerTask[taskEntryID][0].SatyamTaskTableEntryID;
aggEntry.UserID = ResultsPerTask[taskEntryID][0].UserID;
aggEntry.ResultString = aggResultString;
/// aggregation happen
// record logs
if (noResultsNeededForAggregation == null || !noResultsNeededForAggregation.ContainsKey(taskEntryID))
{
noResultsNeededForAggregation_new.Add(taskEntryID, ResultsPerTask[taskEntryID].Count);
}
///
//if (aggResult.Category == "None of the Above")
//{
// continue;
//}
aggEntries.Add(aggEntry);
SatyamSaveAggregatedDataSatyam data = new SatyamSaveAggregatedDataSatyam(aggEntry);
String resultString = data.AggregatedResultString;
SingleObjectLabelingAggregatedResult aggResult = JSonUtils.ConvertJSonToObject <SingleObjectLabelingAggregatedResult>(resultString);
if (!confusionMatrix_res_groundtruth.ContainsKey(aggResult.Category))
{
confusionMatrix_res_groundtruth.Add(aggResult.Category, new Dictionary <string, int>()
{
{ GroundTruth[imageCategoryName], 0 }
});
}
else
{
if (!confusionMatrix_res_groundtruth[aggResult.Category].ContainsKey(GroundTruth[imageCategoryName]))
{
confusionMatrix_res_groundtruth[aggResult.Category].Add(GroundTruth[imageCategoryName], 0);
}
}
if (!confusionMatrix_groundtruth_res.ContainsKey(GroundTruth[imageCategoryName]))
{
confusionMatrix_groundtruth_res.Add(GroundTruth[imageCategoryName], new Dictionary <string, int>()
{
{ aggResult.Category, 0 }
});
}
else
{
if (!confusionMatrix_groundtruth_res[GroundTruth[imageCategoryName]].ContainsKey(aggResult.Category))
{
confusionMatrix_groundtruth_res[GroundTruth[imageCategoryName]].Add(aggResult.Category, 0);
}
}
if (aggResult.Category.Equals(GroundTruth[imageCategoryName], StringComparison.InvariantCultureIgnoreCase))
{
noCorrect++;
}
else
{
//Console.WriteLine("{0}, Groundtruth: {1}, Aggregated: {2}, Votes: {3}",
// fileName, GroundTruth[imageCategoryName], aggResult.Category,
// JSonUtils.ConvertObjectToJSon(aggResult.metaData));
String record = String.Format("<p>{0}, Groundtruth: {1}, Aggregated: {2}, Votes: {3}</p>\n",
fileName, GroundTruth[imageCategoryName], aggResult.Category,
JSonUtils.ConvertObjectToJSon(aggResult.metaData));
String img = String.Format("<img src=\"{0}\" >\n", task.SatyamURI);
s.Append(record);
s.Append(img);
}
noTotalConverged++;
if (ResultsPerTask[taskEntryID].Count >= MaxResults)
{
noTerminatedTasks++;
}
confusionMatrix_res_groundtruth[aggResult.Category][GroundTruth[imageCategoryName]]++;
confusionMatrix_groundtruth_res[GroundTruth[imageCategoryName]][aggResult.Category]++;
aggregatedTasks.Add(taskEntryID);
}
}
SatyamResultsAnalysis.RecordAggregationLog(noResultsNeededForAggregation_new, configString, guid);
s.Append("</body>\n");
s.Append("</html>\n");
string dataToBeSaved = s.ToString();
SatyamJobStorageAccountAccess storage = new SatyamJobStorageAccountAccess();
string FileName = String.Format("AggregatedIncorrectResults-{0}_Min{1}Max{2}Thresh{3}.html", guid, MinResults, MaxResults, MajorityThreshold);
storage.SaveATextFile("singleobjectlabeling", guid, FileName, dataToBeSaved);
s.Clear();
s.Append("<!DOCTYPE html>\n");
s.Append("<html>\n");
s.Append("<body>\n");
s.Append(title);
string resultSummary = String.Format("<p>Result: {0}/{1}, precision: {2}</p>\n", noCorrect, noTotalConverged, (double)noCorrect / noTotalConverged);
resultSummary += String.Format("<p>Terminated: {0}, Not Enough Results: {1}</p>\n", noTerminatedTasks, ResultsPerTask.Count - noTotalConverged);
Console.WriteLine(resultSummary);
s.Append(resultSummary);
// write the confusion matrix
s.Append("<p>");
String row = "\t\t";
foreach (string resultCategory in confusionMatrix_res_groundtruth.Keys)
{
row += resultCategory + "\t";
}
row += "<br>\n";
s.Append(row);
Console.WriteLine(row);
string matString = "";
foreach (string groundTruthCategory in confusionMatrix_groundtruth_res.Keys)
{
row = groundTruthCategory + "\t";
foreach (string resultCategory in confusionMatrix_res_groundtruth.Keys)
{
if (confusionMatrix_groundtruth_res[groundTruthCategory].ContainsKey(resultCategory))
{
row += confusionMatrix_groundtruth_res[groundTruthCategory][resultCategory].ToString();
}
else
{
row += "0";
}
row += "\t";
}
row += "<br>\n";
s.Append(row);
Console.WriteLine(row);
matString += row;
}
s.Append("</p>\n");
s.Append("</body>\n");
s.Append("</html>\n");
string summaryToBeSaved = s.ToString();
FileName = String.Format("Aggregated_Summary-{0}_Min{1}Max{2}Thresh{3}.html", guid, MinResults, MaxResults, MajorityThreshold);
storage.SaveATextFile("singleobjectlabeling", guid, FileName, summaryToBeSaved);
/// local file
string outputString = String.Format("{0} {1} {2} {3} {4} {5}\n", configString, noCorrect, noTotalConverged, (double)noCorrect / noTotalConverged, noTerminatedTasks, ResultsPerTask.Count - noTotalConverged);
string outputfile = DirectoryConstants.defaultTempDirectory + guid + "\\resultSummary.txt";
File.AppendAllText(outputfile, outputString);
string outputmatFile = DirectoryConstants.defaultTempDirectory + guid + "\\" + configString + "_mat.txt";
File.WriteAllText(outputmatFile, matString);
//for (double prob = 0; prob < 1;prob +=0.2)
//{
// SatyamResultsAnalysis.AnalyzeApprovalRate(aggEntries, entries, guid, configString, anotherChanceProbablity: prob);
//}
//for (double ratio = 0; ratio < 1; ratio += 0.2)
//{
// SatyamResultsAnalysis.AnalyzeApprovalRate(aggEntries, entriesBySubmitTime, noResultsNeededForAggregation, noResultsNeededForAggregation_new, guid, configString, approvalRatioThreshold: ratio);
//}
SatyamResultsAnalysis.AggregationAnalysis(aggEntries, entriesBySubmitTime, noResultsNeededForAggregation, noResultsNeededForAggregation_new, guid, configString);
}