public static void TestAggregationAnalysis()
{
string guid = "86cbfe43-175a-4a5a-ba89-073f3004e7df";//20 class mturk all pascal
//string guid = "61bf62b5-d796-4b3c-9de9-9c67f04cf26f";// changed taskpage
string configString = "Min_" + 10 + "_Max_" + 10 + "_Majority_" + 0.5 + "_Ratio_" + 0.5;
SatyamAggregatedResultsTableAccess aggDB = new SatyamAggregatedResultsTableAccess();
List <SatyamAggregatedResultsTableEntry> aggEntries = aggDB.getEntriesByGUID(guid);
aggDB.close();
SatyamResultsTableAccess resDB = new SatyamResultsTableAccess();
List <SatyamResultsTableEntry> entries = resDB.getEntriesByGUID(guid);
resDB.close();
List <SatyamResultsTableEntry> filteredResults = new List <SatyamResultsTableEntry>();
//foreach (SatyamResultsTableEntry entry in entries)
//{
// SatyamResult res = JSonUtils.ConvertJSonToObject<SatyamResult>(entry.ResultString);
// string workerID = res.amazonInfo.WorkerID;
// string assignmentID = res.amazonInfo.AssignmentID;
// if (assignmentID == "" || assignmentID == "ASSIGNMENT_ID_NOT_AVAILABLE") continue;
// if (workerID == "") continue;
// /// im seg only
// ImageSegmentationResult res_imseg = JSonUtils.ConvertJSonToObject<ImageSegmentationResult>(res.TaskResult);
// if (res_imseg == null) continue;
// filteredResults.Add(entry);
//}
filteredResults = entries;
SortedDictionary <DateTime, List <SatyamResultsTableEntry> > entriesBySubmitTime =
SatyamResultValidationToolKit.SortResultsBySubmitTime_OneResultPerTurkerPerTask(filteredResults);
SatyamResultsAnalysis.AggregationAnalysis(aggEntries, entriesBySubmitTime, null, null, guid, configString,
usePreApprovalResult: false,
approvalRatioThreshold: 0.5);
}
private void populateRows()
{
SatyamJobSubmissionsTableAccess dbaccess = new SatyamJobSubmissionsTableAccess();
List <SatyamJobSubmissionsTableAccessEntry> entries = dbaccess.getAllEntriesByUserID(UserId);
dbaccess.close();
List <TableRow> rows = new List <TableRow>();
int maxShow = 10;
for (int i = entries.Count - 1; i >= Math.Max(0, entries.Count - 1 - maxShow); i--)
{
TableRow row = new TableRow();
row.JobGUID = entries[i].JobGUID;
row.JobTemplateType = entries[i].JobTemplateType;
row.JobSubmitTime = entries[i].JobSubmitTime.ToLocalTime().ToString();
row.JobStatus = entries[i].JobStatus;
SatyamTaskTableAccess taskDB = new SatyamTaskTableAccess();
int remainingTasks = taskDB.getTasksCountByGUID(row.JobGUID);
taskDB.close();
row.TaskPending = remainingTasks.ToString();
if (entries[i].JobProgress == "")
{
row.JobProgress = "1";
}
else
{
row.JobProgress = (1 - (double)remainingTasks / Convert.ToDouble(entries[i].JobProgress)).ToString("0.000");
}
int totalResults, totalAggregated;
row.ApprovalRate = SatyamResultsAnalysis.getPaymentRateByGUID(row.JobGUID, out totalResults, out totalAggregated).ToString("0.000");
row.TotalResults = totalResults.ToString();
row.TotalAggregated = totalAggregated.ToString();
rows.Add(row);
}
rpt.DataSource = rows;
rpt.DataBind();
}
public static void AggregateWithParameterAndValidateSatyamVideoClassificationResultByGUID(string guid,
int MinResults = TaskConstants.SINGLE_OBJECT_LABLING_MTURK_MIN_RESULTS_TO_AGGREGATE,
int MaxResults = TaskConstants.SINGLE_OBJECT_LABLING_MTURK_MAX_RESULTS_TO_AGGREGATE,
double MajorityThreshold = TaskConstants.SINGLE_OBJECT_LABLING_MTURK_MAJORITY_THRESHOLD,
bool prepareDataForTraining = false, string outputDirectory = null,
string confusingVideoListFilePath = null)
{
string configString = "Min_" + MinResults + "_Max_" + MaxResults + "_Thresh_" + MajorityThreshold;
Console.WriteLine("Aggregating for param set " + configString);
int noTerminatedTasks = 0;
SatyamResultsTableAccess resultsDB = new SatyamResultsTableAccess();
List <SatyamResultsTableEntry> entries = resultsDB.getEntriesByGUIDOrderByID(guid);
resultsDB.close();
SortedDictionary <DateTime, List <SatyamResultsTableEntry> > entriesBySubmitTime = SatyamResultValidationToolKit.SortResultsBySubmitTime(entries);
int noTotalConverged = 0;
int noCorrect = 0;
Dictionary <int, List <SatyamResultsTableEntry> > ResultsPerTask = new Dictionary <int, List <SatyamResultsTableEntry> >();
List <int> aggregatedTasks = new List <int>();
SortedDictionary <string, Dictionary <string, int> > confusionMatrix_res_groundtruth = new SortedDictionary <string, Dictionary <string, int> >();
SortedDictionary <string, Dictionary <string, int> > confusionMatrix_groundtruth_res = new SortedDictionary <string, Dictionary <string, int> >();
List <SatyamAggregatedResultsTableEntry> aggEntries = new List <SatyamAggregatedResultsTableEntry>();
Dictionary <int, int> noResultsNeededForAggregation = SatyamResultsAnalysis.getNoResultsNeededForAggregationFromLog(configString, guid);
Dictionary <int, int> noResultsNeededForAggregation_new = new Dictionary <int, int>();
foreach (DateTime t in entriesBySubmitTime.Keys)
{
//Console.WriteLine("Processing Results of time: {0}", t);
foreach (SatyamResultsTableEntry entry in entriesBySubmitTime[t])
{
SatyamResult satyamResult = JSonUtils.ConvertJSonToObject <SatyamResult>(entry.ResultString);
SatyamTask task = JSonUtils.ConvertJSonToObject <SatyamTask>(satyamResult.TaskParametersString);
SatyamJob job = task.jobEntry;
string fileName = URIUtilities.filenameFromURI(task.SatyamURI);
string VideoCategoryGroundTruth = getVideoCategoryFromFileName(fileName);
int taskEntryID = entry.SatyamTaskTableEntryID;
if (aggregatedTasks.Contains(taskEntryID))
{
continue;
}
if (!ResultsPerTask.ContainsKey(taskEntryID))
{
ResultsPerTask.Add(taskEntryID, new List <SatyamResultsTableEntry>());
}
ResultsPerTask[taskEntryID].Add(entry);
// check log if enough results are collected
if (noResultsNeededForAggregation != null && noResultsNeededForAggregation.ContainsKey(taskEntryID) &&
ResultsPerTask[taskEntryID].Count < noResultsNeededForAggregation[taskEntryID])
{
continue;
}
//SingleObjectLabelingAggregatedResult aggResult = SingleObjectLabelingAggregator.getAggregatedResult(ResultsPerTask[taskEntryID], MinResults, MaxResult, MajorityThreshold);
string aggResultString = SingleObjectLabelingAggregator.GetAggregatedResultString(ResultsPerTask[taskEntryID], MinResults, MaxResults, MajorityThreshold);
if (aggResultString == null)
{
continue;
}
SatyamAggregatedResultsTableEntry aggEntry = new SatyamAggregatedResultsTableEntry();
aggEntry.JobGUID = ResultsPerTask[taskEntryID][0].JobGUID;
aggEntry.JobTemplateType = ResultsPerTask[taskEntryID][0].JobTemplateType;
aggEntry.SatyamTaskTableEntryID = taskEntryID;
aggEntry.UserID = ResultsPerTask[taskEntryID][0].UserID;
aggEntry.ResultString = aggResultString;
/// aggregation happen
// record logs
if (noResultsNeededForAggregation == null || !noResultsNeededForAggregation.ContainsKey(taskEntryID))
{
noResultsNeededForAggregation_new.Add(taskEntryID, ResultsPerTask[taskEntryID].Count);
}
aggEntries.Add(aggEntry);
noTotalConverged++;
if (ResultsPerTask[taskEntryID].Count >= MaxResults)
{
noTerminatedTasks++;
}
aggregatedTasks.Add(taskEntryID);
}
}
SatyamResultsAnalysis.RecordAggregationLog(noResultsNeededForAggregation_new, configString, guid);
string outputmatFile = DirectoryConstants.defaultTempDirectory + guid + "\\" + configString + "_mat.txt";
EvaluateAndPrintConfusionMatrixOfAggregatedResultEntries(aggEntries, outputmatFile, out noCorrect, prepareDataForTraining, outputDirectory);
Console.WriteLine("noTotalConverged {0}", noTotalConverged);
Console.WriteLine("noTerminatedTasks {0}", noTerminatedTasks);
Console.WriteLine("Result: {0}/{1}, precision: {2}", noCorrect, noTotalConverged, (double)noCorrect / noTotalConverged);
/// local file
string outputString = String.Format("{0} {1} {2} {3} {4} {5}\n", configString, noCorrect, noTotalConverged, (double)noCorrect / noTotalConverged, noTerminatedTasks, ResultsPerTask.Count - noTotalConverged);
string outputfile = DirectoryConstants.defaultTempDirectory + guid + "\\resultSummary.txt";
File.AppendAllText(outputfile, outputString);
//for (double prob = 0; prob < 1;prob +=0.2)
//{
// SatyamResultsAnalysis.AnalyzeApprovalRate(aggEntries, entries, guid, configString, anotherChanceProbablity: prob);
//}
//for (double ratio = 0; ratio < 1; ratio += 0.2)
//{
// SatyamResultsAnalysis.AnalyzeApprovalRate(aggEntries, entriesBySubmitTime, noResultsNeededForAggregation, noResultsNeededForAggregation_new, guid, configString, approvalRatioThreshold: ratio);
//}
SatyamResultsAnalysis.AggregationAnalysis(aggEntries, entriesBySubmitTime, noResultsNeededForAggregation, noResultsNeededForAggregation_new, guid, configString);
}
public static void TestSatyamResultAnalysis()
{
List <List <string> > jobs = new List <List <string> >()
{
//// Video Classification
//new List<string>(){ "ca2a97e3-3c32-48cc-b7d9-fdd945b50f23" }, //10 class human gesture in JHMDB
//// Image Classification
//new List<string>(){ "9c6283dd-f218-4588-a0d9-8c0afe841503" }, // all img 10class
//new List<string>(){ "03977c5f-0e87-49ab-9c49-f20bfb47726a" }, // 2img 10 class
//// Image Counting
//new List<string>(){
// "94979929-4315-4c81-9782-55c8955222a8",// CARPK NTU 20161029
// "58a85bc8-4fb9-4672-8719-38cf62352fbc", // KITTI all
//},
//// Image Detection
new List <string>()
{
//"3609af9c-e734-41dc-997c-94c060a1f63d",
//"5f14ecaa-1c4c-41d6-ad05-6ee0929e494e",
//"8fbc6ca1-e6c0-4720-93a3-1d49aa873a65",
//"d6f1c3ee-cd59-4197-b40a-0f45512596eb", // KITTI Detection all
//"a7e3fe3a-5097-4149-99bc-bae06da18aba", // rivision tasks satyam
//"aefd12fc-42ed-4ec5-8c9a-6c41703a843f", // revision task kitti
//"2c239102-e38f-4f0e-b639-6fae05250475", // without revision task kitti
//"2b1a86ee-7d39-49db-90bc-fb8d2ff63019", // revision kitti 10 internal test with recordings
//"a328f600-3b1f-4845-87a0-402648ad51b1", // revision Seattle 10, TF Serving backend
//"5c403d7f-6c48-4638-b545-472b9f190606", // no revision Seattle 10
//"40eee69b-c260-4760-940f-92f30cdfbb7f", // RevisionTF Seattle9000 on TestStewart 10
//"a55c9c21-16c0-432a-930e-de428d8213a5", // RevisionTF Fastrcnn_KITTI on TestStewart 10
//"9e3096f6-e898-45eb-8b15-e2854485a59c", // RevisionTF Fastrcnn_KITTI on TestStewart 30, but aggregation is not functioning for most of the time
"3e835157-d058-46ee-a13b-9c1566cba505", //RevisionTF Seattle9000 on TestStewart 30
},
////// Video Tracking
//new List<string>(){
// "1e43a983-548d-4a2e-8161-5537eb985902",
// "77c4f6af-f669-42fb-925a-4ff670cf9ae2"}, // KITTI Tracking all
////// Caprk Counting Adaptive Pricing Test
//new List<string>(){
// //"f7c498d8-c06e-4b46-9169-71048c0e4eac",// carpk 10
// "e5fdff10-f018-4779-9d18-70c25fa3259f",}, // carpk all
//new List<string>(){
// "a388df30-8742-4b6e-911d-32093502c2e8"}, // KITTI all
//////// Image Segmentation
//new List<string>()
//{
// "86cbfe43-175a-4a5a-ba89-073f3004e7df", // pascal VOC
//},
////// Cam Pose Estimation
//new List<string>()
//{
// "dff5415b-b278-43db-8e32-4f654ca4b3a1", //3000 results
//}
};
foreach (List <string> guids in jobs)
{
SatyamResultsAnalysis.AnalyzeFromResultsTable(guids);
}
}
public static void AdaptTaskParameters()
{
double TargetPricePerHour = -1;
double Budget = -1;
//double TargetPricePerHour = 13.56;
//double MinPricePerTask = 0.01;
//double MaxPricePerTask = 0.5;
double MinPricePerTaskDifferenceToAdjust = 0.01;//USD
int MinNoResultsToAdjust = 10;
Dictionary <string, List <SatyamTaskTableEntry> > MTurkTasksByGUID = OrganizeMTurkTasksByGUID();
if (MTurkTasksByGUID == null)
{
return;
}
// get statistics
foreach (string guid in MTurkTasksByGUID.Keys)
{
// testing for counting only
List <string> ACGuids = new List <string>()
{
"e5fdff10-f018-4779-9d18-70c25fa3259f", //carpk 164
"a388df30-8742-4b6e-911d-32093502c2e8", //kitti 100
};
if (!ACGuids.Contains(guid))
{
continue;
}
SatyamTaskTableEntry task0 = MTurkTasksByGUID[guid][0];
double CurrentPricePerTask = task0.PricePerTask;
SatyamTask t = JSonUtils.ConvertJSonToObject <SatyamTask>(task0.TaskParametersString);
TargetPricePerHour = t.jobEntry.TargetPricePerTask;
//TargetPricePerHour = 13.56; //temporary
Budget = t.jobEntry.TotalBudget;
if (TargetPricePerHour <= 0)
{
continue; //invalid
}
SatyamResultPerJobDataAnalysis ana = SatyamResultsAnalysis.AnalyzeFromResultsTable(new List <string>()
{
guid
});
if (ana == null)
{
continue;
}
int noAcceptedResults = ana.totalAcceptedResults;
if (noAcceptedResults < MinNoResultsToAdjust)
{
continue;
}
double medianTimePerTaskInSeconds = ana.medianAcceptedTimeTakenPerResultInSeconds;
double TargetPricePerTask = TargetPricePerHour / 3600 * medianTimePerTaskInSeconds;
//TargetPricePerTask = Math.Max(TargetPricePerTask, MinPricePerTask);
//TargetPricePerTask = Math.Min(TargetPricePerTask, MaxPricePerTask);
TargetPricePerTask = Math.Round(TargetPricePerTask, 2);// round to cents.
// is the price within the budget?
double MinBudgetNeeded = MTurkTasksByGUID[guid].Count * TargetPricePerTask * 3;// at least 3 results per task.
if (Budget < MinBudgetNeeded)
{
Console.WriteLine("Budget too small to adapt price");
continue;
}
if (Math.Abs(CurrentPricePerTask - TargetPricePerTask) > MinPricePerTaskDifferenceToAdjust)
{
// need to adjust
Console.WriteLine("Adapting the price to {0}", TargetPricePerTask);
ExpireHitByGUID(guid);
AdjustTasksByGUID(guid, TargetPricePerTask);
}
}
}
public static void AggregateWithParameter(string guid,
int MinResults = TaskConstants.TRACKLET_LABELING_MTURK_MIN_RESULTS_TO_AGGREGATE,
int MaxResults = TaskConstants.TRACKLET_LABELING_MTURK_MAX_RESULTS_TO_AGGREGATE,
double boxToleranceThreshold = TaskConstants.TRACKLET_LABELING_BOX_DEVIATION_THRESHOLD,
double ObjectCoverageApprovalThresholdPerVideo = TaskConstants.TRACKLET_LABELING_APPROVALRATIO_PER_VIDEO,
double BoxCoverageApprovalThresholdPerTrack = TaskConstants.TRACKLET_LABELING_APPROVALRATIO_PER_TRACK,
int consensusNumber = TaskConstants.TRACKLET_LABELING_MIN_RESULTS_FOR_CONSENSUS,
double minTubeletIoUSimilarityThreshold = TaskConstants.TRACKLET_LABELING_MIN_TUBELET_SIMILARITY_THRESHOLD,
double attributeMajority = TaskConstants.TRACKLET_LABELING_MTURK_ATTRIBUTE_MAJORITY_THRESHOLD,
bool allFalseAttributeInvalid = false
)
{
string configString = "Min_" + MinResults + "_Max_" + MaxResults + "_IoU_" + minTubeletIoUSimilarityThreshold + "_Ratio_" + ObjectCoverageApprovalThresholdPerVideo;
Console.WriteLine("Aggregating for " + guid + " with param set " + configString);
SatyamResultsTableAccess resultsDB = new SatyamResultsTableAccess();
List <SatyamResultsTableEntry> entries = resultsDB.getEntriesByGUID(guid);
resultsDB.close();
SortedDictionary <DateTime, List <SatyamResultsTableEntry> > entriesBySubmitTime = SatyamResultValidationToolKit.SortResultsBySubmitTime_OneResultPerTurkerPerTask(entries);
Dictionary <int, List <MultiObjectTrackingResult> > ResultsPerTask = new Dictionary <int, List <MultiObjectTrackingResult> >();
List <int> aggregatedTasks = new List <int>();
int noTotalConverged = 0;
//int noCorrect = 0;
int noTerminatedTasks = 0;
List <SatyamAggregatedResultsTableEntry> aggEntries = new List <SatyamAggregatedResultsTableEntry>();
Dictionary <int, int> noResultsNeededForAggregation = SatyamResultsAnalysis.getNoResultsNeededForAggregationFromLog(configString, guid);
Dictionary <int, int> noResultsNeededForAggregation_new = new Dictionary <int, int>();
// play back by time
Dictionary <int, List <string> > WorkersPerTask = new Dictionary <int, List <string> >();
foreach (DateTime t in entriesBySubmitTime.Keys)
{
//Console.WriteLine("Processing Results of time: {0}", t);
List <SatyamResultsTableEntry> ResultEntries = entriesBySubmitTime[t];
foreach (SatyamResultsTableEntry entry in ResultEntries)
{
SatyamResult satyamResult = JSonUtils.ConvertJSonToObject <SatyamResult>(entry.ResultString);
SatyamTask task = JSonUtils.ConvertJSonToObject <SatyamTask>(satyamResult.TaskParametersString);
MultiObjectTrackingSubmittedJob job = JSonUtils.ConvertJSonToObject <MultiObjectTrackingSubmittedJob>(task.jobEntry.JobParameters);
string fileName = URIUtilities.filenameFromURINoExtension(task.SatyamURI);
int taskEntryID = entry.SatyamTaskTableEntryID;
if (aggregatedTasks.Contains(taskEntryID))
{
continue;
}
if (!ResultsPerTask.ContainsKey(taskEntryID))
{
ResultsPerTask.Add(taskEntryID, new List <MultiObjectTrackingResult>());
WorkersPerTask.Add(taskEntryID, new List <string>());
}
// remove duplicate workers result
string workerID = satyamResult.amazonInfo.WorkerID;
if (WorkersPerTask[taskEntryID].Contains(workerID))
{
continue;
}
//enclose only non-duplicate results, one per each worker.
WorkersPerTask[taskEntryID].Add(workerID);
string videoName = URIUtilities.localDirectoryNameFromURI(task.SatyamURI);
string[] fields = videoName.Split('_');
int startingFrame = Convert.ToInt32(fields[fields.Length - 1]);
int maxChunkEndFrame = startingFrame + job.ChunkDuration * job.FrameRate;
int noFrameOverlap = (int)(job.ChunkOverlap * job.FrameRate);
if (startingFrame != 0)
{
startingFrame -= noFrameOverlap;
}
string blobDir = URIUtilities.localDirectoryFullPathFromURI(task.SatyamURI);
VATIC_DVA_CrowdsourcedResult taskr = TrackletLabelingAggregator.createVATIC_DVA_CrowdsourcedResultUsingSatyamBlobImageCount(satyamResult.TaskResult, blobDir, entry.SatyamTaskTableEntryID.ToString(), job.FrameRate);
//MultiObjectTrackingResult result_tmp = vatic_tmp.getCompressedTracksInTimeSegment();
//VATIC_DVA_CrowdsourcedResult taskr = new VATIC_DVA_CrowdsourcedResult(satyamResult.TaskResult, videoName, entry.ID.ToString(), ImageURLs.Count, job.FrameRate);
MultiObjectTrackingResult res = taskr.getCompressedTracksInTimeSegment();
if (allFalseAttributeInvalid)
{
if (TrackletLabelingAggregator.AllAttributeAllFalse(res))
{
continue;
}
}
ResultsPerTask[taskEntryID].Add(res);
// check log if enough results are collected
if (noResultsNeededForAggregation != null && noResultsNeededForAggregation.ContainsKey(taskEntryID) &&
ResultsPerTask[taskEntryID].Count < noResultsNeededForAggregation[taskEntryID])
{
continue;
}
// hack for masters
int tempMin = MinResults;
if (TaskConstants.masterGUIDs.Contains(entry.JobGUID))
{
tempMin = 1;
}
TrackletLabelingAggregatedResult aggResult = TrackletLabelingAggregator.getAggregatedResult(ResultsPerTask[taskEntryID],
guid, task.SatyamURI,
job.FrameRate, job.BoundaryLines,
MinResults, MaxResults, boxToleranceThreshold,
ObjectCoverageApprovalThresholdPerVideo,
BoxCoverageApprovalThresholdPerTrack,
consensusNumber, minTubeletIoUSimilarityThreshold,
attributeMajority);
if (aggResult == null)
{
continue;
}
//////////////// aggregation happen
// record logs
if (noResultsNeededForAggregation == null || !noResultsNeededForAggregation.ContainsKey(taskEntryID))
{
noResultsNeededForAggregation_new.Add(taskEntryID, ResultsPerTask[taskEntryID].Count);
}
aggregatedTasks.Add(taskEntryID);
noTotalConverged++;
if (ResultsPerTask[taskEntryID].Count >= MaxResults)
{
noTerminatedTasks++;
}
SatyamAggregatedResult SatyamAggResult = new SatyamAggregatedResult();
SatyamAggResult.SatyamTaskTableEntryID = taskEntryID;
SatyamAggResult.AggregatedResultString = JSonUtils.ConvertObjectToJSon <TrackletLabelingAggregatedResult>(aggResult);
SatyamAggResult.TaskParameters = JSonUtils.ConvertJSonToObject <SatyamResult>(entry.ResultString).TaskParametersString;
SatyamAggregatedResultsTableEntry aggEntry = new SatyamAggregatedResultsTableEntry();
aggEntry.SatyamTaskTableEntryID = taskEntryID;
aggEntry.JobGUID = entry.JobGUID;
aggEntry.ResultString = JSonUtils.ConvertObjectToJSon <SatyamAggregatedResult>(SatyamAggResult);
aggEntries.Add(aggEntry);
}
}
Console.WriteLine("Total_Aggregated_Tasks: {0}", noTotalConverged);
Console.WriteLine("Total_Terminated_Tasks: {0}", noTerminatedTasks);
SatyamResultsAnalysis.RecordAggregationLog(noResultsNeededForAggregation_new, configString, guid);
TrackletLabelingAnalyzer.GroupEntriesByVideoNameAndStitchAndSaveAggregatedResultVideosLocally(aggEntries, DirectoryConstants.defaultTempDirectory + guid);
}
public static void AggregateWithParameterAndValidateObjectCountingResultByGUID(string guid,
//bool saveImage = false,
bool prepareTrainingSet = false, string outputDirectory = null,
int MinHeight = TaskConstants.OBJECT_COUNTING_VALIDATION_MIN_HEIGHT,
int MaxOcclusion = TaskConstants.OBJECT_COUNTING_VALIDATION_MAX_OCCLUSION,
double Max_Truncation = TaskConstants.OBJECT_COUNTING_VALIDATION_MIN_TRUNCATION,
int MinResults = TaskConstants.OBJECT_COUNTING_MTURK_MIN_RESULTS_TO_AGGREGATE,
int MaxResults = TaskConstants.OBJECT_COUNTING_MTURK_MAX_RESULTS_TO_AGGREGATE,
double MAX_ABSOLUTE_COUNT_DEVIATION_LOWERBOUND = TaskConstants.OBJECT_COUNTING_MTURK_MAX_ABSOLUTE_COUNT_DEVIATION_LOWERBOUND,
double MAX_DEVIATION_FRACTION = TaskConstants.OBJECT_COUNTING_MTURK_MAX_DEVIATION_FRACTION,
double SUPER_MAJORITY_VALUE = TaskConstants.OBJECT_COUNTING_MTURK_SUPER_MAJORITY_VALUE
)
{
if (!Directory.Exists(DirectoryConstants.defaultTempDirectory + guid))
{
Directory.CreateDirectory(DirectoryConstants.defaultTempDirectory + guid);
}
string configString = "Min_" + MinResults + "_Max_" + MaxResults + "_Dev_" + MAX_DEVIATION_FRACTION + "_Majority_" + SUPER_MAJORITY_VALUE;
Console.WriteLine("Aggregating for param set " + configString);
int noTerminatedTasks = 0;
SatyamResultsTableAccess resultsDB = new SatyamResultsTableAccess();
List <SatyamResultsTableEntry> entries = resultsDB.getEntriesByGUIDOrderByID(guid);
resultsDB.close();
SortedDictionary <DateTime, List <SatyamResultsTableEntry> > entriesBySubmitTime = SatyamResultValidationToolKit.SortResultsBySubmitTime(entries);
int noTotalConverged = 0;
//int noCorrect = 0;
Dictionary <int, List <SatyamResultsTableEntry> > ResultsPerTask = new Dictionary <int, List <SatyamResultsTableEntry> >();
List <int> aggregatedTasks = new List <int>();
List <SatyamAggregatedResultsTableEntry> aggEntries = new List <SatyamAggregatedResultsTableEntry>();
Dictionary <int, int> noResultsNeededForAggregation = SatyamResultsAnalysis.getNoResultsNeededForAggregationFromLog(configString, guid);
Dictionary <int, int> noResultsNeededForAggregation_new = new Dictionary <int, int>();
// play back by time
foreach (DateTime t in entriesBySubmitTime.Keys)
{
//Console.WriteLine("Processing Results of time: {0}", t);
foreach (SatyamResultsTableEntry entry in entriesBySubmitTime[t])
{
SatyamResult satyamResult = JSonUtils.ConvertJSonToObject <SatyamResult>(entry.ResultString);
SatyamTask task = JSonUtils.ConvertJSonToObject <SatyamTask>(satyamResult.TaskParametersString);
SatyamJob job = task.jobEntry;
string fileName = URIUtilities.filenameFromURI(task.SatyamURI);
int taskEntryID = entry.SatyamTaskTableEntryID;
if (aggregatedTasks.Contains(taskEntryID))
{
continue;
}
if (!ResultsPerTask.ContainsKey(taskEntryID))
{
ResultsPerTask.Add(taskEntryID, new List <SatyamResultsTableEntry>());
}
ResultsPerTask[taskEntryID].Add(entry);
// check log if enough results are collected
if (noResultsNeededForAggregation != null && noResultsNeededForAggregation.ContainsKey(taskEntryID) &&
ResultsPerTask[taskEntryID].Count < noResultsNeededForAggregation[taskEntryID])
{
continue;
}
string aggResultString = ObjectCountingAggregator.GetAggregatedResultString(ResultsPerTask[taskEntryID], MinResults, MaxResults, MAX_ABSOLUTE_COUNT_DEVIATION_LOWERBOUND, MAX_DEVIATION_FRACTION, SUPER_MAJORITY_VALUE);
if (aggResultString == null)
{
continue;
}
SatyamAggregatedResultsTableEntry aggEntry = new SatyamAggregatedResultsTableEntry();
aggEntry.JobGUID = ResultsPerTask[taskEntryID][0].JobGUID;
aggEntry.JobTemplateType = ResultsPerTask[taskEntryID][0].JobTemplateType;
aggEntry.SatyamTaskTableEntryID = taskEntryID;
aggEntry.UserID = ResultsPerTask[taskEntryID][0].UserID;
aggEntry.ResultString = aggResultString;
/// aggregation happen
/// // record logs
// record logs
if (noResultsNeededForAggregation == null || !noResultsNeededForAggregation.ContainsKey(taskEntryID))
{
noResultsNeededForAggregation_new.Add(taskEntryID, ResultsPerTask[taskEntryID].Count);
}
aggEntries.Add(aggEntry);
noTotalConverged++;
if (ResultsPerTask[taskEntryID].Count >= MaxResults)
{
noTerminatedTasks++;
}
aggregatedTasks.Add(taskEntryID);
}
}
SatyamResultsAnalysis.RecordAggregationLog(noResultsNeededForAggregation_new, configString, guid);
double totalError, totalGroundtruth;
//ValidateSatyamKITTIObjectCountingAggregationResult(aggEntries, out totalError, out totalGroundtruth,
// MinHeight, MaxOcclusion, Max_Truncation);
ValidateSatyamCARPKObjectCountingAggregationResult(aggEntries, out totalError, out totalGroundtruth);
Console.WriteLine("noTotalConverged {0}", noTotalConverged);
Console.WriteLine("noTerminatedTasks {0}", noTerminatedTasks);
/// local file
string outputString = String.Format("{0} {1} {2} {3} {4} {5} {6}\n", configString, totalError, totalGroundtruth, totalError / totalGroundtruth, noTotalConverged, noTerminatedTasks, ResultsPerTask.Count - noTotalConverged);
string outputfile = DirectoryConstants.defaultTempDirectory + guid + "\\resultSummary.txt";
File.AppendAllText(outputfile, outputString);
string approvalString = configString + "_PayDev_" + TaskConstants.OBJECT_COUNTING_MTURK_MAX_ABSOLUTE_COUNT_DEVIATION_FOR_PAYMENT +
"_PayFrac_" + TaskConstants.OBJECT_COUNTING_MTURK_MAX_DEVIATION_FRACTION_FOR_PAYMENT;
//for (double prob = 0; prob < 1;prob +=0.2)
//{
// SatyamResultsAnalysis.AnalyzeApprovalRate(aggEntries, entries, guid, approvalString, anotherChanceProbablity: prob);
//}
//for (double ratio = 0; ratio < 1; ratio += 0.2)
//{
// SatyamResultsAnalysis.AnalyzeApprovalRate(aggEntries, entriesBySubmitTime, noResultsNeededForAggregation, noResultsNeededForAggregation_new, guid, configString, approvalRatioThreshold: ratio);
//}
SatyamResultsAnalysis.AggregationAnalysis(aggEntries, entriesBySubmitTime, noResultsNeededForAggregation, noResultsNeededForAggregation_new, guid, configString);
}
public static void AggregateWithParameterAndValidateSatyamImageNetClassificationResultByGUID(string guid,
int MinResults = TaskConstants.SINGLE_OBJECT_LABLING_MTURK_MIN_RESULTS_TO_AGGREGATE,
int MaxResults = TaskConstants.SINGLE_OBJECT_LABLING_MTURK_MAX_RESULTS_TO_AGGREGATE,
double MajorityThreshold = TaskConstants.SINGLE_OBJECT_LABLING_MTURK_MAJORITY_THRESHOLD,
string confusingImageListFilePath = null)
{
string configString = "Min_" + MinResults + "_Max_" + MaxResults + "_Thresh_" + MajorityThreshold;
Console.WriteLine("Aggregating for param set " + configString);
int noTerminatedTasks = 0;
SatyamResultsTableAccess resultsDB = new SatyamResultsTableAccess();
List <SatyamResultsTableEntry> entries = resultsDB.getEntriesByGUIDOrderByID(guid);
resultsDB.close();
//SortedDictionary<DateTime, List<SatyamResultsTableEntry>> entriesBySubmitTime =
// SatyamResultValidation.SortResultsBySubmitTime_OneResultPerTurkerPerTask(entries);
SortedDictionary <DateTime, List <SatyamResultsTableEntry> > entriesBySubmitTime =
SatyamResultValidationToolKit.SortResultsBySubmitTime(entries);
Dictionary <string, ConfusingReason> imageBlackListReason = new Dictionary <string, ConfusingReason>();
if (confusingImageListFilePath != null)
{
imageBlackListReason = getConfusingImageList(confusingImageListFilePath);
}
//Dictionary<int, List<SingleObjectLabelingResult>> ResultsPerTask = new Dictionary<int, List<SingleObjectLabelingResult>>();
Dictionary <int, List <SatyamResultsTableEntry> > ResultsPerTask = new Dictionary <int, List <SatyamResultsTableEntry> >();
List <int> aggregatedTasks = new List <int>();
int noTotalConverged = 0;
int noCorrect = 0;
SortedDictionary <string, Dictionary <string, int> > confusionMatrix_res_groundtruth = new SortedDictionary <string, Dictionary <string, int> >();
SortedDictionary <string, Dictionary <string, int> > confusionMatrix_groundtruth_res = new SortedDictionary <string, Dictionary <string, int> >();
StringBuilder s = new StringBuilder();
s.Append("<!DOCTYPE html>\n");
s.Append("<html>\n");
s.Append("<body>\n");
String title = String.Format("<h1>Job GUID {0} Incorrect Result Summary</h1>\n", guid);
s.Append(title);
Dictionary <int, int> noResultsNeededForAggregation = SatyamResultsAnalysis.getNoResultsNeededForAggregationFromLog(configString, guid);
Dictionary <int, int> noResultsNeededForAggregation_new = new Dictionary <int, int>();
List <SatyamAggregatedResultsTableEntry> aggEntries = new List <SatyamAggregatedResultsTableEntry>();
foreach (DateTime t in entriesBySubmitTime.Keys)
{
foreach (SatyamResultsTableEntry entry in entriesBySubmitTime[t])
{
//SatyamResultsTableEntry entry = entries[i];
SatyamResult satyamResult = JSonUtils.ConvertJSonToObject <SatyamResult>(entry.ResultString);
SatyamTask task = JSonUtils.ConvertJSonToObject <SatyamTask>(satyamResult.TaskParametersString);
SatyamJob job = task.jobEntry;
string fileName = URIUtilities.filenameFromURI(task.SatyamURI);
string imageCategoryName = fileName.Split('_')[0];
if (IsBlackListed(task.SatyamURI, imageBlackListReason))
{
continue;
}
int taskEntryID = entry.SatyamTaskTableEntryID;
if (aggregatedTasks.Contains(taskEntryID))
{
continue;
}
if (!ResultsPerTask.ContainsKey(taskEntryID))
{
ResultsPerTask.Add(taskEntryID, new List <SatyamResultsTableEntry>());
}
//ResultEntriesPerTask[taskEntryID].Add(JSonUtils.ConvertJSonToObject<SingleObjectLabelingResult>(satyamResult.TaskResult));
ResultsPerTask[taskEntryID].Add(entry);
// check log if enough results are collected
if (noResultsNeededForAggregation != null && noResultsNeededForAggregation.ContainsKey(taskEntryID) &&
ResultsPerTask[taskEntryID].Count < noResultsNeededForAggregation[taskEntryID])
{
continue;
}
string aggResultString = SingleObjectLabelingAggregator.GetAggregatedResultString(ResultsPerTask[taskEntryID], MinResults, MaxResults, MajorityThreshold);
//SingleObjectLabelingAggregatedResult aggResult = SingleObjectLabelingAggregator.getAggregatedResult(ResultEntriesPerTask[taskEntryID], MinResults, MaxResults, MajorityThreshold);
if (aggResultString == null)
{
continue;
}
SatyamAggregatedResultsTableEntry aggEntry = new SatyamAggregatedResultsTableEntry();
aggEntry.JobGUID = ResultsPerTask[taskEntryID][0].JobGUID;
aggEntry.JobTemplateType = ResultsPerTask[taskEntryID][0].JobTemplateType;
aggEntry.SatyamTaskTableEntryID = ResultsPerTask[taskEntryID][0].SatyamTaskTableEntryID;
aggEntry.UserID = ResultsPerTask[taskEntryID][0].UserID;
aggEntry.ResultString = aggResultString;
/// aggregation happen
// record logs
if (noResultsNeededForAggregation == null || !noResultsNeededForAggregation.ContainsKey(taskEntryID))
{
noResultsNeededForAggregation_new.Add(taskEntryID, ResultsPerTask[taskEntryID].Count);
}
///
//if (aggResult.Category == "None of the Above")
//{
// continue;
//}
aggEntries.Add(aggEntry);
SatyamSaveAggregatedDataSatyam data = new SatyamSaveAggregatedDataSatyam(aggEntry);
String resultString = data.AggregatedResultString;
SingleObjectLabelingAggregatedResult aggResult = JSonUtils.ConvertJSonToObject <SingleObjectLabelingAggregatedResult>(resultString);
if (!confusionMatrix_res_groundtruth.ContainsKey(aggResult.Category))
{
confusionMatrix_res_groundtruth.Add(aggResult.Category, new Dictionary <string, int>()
{
{ GroundTruth[imageCategoryName], 0 }
});
}
else
{
if (!confusionMatrix_res_groundtruth[aggResult.Category].ContainsKey(GroundTruth[imageCategoryName]))
{
confusionMatrix_res_groundtruth[aggResult.Category].Add(GroundTruth[imageCategoryName], 0);
}
}
if (!confusionMatrix_groundtruth_res.ContainsKey(GroundTruth[imageCategoryName]))
{
confusionMatrix_groundtruth_res.Add(GroundTruth[imageCategoryName], new Dictionary <string, int>()
{
{ aggResult.Category, 0 }
});
}
else
{
if (!confusionMatrix_groundtruth_res[GroundTruth[imageCategoryName]].ContainsKey(aggResult.Category))
{
confusionMatrix_groundtruth_res[GroundTruth[imageCategoryName]].Add(aggResult.Category, 0);
}
}
if (aggResult.Category.Equals(GroundTruth[imageCategoryName], StringComparison.InvariantCultureIgnoreCase))
{
noCorrect++;
}
else
{
//Console.WriteLine("{0}, Groundtruth: {1}, Aggregated: {2}, Votes: {3}",
// fileName, GroundTruth[imageCategoryName], aggResult.Category,
// JSonUtils.ConvertObjectToJSon(aggResult.metaData));
String record = String.Format("<p>{0}, Groundtruth: {1}, Aggregated: {2}, Votes: {3}</p>\n",
fileName, GroundTruth[imageCategoryName], aggResult.Category,
JSonUtils.ConvertObjectToJSon(aggResult.metaData));
String img = String.Format("<img src=\"{0}\" >\n", task.SatyamURI);
s.Append(record);
s.Append(img);
}
noTotalConverged++;
if (ResultsPerTask[taskEntryID].Count >= MaxResults)
{
noTerminatedTasks++;
}
confusionMatrix_res_groundtruth[aggResult.Category][GroundTruth[imageCategoryName]]++;
confusionMatrix_groundtruth_res[GroundTruth[imageCategoryName]][aggResult.Category]++;
aggregatedTasks.Add(taskEntryID);
}
}
SatyamResultsAnalysis.RecordAggregationLog(noResultsNeededForAggregation_new, configString, guid);
s.Append("</body>\n");
s.Append("</html>\n");
string dataToBeSaved = s.ToString();
SatyamJobStorageAccountAccess storage = new SatyamJobStorageAccountAccess();
string FileName = String.Format("AggregatedIncorrectResults-{0}_Min{1}Max{2}Thresh{3}.html", guid, MinResults, MaxResults, MajorityThreshold);
storage.SaveATextFile("singleobjectlabeling", guid, FileName, dataToBeSaved);
s.Clear();
s.Append("<!DOCTYPE html>\n");
s.Append("<html>\n");
s.Append("<body>\n");
s.Append(title);
string resultSummary = String.Format("<p>Result: {0}/{1}, precision: {2}</p>\n", noCorrect, noTotalConverged, (double)noCorrect / noTotalConverged);
resultSummary += String.Format("<p>Terminated: {0}, Not Enough Results: {1}</p>\n", noTerminatedTasks, ResultsPerTask.Count - noTotalConverged);
Console.WriteLine(resultSummary);
s.Append(resultSummary);
// write the confusion matrix
s.Append("<p>");
String row = "\t\t";
foreach (string resultCategory in confusionMatrix_res_groundtruth.Keys)
{
row += resultCategory + "\t";
}
row += "<br>\n";
s.Append(row);
Console.WriteLine(row);
string matString = "";
foreach (string groundTruthCategory in confusionMatrix_groundtruth_res.Keys)
{
row = groundTruthCategory + "\t";
foreach (string resultCategory in confusionMatrix_res_groundtruth.Keys)
{
if (confusionMatrix_groundtruth_res[groundTruthCategory].ContainsKey(resultCategory))
{
row += confusionMatrix_groundtruth_res[groundTruthCategory][resultCategory].ToString();
}
else
{
row += "0";
}
row += "\t";
}
row += "<br>\n";
s.Append(row);
Console.WriteLine(row);
matString += row;
}
s.Append("</p>\n");
s.Append("</body>\n");
s.Append("</html>\n");
string summaryToBeSaved = s.ToString();
FileName = String.Format("Aggregated_Summary-{0}_Min{1}Max{2}Thresh{3}.html", guid, MinResults, MaxResults, MajorityThreshold);
storage.SaveATextFile("singleobjectlabeling", guid, FileName, summaryToBeSaved);
/// local file
string outputString = String.Format("{0} {1} {2} {3} {4} {5}\n", configString, noCorrect, noTotalConverged, (double)noCorrect / noTotalConverged, noTerminatedTasks, ResultsPerTask.Count - noTotalConverged);
string outputfile = DirectoryConstants.defaultTempDirectory + guid + "\\resultSummary.txt";
File.AppendAllText(outputfile, outputString);
string outputmatFile = DirectoryConstants.defaultTempDirectory + guid + "\\" + configString + "_mat.txt";
File.WriteAllText(outputmatFile, matString);
//for (double prob = 0; prob < 1;prob +=0.2)
//{
// SatyamResultsAnalysis.AnalyzeApprovalRate(aggEntries, entries, guid, configString, anotherChanceProbablity: prob);
//}
//for (double ratio = 0; ratio < 1; ratio += 0.2)
//{
// SatyamResultsAnalysis.AnalyzeApprovalRate(aggEntries, entriesBySubmitTime, noResultsNeededForAggregation, noResultsNeededForAggregation_new, guid, configString, approvalRatioThreshold: ratio);
//}
SatyamResultsAnalysis.AggregationAnalysis(aggEntries, entriesBySubmitTime, noResultsNeededForAggregation, noResultsNeededForAggregation_new, guid, configString);
}
/// <summary>
/// function generalizable.. TODO... to all tasks
/// </summary>
/// <param name="guids"></param>
/// <param name="IoUTreshold"></param>
/// <param name="saveImage"></param>
/// <param name="outputDirectory"></param>
/// <param name="MinResults"></param>
/// <param name="MaxResults"></param>
/// <param name="CategoryMajorityThreshold"></param>
/// <param name="PolygonBoundaryMajorityThreshold"></param>
/// <param name="ObjectsCoverageThreshold"></param>
/// <param name="overwrite"></param>
/// <param name="approvalAnalysis"></param>
public static void AggregateWithParameterAndValidatePascalVOCImageSegmentationByGUID(List <string> guids,
double IoUTreshold,
bool saveImage = false, string outputDirectory = null,
int MinResults = TaskConstants.IMAGE_SEGMENTATION_MTURK_MIN_RESULTS_TO_AGGREGATE,
int MaxResults = TaskConstants.IMAGE_SEGMENTATION_MTURK_MAX_RESULTS_TO_AGGREGATE,
double CategoryMajorityThreshold = TaskConstants.IMAGE_SEGMENTATION_MTURK_MAJORITY_CATEGORY_THRESHOLD,
double PolygonBoundaryMajorityThreshold = TaskConstants.IMAGE_SEGMENTATION_MTURK_MAJORITY_POLYGON_BOUNDARY_THRESHOLD,
double ObjectsCoverageThreshold = TaskConstants.IMAGE_SEGMENTATION_MTURK_OBJECT_COVERAGE_THRESHOLD_FOR_AGGREGATION_TERMINATION,
double minSimilarityThreshold = TaskConstants.IMAGE_SEGMENTATION_MTURK_POLYGON_IOU_THRESHOLD,
int minResultsForConsensus = TaskConstants.IMAGE_SEGMENTATION_MTURK_MIN_RESULTS_FOR_CONSENSUS,
bool overwrite = false,
bool approvalAnalysis = false)
{
string configString = "Min_" + MinResults + "_Max_" + MaxResults + "_Majority_" + PolygonBoundaryMajorityThreshold + "_Ratio_" + ObjectsCoverageThreshold;
Console.WriteLine("Aggregating for param set " + configString);
if (!overwrite && File.Exists(DirectoryConstants.defaultTempDirectory + "\\ImageSegmentationResult\\" + configString + ".txt"))
{
return;
}
SatyamResultsTableAccess resultsDB = new SatyamResultsTableAccess();
Dictionary <int, List <string> > WorkersPerTask = new Dictionary <int, List <string> >();
List <SatyamResultsTableEntry> entries = new List <SatyamResultsTableEntry>();
foreach (string guid in guids)
{
entries.AddRange(resultsDB.getEntriesByGUIDOrderByID(guid));
}
resultsDB.close();
SortedDictionary <DateTime, List <SatyamResultsTableEntry> > entriesBySubmitTime =
SatyamResultValidationToolKit.SortResultsBySubmitTime_OneResultPerTurkerPerTask(entries);
Dictionary <int, List <ImageSegmentationResult> > ResultsPerTask = new Dictionary <int, List <ImageSegmentationResult> >();
List <int> aggregatedTasks = new List <int>();
int noTotalConverged = 0;
//int noCorrect = 0;
int noTerminatedTasks = 0;
List <SatyamAggregatedResultsTableEntry> aggEntries = new List <SatyamAggregatedResultsTableEntry>();
Dictionary <int, int> noResultsNeededForAggregation = SatyamResultsAnalysis.getNoResultsNeededForAggregationFromLog(configString, guids[0]);
Dictionary <int, int> noResultsNeededForAggregation_new = new Dictionary <int, int>();
// play back by time
foreach (DateTime t in entriesBySubmitTime.Keys)
{
//Console.WriteLine("Processing Results of time: {0}", t);
List <SatyamResultsTableEntry> ResultEntries = entriesBySubmitTime[t];
foreach (SatyamResultsTableEntry entry in ResultEntries)
{
SatyamResult satyamResult = JSonUtils.ConvertJSonToObject <SatyamResult>(entry.ResultString);
SatyamTask task = JSonUtils.ConvertJSonToObject <SatyamTask>(satyamResult.TaskParametersString);
SatyamJob job = task.jobEntry;
string fileName = URIUtilities.filenameFromURINoExtension(task.SatyamURI);
int taskEntryID = entry.SatyamTaskTableEntryID;
if (aggregatedTasks.Contains(taskEntryID))
{
continue;
}
if (!ResultsPerTask.ContainsKey(taskEntryID))
{
ResultsPerTask.Add(taskEntryID, new List <ImageSegmentationResult>());
}
ResultsPerTask[taskEntryID].Add(JSonUtils.ConvertJSonToObject <ImageSegmentationResult>(satyamResult.TaskResult));
// check log if enough results are collected
if (noResultsNeededForAggregation != null && noResultsNeededForAggregation.ContainsKey(taskEntryID) &&
ResultsPerTask[taskEntryID].Count < noResultsNeededForAggregation[taskEntryID])
{
continue;
}
ImageSegmentationAggregatedResult aggResult = ImageSegmentationAggregator.getAggregatedResult(
ResultsPerTask[taskEntryID], task.SatyamURI, entry.JobGUID,
MinResults, MaxResults, CategoryMajorityThreshold,
PolygonBoundaryMajorityThreshold, ObjectsCoverageThreshold,
minSimilarityThreshold, minResultsForConsensus);
if (aggResult == null)
{
continue;
}
// aggregation happen
// record logs
if (noResultsNeededForAggregation == null || !noResultsNeededForAggregation.ContainsKey(taskEntryID))
{
noResultsNeededForAggregation_new.Add(taskEntryID, ResultsPerTask[taskEntryID].Count);
}
aggregatedTasks.Add(taskEntryID);
noTotalConverged++;
if (ResultsPerTask[taskEntryID].Count >= MaxResults)
{
noTerminatedTasks++;
}
SatyamAggregatedResult SatyamAggResult = new SatyamAggregatedResult();
SatyamAggResult.SatyamTaskTableEntryID = taskEntryID;
SatyamAggResult.AggregatedResultString = JSonUtils.ConvertObjectToJSon <ImageSegmentationAggregatedResult>(aggResult);
SatyamAggResult.TaskParameters = JSonUtils.ConvertJSonToObject <SatyamResult>(entry.ResultString).TaskParametersString;
SatyamAggregatedResultsTableEntry aggEntry = new SatyamAggregatedResultsTableEntry();
aggEntry.SatyamTaskTableEntryID = taskEntryID;
aggEntry.JobGUID = entry.JobGUID;
aggEntry.ResultString = JSonUtils.ConvertObjectToJSon <SatyamAggregatedResult>(SatyamAggResult);
aggEntries.Add(aggEntry);
List <SatyamAggregatedResultsTableEntry> tmpEntries = new List <SatyamAggregatedResultsTableEntry>();
tmpEntries.Add(aggEntry);
//ValidateSatyamKITTIDetectionAggregationResult(tmpEntries, saveImage, MinHeight, MaxOcclusion, Max_Truncation);
}
}
Console.WriteLine("Total_Aggregated_Tasks: {0}", noTotalConverged);
Console.WriteLine("Total_Terminated_Tasks: {0}", noTerminatedTasks);
SatyamResultsAnalysis.RecordAggregationLog(noResultsNeededForAggregation_new, configString, guids[0]);
string r = ValidatePascalVOCImageSegmentationResult_InstanceLevel(aggEntries, IoUTreshold);
r = noTotalConverged + " " + noTerminatedTasks + " " + r;
File.WriteAllText(DirectoryConstants.defaultTempDirectory + "\\ImageSegmentationResult\\" + configString + ".txt", r);
if (approvalAnalysis)
{
string approvalString = configString + "_PayCover_" + TaskConstants.IMAGE_SEGMENTATION_MTURK_OBJECT_COVERAGE_THRESHOLD_FOR_PAYMENT +
"_PayIoU_" + TaskConstants.IMAGE_SEGMENTATION_MTURK_POLYGON_IOU_THRESHOLD_FOR_PAYMENT;
//for (double ratio = 0; ratio < 1; ratio += 0.2)
//{
// SatyamResultsAnalysis.AnalyzeApprovalRate(aggEntries, entriesBySubmitTime, noResultsNeededForAggregation, noResultsNeededForAggregation_new, guids[0], configString, approvalRatioThreshold: ratio);
//}
SatyamResultsAnalysis.AggregationAnalysis(aggEntries, entriesBySubmitTime, noResultsNeededForAggregation, noResultsNeededForAggregation_new, guids[0], configString);
}
}