public static string GetAggregatedResultString(List <SatyamResultsTableEntry> results,
int MinResults = TaskConstants.SINGLE_OBJECT_LABLING_MTURK_MIN_RESULTS_TO_AGGREGATE,
int MaxResults = TaskConstants.SINGLE_OBJECT_LABLING_MTURK_MAX_RESULTS_TO_AGGREGATE,
double probabilityThreshold = TaskConstants.SINGLE_OBJECT_LABLING_MTURK_MAJORITY_THRESHOLD)
{
string resultString = null;
List <SingleObjectLabelingResult> resultList = new List <SingleObjectLabelingResult>();
foreach (SatyamResultsTableEntry entry in results)
{
SatyamResult res = JSonUtils.ConvertJSonToObject <SatyamResult>(entry.ResultString);
SingleObjectLabelingResult taskr = JSonUtils.ConvertJSonToObject <SingleObjectLabelingResult>(res.TaskResult);
resultList.Add(taskr);
}
SingleObjectLabelingAggregatedResult r = getAggregatedResult(resultList, MinResults, MaxResults, probabilityThreshold);
if (r != null)
{
string rString = JSonUtils.ConvertObjectToJSon <SingleObjectLabelingAggregatedResult>(r);
SatyamAggregatedResult aggResult = new SatyamAggregatedResult();
aggResult.SatyamTaskTableEntryID = results[0].SatyamTaskTableEntryID;
aggResult.AggregatedResultString = rString;
SatyamResult res = JSonUtils.ConvertJSonToObject <SatyamResult>(results[0].ResultString);
aggResult.TaskParameters = res.TaskParametersString;
resultString = JSonUtils.ConvertObjectToJSon <SatyamAggregatedResult>(aggResult);
}
return(resultString);
}
public static string GetAggregatedResultString(List <SatyamResultsTableEntry> results,
int MinResults = TaskConstants.OBJECT_COUNTING_MTURK_MIN_RESULTS_TO_AGGREGATE,
int MaxResults = TaskConstants.OBJECT_COUNTING_MTURK_MAX_RESULTS_TO_AGGREGATE,
double MAX_ABSOLUTE_COUNT_DEVIATION_LOWERBOUND = TaskConstants.OBJECT_COUNTING_MTURK_MAX_ABSOLUTE_COUNT_DEVIATION_LOWERBOUND,
double MAX_DEVIATION_FRACTION = TaskConstants.OBJECT_COUNTING_MTURK_MAX_DEVIATION_FRACTION,
double SUPER_MAJORITY_VALUE = TaskConstants.OBJECT_COUNTING_MTURK_SUPER_MAJORITY_VALUE)
{
Console.WriteLine("Aggregating");
string resultString = null;
List <ObjectCountingResult> resultList = new List <ObjectCountingResult>();
List <SatyamResult> SatyamResultList = new List <SatyamResult>();
foreach (SatyamResultsTableEntry entry in results)
{
SatyamResult res = JSonUtils.ConvertJSonToObject <SatyamResult>(entry.ResultString);
SatyamResultList.Add(res);
ObjectCountingResult taskr = JSonUtils.ConvertJSonToObject <ObjectCountingResult>(res.TaskResult);
resultList.Add(taskr);
}
//ObjectCountingAggregatedResult r = getAggregatedResult(resultList, MinResults, MaxResults, MAX_ABSOLUTE_COUNT_DEVIATION_LOWERBOUND, MAX_DEVIATION_FRACTION, SUPER_MAJORITY_VALUE);
ObjectCountingAggregatedResult r = getAggregatedResultUsingWorkerStatistics(SatyamResultList, MinResults, MaxResults, MAX_ABSOLUTE_COUNT_DEVIATION_LOWERBOUND, MAX_DEVIATION_FRACTION, SUPER_MAJORITY_VALUE);
if (r != null)
{
string rString = JSonUtils.ConvertObjectToJSon <ObjectCountingAggregatedResult>(r);
SatyamAggregatedResult aggResult = new SatyamAggregatedResult();
aggResult.SatyamTaskTableEntryID = results[0].SatyamTaskTableEntryID;
aggResult.AggregatedResultString = rString;
SatyamResult res = JSonUtils.ConvertJSonToObject <SatyamResult>(results[0].ResultString);
aggResult.TaskParameters = res.TaskParametersString;
resultString = JSonUtils.ConvertObjectToJSon <SatyamAggregatedResult>(aggResult);
}
return(resultString);
}
public static void SaveResultVideosLocally(List <SatyamResultsTableEntry> entries, string directoryName, int fps = 10)
{
directoryName = directoryName + "\\Raw";
if (!Directory.Exists(directoryName))
{
Directory.CreateDirectory(directoryName);
}
SatyamJobStorageAccountAccess satyamStorage = new SatyamJobStorageAccountAccess();
for (int i = 0; i < entries.Count; i++)
{
SatyamResultsTableEntry entry = entries[i];
SatyamResult satyamResult = JSonUtils.ConvertJSonToObject <SatyamResult>(entry.ResultString);
SatyamTask task = JSonUtils.ConvertJSonToObject <SatyamTask>(satyamResult.TaskParametersString);
MultiObjectTrackingSubmittedJob job = JSonUtils.ConvertJSonToObject <MultiObjectTrackingSubmittedJob>(task.jobEntry.JobParameters);
//VATIC_DVA_CrowdsourcedResult taskr = new VATIC_DVA_CrowdsourcedResult(satyamResult.TaskResult, task.SatyamURI, start, end, task.SatyamJobSubmissionsTableEntryID.ToString());
VATIC_DVA_CrowdsourcedResult taskr = MultiObjectTrackingAggregator.createVATIC_DVA_CrowdsourcedResultUsingSatyamBlobImageCount(satyamResult.TaskResult, task.SatyamURI, entry.ID.ToString(), job.FrameRate);
MultiObjectTrackingResult res = taskr.getCompressedTracksInTimeSegment();
List <string> ImageURLs = satyamStorage.getURLListOfSubDirectoryByURL(task.SatyamURI);
string[] names = task.SatyamURI.Split('/');
string videoName = names[names.Length - 2] + "_" + entry.ID;
generateVideoForEvaluation(ImageURLs, res, directoryName, videoName, job.FrameRate);
}
}
public static string GetAggregatedResultString(List <SatyamResultsTableEntry> results)
{
if (results.Count == 0)
{
return(null);
}
string resultString = null;
List <ImageSegmentationResult> resultList = new List <ImageSegmentationResult>();
List <string> WorkersPerTask = new List <string>();
SatyamResult res0 = JSonUtils.ConvertJSonToObject <SatyamResult>(results[0].ResultString);
SatyamTask task = JSonUtils.ConvertJSonToObject <SatyamTask>(res0.TaskParametersString);
string SatyamURL = task.SatyamURI;
string guid = results[0].JobGUID;
foreach (SatyamResultsTableEntry entry in results)
{
SatyamResult res = JSonUtils.ConvertJSonToObject <SatyamResult>(entry.ResultString);
// remove duplicate workers result
string workerID = res.amazonInfo.WorkerID;
if (WorkersPerTask.Contains(workerID))
{
continue;
}
string assignmentID = res.amazonInfo.AssignmentID;
if (assignmentID == "" || assignmentID == "ASSIGNMENT_ID_NOT_AVAILABLE")
{
continue;
}
//enclose only non-duplicate results, one per each worker.
if (workerID != "" && workerID != TaskConstants.AdminID)
{
// make a pass for test and admins
WorkersPerTask.Add(workerID);
}
ImageSegmentationResult taskr = JSonUtils.ConvertJSonToObject <ImageSegmentationResult>(res.TaskResult);
resultList.Add(taskr);
}
ImageSegmentationAggregatedResult r = getAggregatedResult(resultList, SatyamURL, guid);
if (r != null)
{
string rString = JSonUtils.ConvertObjectToJSon <ImageSegmentationAggregatedResult>(r);
SatyamAggregatedResult aggResult = new SatyamAggregatedResult();
aggResult.SatyamTaskTableEntryID = results[0].SatyamTaskTableEntryID;
aggResult.AggregatedResultString = rString;
SatyamResult res = JSonUtils.ConvertJSonToObject <SatyamResult>(results[0].ResultString);
aggResult.TaskParameters = res.TaskParametersString;
resultString = JSonUtils.ConvertObjectToJSon <SatyamAggregatedResult>(aggResult);
}
return(resultString);
}
//public static string GetAggregatedResultString(List<SatyamResultsTableEntry> results,
// int MinResults = TaskConstants.OBJECT_COUNTING_MTURK_MIN_RESULTS_TO_AGGREGATE,
// int MaxResults = TaskConstants.OBJECT_COUNTING_MTURK_MAX_RESULTS_TO_AGGREGATE,
// double MAX_ABSOLUTE_COUNT_DEVIATION_LOWERBOUND = TaskConstants.OBJECT_COUNTING_MTURK_MAX_ABSOLUTE_COUNT_DEVIATION_LOWERBOUND,
// double MAX_DEVIATION_FRACTION = TaskConstants.OBJECT_COUNTING_MTURK_MAX_DEVIATION_FRACTION,
// double SUPER_MAJORITY_VALUE = TaskConstants.OBJECT_COUNTING_MTURK_SUPER_MAJORITY_VALUE)
//{
// string resultString = null;
// List<ObjectCountingResult> resultList = new List<ObjectCountingResult>();
// foreach (SatyamResultsTableEntry entry in results)
// {
// SatyamResult res = JSonUtils.ConvertJSonToObject<SatyamResult>(entry.ResultString);
// ObjectCountingResult taskr = JSonUtils.ConvertJSonToObject<ObjectCountingResult>(res.TaskResult);
// resultList.Add(taskr);
// }
// ObjectCountingAggregatedResult r = getAggregatedResult(resultList, MinResults, MaxResults, MAX_ABSOLUTE_COUNT_DEVIATION_LOWERBOUND, MAX_DEVIATION_FRACTION, SUPER_MAJORITY_VALUE);
// if (r != null)
// {
// string rString = JSonUtils.ConvertObjectToJSon<ObjectCountingAggregatedResult>(r);
// SatyamAggregatedResult aggResult = new SatyamAggregatedResult();
// aggResult.SatyamTaskTableEntryID = results[0].SatyamTaskTableEntryID;
// aggResult.AggregatedResultString = rString;
// SatyamResult res = JSonUtils.ConvertJSonToObject<SatyamResult>(results[0].ResultString);
// aggResult.TaskParameters = res.TaskParametersString;
// resultString = JSonUtils.ConvertObjectToJSon<SatyamAggregatedResult>(aggResult);
// }
// return resultString;
//}
public static bool IsAcceptable(SatyamAggregatedResultsTableEntry aggResult, SatyamResultsTableEntry result,
double MAX_ABSOLUTE_COUNT_DEVIATION = TaskConstants.OBJECT_COUNTING_MTURK_MAX_ABSOLUTE_COUNT_DEVIATION_FOR_PAYMENT,
double MAX_DEVIATION_FRACTION = TaskConstants.OBJECT_COUNTING_MTURK_MAX_DEVIATION_FRACTION_FOR_PAYMENT)
{
SatyamResult res = JSonUtils.ConvertJSonToObject <SatyamResult>(result.ResultString);
SatyamAggregatedResult aggres = JSonUtils.ConvertJSonToObject <SatyamAggregatedResult>(aggResult.ResultString);
ObjectCountingResult r = JSonUtils.ConvertJSonToObject <ObjectCountingResult>(res.TaskResult);
ObjectCountingAggregatedResult ragg = JSonUtils.ConvertJSonToObject <ObjectCountingAggregatedResult>(aggres.AggregatedResultString);
double maxdev = MAX_ABSOLUTE_COUNT_DEVIATION;
if (ragg.Count * MAX_DEVIATION_FRACTION > maxdev)
{
maxdev = ragg.Count * MAX_DEVIATION_FRACTION;
}
if (Math.Abs(ragg.Count - r.Count) <= maxdev)
{
return(true);
}
else
{
return(false);
}
}
public static void SaveResultVideosLocally(List <SatyamResultsTableEntry> entries, string directoryName, int fps = 10)
{
directoryName = directoryName + "\\Raw";
if (!Directory.Exists(directoryName))
{
Directory.CreateDirectory(directoryName);
}
SatyamJobStorageAccountAccess satyamStorage = new SatyamJobStorageAccountAccess();
for (int i = 0; i < entries.Count; i++)
{
SatyamResultsTableEntry entry = entries[i];
SatyamResult satyamResult = JSonUtils.ConvertJSonToObject <SatyamResult>(entry.ResultString);
SatyamTask task = JSonUtils.ConvertJSonToObject <SatyamTask>(satyamResult.TaskParametersString);
MultiObjectTrackingSubmittedJob job = JSonUtils.ConvertJSonToObject <MultiObjectTrackingSubmittedJob>(task.jobEntry.JobParameters);
string blobDir = URIUtilities.localDirectoryFullPathFromURI(task.SatyamURI);
//VATIC_DVA_CrowdsourcedResult taskr = new VATIC_DVA_CrowdsourcedResult(satyamResult.TaskResult, task.SatyamURI, start, end, task.SatyamJobSubmissionsTableEntryID.ToString());
VATIC_DVA_CrowdsourcedResult taskr = MultiObjectTrackingAggregator.createVATIC_DVA_CrowdsourcedResultUsingSatyamBlobImageCount(satyamResult.TaskResult, blobDir, entry.ID.ToString(), job.FrameRate);
MultiObjectTrackingResult res = taskr.getCompressedTracksInTimeSegment();
List <string> ImageURLs = satyamStorage.getURLListOfSpecificExtensionUnderSubDirectoryByURI(blobDir, new List <string>()
{
"jpg", "png"
});
string videoName = URIUtilities.localDirectoryNameFromURI(blobDir) + "_" + URIUtilities.filenameFromURINoExtension(task.SatyamURI) + "_" + entry.ID;
MultiObjectTrackingAnalyzer.generateVideoForEvaluation(ImageURLs, res, directoryName, videoName, job.FrameRate);
}
}
public static void SaveKITTIResultVideosLocally(List <SatyamResultsTableEntry> entries, string directoryName, int fps = 10)
{
directoryName = directoryName + "\\Raw";
if (!Directory.Exists(directoryName))
{
Directory.CreateDirectory(directoryName);
}
SatyamJobStorageAccountAccess satyamStorage = new SatyamJobStorageAccountAccess();
for (int i = 0; i < entries.Count; i++)
{
SatyamResultsTableEntry entry = entries[i];
SatyamResult satyamResult = JSonUtils.ConvertJSonToObject <SatyamResult>(entry.ResultString);
SatyamTask task = JSonUtils.ConvertJSonToObject <SatyamTask>(satyamResult.TaskParametersString);
MultiObjectTrackingSubmittedJob job = JSonUtils.ConvertJSonToObject <MultiObjectTrackingSubmittedJob>(task.jobEntry.JobParameters);
//VATIC_DVA_CrowdsourcedResult taskr = VATIC_DVA_CrowdsourcedResult.createVATIC_DVA_CrowdsourcedResultUsingSatyamBlobImageCount(satyamResult.TaskResult, task.SatyamURI, task.SatyamJobSubmissionsTableEntryID.ToString(), job.FrameRate);
//MultiObjectTrackingResult res = taskr.getCompressedTracksInTimeSegment();
string[] names = task.SatyamURI.Split('/');
string videoName = names[names.Length - 2] + "_" + entry.ID;
string[] fields = videoName.Split('_');
string videoSequence = fields[fields.Length - 5];
int startingFrame = Convert.ToInt32(fields[fields.Length - 2]);
int maxChunkEndFrame = startingFrame + job.ChunkDuration * job.FrameRate;
int noFrameOverlap = (int)(job.ChunkOverlap * job.FrameRate);
if (startingFrame != 0)
{
startingFrame -= noFrameOverlap;
}
if (entry.JobGUID == "1e43a983-548d-4a2e-8161-5537eb985902")
{
videoSequence = getCorrectSequenceNo(videoSequence);
}
string videoFrameDir = DirectoryConstants.KITTITrackingImages + videoSequence;
List <string> files = Directory.GetFiles(videoFrameDir).ToList();
List <string> ImageURLs = new List <string>();
for (int j = startingFrame; j < files.Count && j < maxChunkEndFrame; j++)
{
ImageURLs.Add(files[j]);
}
if (ImageURLs.Count == 0)
{
continue;
}
VATIC_DVA_CrowdsourcedResult taskr = new VATIC_DVA_CrowdsourcedResult(satyamResult.TaskResult, videoName, entry.ID.ToString(), ImageURLs.Count, job.FrameRate);
MultiObjectTrackingResult res = taskr.getCompressedTracksInTimeSegment();
generateVideoForEvaluation(ImageURLs, res, directoryName, videoName, job.FrameRate);
}
}
//protected void SubmitButton_Click(object sender, EventArgs e)
private void storeResult()
{
DateTime SubmitTime = DateTime.Now;
DateTime PageLoadTime = Convert.ToDateTime(Hidden_PageLoadTime.Value);
SatyamTaskTableEntry taskEntry = JSonUtils.ConvertJSonToObject <SatyamTaskTableEntry>(Hidden_TaskEntryString.Value);
string tracksString = TracksOutput_Hidden.Value;
string urlList = Hidden_ImageURLList.Value;
string[] fields = urlList.Split(',');
DateTime start = DateTime.MinValue;
List <DateTime> frameTimes = new List <DateTime>();
//double frameTimeSpanInMiliseconds = (Convert.ToDouble(Hidden_ChunkDuration.Value) / (double)fields.Length) * 1000;
double frameTimeSpanInMiliseconds = (double)(1000) / Convert.ToDouble(fps_Hidden.Value);
for (int i = 0; i < fields.Length; i++)
{
DateTime t;
t = start.AddMilliseconds(frameTimeSpanInMiliseconds * i);
frameTimes.Add(t);
}
string s = Raw_VATIC_DVA_Crowdsourced_Track_Collection.Raw_VATIC_DVA_Crowdsourced_Track_Collection_ToTrackStrings(tracksString, frameTimes);
SatyamResult result = new SatyamResult();
result.TaskParametersString = taskEntry.TaskParametersString;
result.TaskStartTime = PageLoadTime;
result.TaskEndTime = SubmitTime;
result.TaskTableEntryID = taskEntry.ID;
AmazonTaskResultInfo amazonInfo = new AmazonTaskResultInfo();
amazonInfo.AssignmentID = "";
amazonInfo.WorkerID = "";
amazonInfo.HITID = "";
result.amazonInfo = amazonInfo;
result.TaskResult = s;
string resultString = JSonUtils.ConvertObjectToJSon <SatyamResult>(result);
SatyamResultsTableAccess resultdb = new SatyamResultsTableAccess();
resultdb.AddEntry(taskEntry.JobTemplateType, taskEntry.UserID, taskEntry.JobGUID, resultString, taskEntry.ID, PageLoadTime, SubmitTime);
resultdb.close();
//SatyamTaskTableAccess taskDB = new SatyamTaskTableAccess();
//taskDB.IncrementDoneScore(taskEntry.ID);
bool NotDone = getNewRandomJob();
if (NotDone == false)
{
Response.Redirect("AllJobsDone.aspx");
}
}
public static void SaveResultOnlyText(string jobGUID)
{
string directoryName = DirectoryConstants.defaultTempDirectory + "\\" + jobGUID + "\\";
SatyamResultsTableAccess resultsDB = new SatyamResultsTableAccess();
List <SatyamResultsTableEntry> entries = resultsDB.getEntriesByGUID(jobGUID);
resultsDB.close();
if (!Directory.Exists(directoryName))
{
Directory.CreateDirectory(directoryName);
}
directoryName = directoryName + "\\Raw\\";
if (!Directory.Exists(directoryName))
{
Directory.CreateDirectory(directoryName);
}
for (int i = 0; i < entries.Count; i++)
{
SatyamResultsTableEntry entry = entries[i];
SatyamResult satyamResult = JSonUtils.ConvertJSonToObject <SatyamResult>(entry.ResultString);
SatyamTask task = JSonUtils.ConvertJSonToObject <SatyamTask>(satyamResult.TaskParametersString);
SatyamJob job = task.jobEntry;
string result = satyamResult.TaskResult;
string ofilename = URIUtilities.filenameFromURI(task.SatyamURI);
string[] fields = ofilename.Split('.');
string fileName = "";
for (int j = 0; j < fields.Count(); j++)
{
fileName += fields[j];
if (j == fields.Count() - 1)
{
break;
}
fileName += ".";
}
//string fileName = ofilename;
fileName = fileName + "-Result";
fileName = fileName + "-" + entry.ID;
Console.WriteLine("Saving " + fileName);
string resultFile = directoryName + fileName + ".txt";
StreamWriter f = new System.IO.StreamWriter(resultFile);
f.WriteLine(result);
f.Close();
}
}
public static void SaveResultImagesLocally(List <SatyamResultsTableEntry> entries, string directoryName)
{
if (!Directory.Exists(directoryName))
{
Directory.CreateDirectory(directoryName);
}
directoryName = directoryName + "\\Raw\\";
if (!Directory.Exists(directoryName))
{
Directory.CreateDirectory(directoryName);
}
for (int i = 0; i < entries.Count; i++)
{
SatyamResultsTableEntry entry = entries[i];
SatyamResult satyamResult = JSonUtils.ConvertJSonToObject <SatyamResult>(entry.ResultString);
SatyamTask task = JSonUtils.ConvertJSonToObject <SatyamTask>(satyamResult.TaskParametersString);
SatyamJob job = task.jobEntry;
string result = satyamResult.TaskResult;
Image originalImage = ImageUtilities.getImageFromURI(task.SatyamURI);
Image ResultImage = DrawResultStringOnImage(result, originalImage);
string ofilename = URIUtilities.filenameFromURI(task.SatyamURI);
string[] fields = ofilename.Split('.');
string fileName = "";
for (int j = 0; j < fields.Count(); j++)
{
fileName += fields[j];
if (j == fields.Count() - 1)
{
break;
}
fileName += ".";
}
//string fileName = ofilename;
fileName = fileName + "-Result";
fileName = fileName + "-" + entry.ID;
Console.WriteLine("Saving " + fileName);
ImageUtilities.saveImage(ResultImage, directoryName, fileName);
string resultFile = directoryName + fileName + ".txt";
StreamWriter f = new System.IO.StreamWriter(resultFile);
f.WriteLine(result);
f.Close();
}
}
//protected void Page_Unload(object sender, EventArgs e)
//{
// if (!DoneFlag)
// {
// SatyamTaskTableEntry taskEntry = JSonUtils.ConvertJSonToObject<SatyamTaskTableEntry>(Hidden_TaskEntryString.Value);
// if (taskEntry == null) return;
// // Decrement the DoneScore, releasing token after latest results are saved.
// SatyamTaskTableAccess taskTableDB = new SatyamTaskTableAccess();
// taskTableDB.DecrementDoneScore(taskEntry.ID);
// taskTableDB.close();
// }
//}
protected void SubmitButton_Click(object sender, EventArgs e)
{
DateTime SubmitTime = DateTime.Now;
DateTime PageLoadTime = Convert.ToDateTime(Hidden_PageLoadTime.Value);
SatyamTaskTableEntry taskEntry = JSonUtils.ConvertJSonToObject <SatyamTaskTableEntry>(Hidden_TaskEntryString.Value);
SatyamResult result = new SatyamResult();
result.TaskParametersString = taskEntry.TaskParametersString;
result.TaskStartTime = PageLoadTime;
result.TaskEndTime = SubmitTime;
result.TaskTableEntryID = taskEntry.ID;
AmazonTaskResultInfo amazonInfo = new AmazonTaskResultInfo();
amazonInfo.AssignmentID = "";
amazonInfo.WorkerID = "";
amazonInfo.HITID = "";
result.amazonInfo = amazonInfo;
result.TaskResult = Hidden_Result.Value;
result.PrevResultID = Convert.ToInt32(Hidden_PrevResultID.Value);
string resultString = JSonUtils.ConvertObjectToJSon <SatyamResult>(result);
SatyamResultsTableAccess resultdb = new SatyamResultsTableAccess();
resultdb.AddEntry(taskEntry.JobTemplateType, taskEntry.UserID, taskEntry.JobGUID, resultString, taskEntry.ID, PageLoadTime, SubmitTime);
resultdb.close();
SatyamTaskTableManagement.UpdateResultNumber(taskEntry.ID);
// Decrement the DoneScore, releasing token after latest results are saved.
SatyamTaskTableAccess taskTableDB = new SatyamTaskTableAccess();
//taskTableDB.DecrementDoneScore(taskEntry.ID);
taskTableDB.UpdateDoneScore(taskEntry.ID, 0);
taskTableDB.close();
//DoneFlag = true;
//bool NotDone = getNewRandomJob();
//if (NotDone == false)
//{
Response.Redirect("AllJobsDone.aspx");
//}
}
public static List <SatyamResultsTableEntry> getAllEntriesByGUIDFilteredByHITID(string guid, string hitid)
{
List <SatyamResultsTableEntry> ret = new List <SatyamResultsTableEntry>();
SatyamResultsTableAccess resDB = new SatyamResultsTableAccess();
List <SatyamResultsTableEntry> res = resDB.getEntriesByGUID(guid);
resDB.close();
foreach (SatyamResultsTableEntry r in res)
{
SatyamResult rr = JSonUtils.ConvertJSonToObject <SatyamResult>(r.ResultString);
if (rr.amazonInfo.HITID == hitid)
{
ret.Add(r);
}
}
return(ret);
}
public static bool IsAcceptable(SatyamAggregatedResultsTableEntry aggResult, SatyamResultsTableEntry result)
{
SatyamResult res = JSonUtils.ConvertJSonToObject <SatyamResult>(result.ResultString);
SatyamAggregatedResult aggres = JSonUtils.ConvertJSonToObject <SatyamAggregatedResult>(aggResult.ResultString);
SingleObjectLabelingResult r = JSonUtils.ConvertJSonToObject <SingleObjectLabelingResult>(res.TaskResult);
SingleObjectLabelingAggregatedResult ragg = JSonUtils.ConvertJSonToObject <SingleObjectLabelingAggregatedResult>(aggres.AggregatedResultString);
if (r.Category == ragg.Category)
{
return(true);
}
else
{
return(false);
}
}
public List <int> getAllTaskIDsAWorkerHasDoneForAGUID(string WorkerID, string guid)
{
List <int> ret = new List <int>();
List <SatyamResultsTableEntry> entries = getEntriesByGUID(guid);
foreach (SatyamResultsTableEntry entry in entries)
{
SatyamResult result = JSonUtils.ConvertJSonToObject <SatyamResult>(entry.ResultString);
if (result.amazonInfo.WorkerID == WorkerID)
{
ret.Add(result.TaskTableEntryID);
}
}
return(ret);
}
protected void SubmitButton_Click(object sender, EventArgs e)
{
DateTime SubmitTime = DateTime.Now;
DateTime PageLoadTime = Convert.ToDateTime(Hidden_PageLoadTime.Value);
SatyamTaskTableEntry taskEntry = JSonUtils.ConvertJSonToObject <SatyamTaskTableEntry>(Hidden_TaskEntryString.Value);
SatyamResult result = new SatyamResult();
result.TaskParametersString = taskEntry.TaskParametersString;
result.TaskStartTime = PageLoadTime;
result.TaskEndTime = SubmitTime;
result.TaskTableEntryID = taskEntry.ID;
AmazonTaskResultInfo amazonInfo = new AmazonTaskResultInfo();
amazonInfo.AssignmentID = Hidden_AmazonAssignmentID.Value;
amazonInfo.WorkerID = Hidden_AmazonWorkerID.Value;
amazonInfo.HITID = Hidden_HITID.Value;
amazonInfo.PricePerHIT = Convert.ToDouble(Hidden_Price.Value);
result.amazonInfo = amazonInfo;
result.TaskResult = Hidden_Result.Value;
string resultString = JSonUtils.ConvertObjectToJSon <SatyamResult>(result);
SatyamResultsTableAccess resultdb = new SatyamResultsTableAccess();
resultdb.AddEntry(taskEntry.JobTemplateType, taskEntry.UserID, taskEntry.JobGUID, resultString, taskEntry.ID, PageLoadTime, SubmitTime);
resultdb.close();
SatyamTaskTableManagement.UpdateResultNumber(taskEntry.ID);
if (!Testing)
{
SatyamAmazonHITTableAccess HITdb = new SatyamAmazonHITTableAccess();
string HITID = result.amazonInfo.HITID;
HITdb.UpdateStatusByHITID(HITID, HitStatus.submitted);
HITdb.close();
AmazonMTurkNotification.submitAmazonTurkHit(result.amazonInfo.AssignmentID, result.amazonInfo.WorkerID, false);
}
Response.Redirect("AllJobsDone.aspx");
//}
}
private MultiObjectLocalizationAndLabelingResult LoadLatestTurkerResult(SatyamTaskTableEntry entry)
{
SatyamResultsTableAccess resultsDB = new SatyamResultsTableAccess();
List <SatyamResultsTableEntry> entries = resultsDB.getEntriesByGUIDAndTaskID(entry.JobGUID, entry.ID);
if (entries.Count == 0)
{
return(null);
}
//organized sequentially
SatyamResultsTableEntry prevResult = entries[entries.Count - 1];
Hidden_PrevResultID.Value = prevResult.ID.ToString();
SatyamResult satyamResult = JSonUtils.ConvertJSonToObject <SatyamResult>(prevResult.ResultString);
MultiObjectLocalizationAndLabelingResult res = JSonUtils.ConvertJSonToObject <MultiObjectLocalizationAndLabelingResult>(satyamResult.TaskResult);
return(res);
}
protected void SubmitButton_Click(object sender, EventArgs e)
{
DateTime SubmitTime = DateTime.Now;
DateTime PageLoadTime = Convert.ToDateTime(Hidden_PageLoadTime.Value);
if (CategorySelection_RadioButtonList.SelectedIndex != -1)
{
SatyamTaskTableEntry taskEntry = JSonUtils.ConvertJSonToObject <SatyamTaskTableEntry>(Hidden_TaskEntryString.Value);
SatyamResult result = new SatyamResult();
result.TaskParametersString = taskEntry.TaskParametersString;
result.TaskStartTime = PageLoadTime;
result.TaskEndTime = SubmitTime;
result.TaskTableEntryID = taskEntry.ID;
AmazonTaskResultInfo amazonInfo = new AmazonTaskResultInfo();
amazonInfo.AssignmentID = "";
amazonInfo.WorkerID = "";
amazonInfo.HITID = "";
result.amazonInfo = amazonInfo;
SingleObjectLabelingResult sresult = new SingleObjectLabelingResult();
sresult.Category = CategorySelection_RadioButtonList.Items[CategorySelection_RadioButtonList.SelectedIndex].Text;
string sresultString = JSonUtils.ConvertObjectToJSon <SingleObjectLabelingResult>(sresult);
result.TaskResult = sresultString;
string resultString = JSonUtils.ConvertObjectToJSon <SatyamResult>(result);
SatyamResultsTableAccess resultdb = new SatyamResultsTableAccess();
resultdb.AddEntry(taskEntry.JobTemplateType, taskEntry.UserID, taskEntry.JobGUID, resultString, taskEntry.ID, PageLoadTime, SubmitTime);
resultdb.close();
//SatyamTaskTableAccess taskDB = new SatyamTaskTableAccess();
//taskDB.IncrementDoneScore(taskEntry.ID);
bool status = getNewRandomJob();
if (!status)
{
Response.Redirect("AllJobsDone.aspx");
}
}
}
public static MultiObjectLabelingAndLocalizationAnalysisPerJob Analyse(List <SatyamResultsTableEntry> entries)
{
MultiObjectLabelingAndLocalizationAnalysisPerJob ana = new MultiObjectLabelingAndLocalizationAnalysisPerJob();
ana.jobTemplateType = entries[0].JobTemplateType;
for (int i = 0; i < entries.Count; i++)
{
SatyamResultsTableEntry entry = entries[i];
SatyamResult satyamResult = JSonUtils.ConvertJSonToObject <SatyamResult>(entry.ResultString);
SatyamTask task = JSonUtils.ConvertJSonToObject <SatyamTask>(satyamResult.TaskParametersString);
SatyamJob job = task.jobEntry;
MultiObjectLocalizationAndLabelingResult res = JSonUtils.ConvertJSonToObject <MultiObjectLocalizationAndLabelingResult>(satyamResult.TaskResult);
}
return(ana);
}
public static void SaveByGUIDRequester(string guid)
{
//get all the results of a GUID
SatyamResultsTableAccess resultsDB = new SatyamResultsTableAccess();
List <SatyamResultsTableEntry> results = resultsDB.getEntriesByGUID(guid);
if (results.Count == 0)
{
resultsDB.close();
return;
}
//get the azure info
SatyamResult result = JSonUtils.ConvertJSonToObject <SatyamResult>(results[0].ResultString);
SatyamTask task = JSonUtils.ConvertJSonToObject <SatyamTask>(result.TaskParametersString);
SatyamJob job = task.jobEntry;
string ConnectionString = job.azureInformation.AzureBlobStorageConnectionString;
string ContainerName = job.azureInformation.AzureBlobStorageContainerName;
string DirectoryName = job.azureInformation.AzureBlobStorageContainerDirectoryName;
//create the data to be saved
List <SatyamResultSaveDataRequester> savingDataList = new List <SatyamResultSaveDataRequester>();
StringBuilder s = new StringBuilder();
for (int i = 0; i < results.Count; i++)
{
SatyamResultSaveDataRequester data = new SatyamResultSaveDataRequester(results[i]);
String jsonString = JSonUtils.ConvertObjectToJSon(data);
s.Append(jsonString);
if (i == results.Count - 1)
{
s.Append("\n");
}
}
string dataToBeSaved = s.ToString();
BlobContainerManager bcm = new BlobContainerManager();
bcm.Connect(ConnectionString);
string FileName = "Results-" + results[0].JobGUID + ".txt";
bcm.SaveATextFile(ContainerName, DirectoryName, FileName, dataToBeSaved);
resultsDB.close();
}
public static void TestGetGlobalStatistics()
{
Dictionary <string, int> workerSubmissions = new Dictionary <string, int>();
Dictionary <string, int> HitSubmissions = new Dictionary <string, int>();
Dictionary <int, int> TaskSubmissions = new Dictionary <int, int>();
SatyamResultsTableAccess resultsDB = new SatyamResultsTableAccess();
List <SatyamResultsTableEntry> entries = new List <SatyamResultsTableEntry>();
entries = (resultsDB.getAllEntries());
for (int i = 0; i < entries.Count; i++)
{
SatyamResultsTableEntry entry = entries[i];
SatyamResult satyamResult = JSonUtils.ConvertJSonToObject <SatyamResult>(entry.ResultString);
SatyamTask task = JSonUtils.ConvertJSonToObject <SatyamTask>(satyamResult.TaskParametersString);
SatyamJob job = task.jobEntry;
string worker = satyamResult.amazonInfo.WorkerID;
string hitID = satyamResult.amazonInfo.HITID;
int taskID = entry.SatyamTaskTableEntryID;
if (!workerSubmissions.ContainsKey(worker))
{
workerSubmissions.Add(worker, 0);
}
workerSubmissions[worker]++;
if (!HitSubmissions.ContainsKey(hitID))
{
HitSubmissions.Add(hitID, 0);
}
HitSubmissions[hitID]++;
if (!TaskSubmissions.ContainsKey(taskID))
{
TaskSubmissions.Add(taskID, 0);
}
TaskSubmissions[taskID]++;
}
Console.WriteLine(workerSubmissions.Count);
Console.WriteLine(HitSubmissions.Count);
Console.WriteLine(TaskSubmissions.Count);
}
protected void SubmitButton_Click(object sender, EventArgs e)
{
DateTime SubmitTime = DateTime.Now;
DateTime PageLoadTime = Convert.ToDateTime(Hidden_PageLoadTime.Value);
SatyamTaskTableEntry taskEntry = JSonUtils.ConvertJSonToObject <SatyamTaskTableEntry>(Hidden_TaskEntryString.Value);
SatyamResult result = new SatyamResult();
result.TaskParametersString = taskEntry.TaskParametersString;
result.TaskStartTime = PageLoadTime;
result.TaskEndTime = SubmitTime;
result.TaskTableEntryID = taskEntry.ID;
AmazonTaskResultInfo amazonInfo = new AmazonTaskResultInfo();
amazonInfo.AssignmentID = "";
amazonInfo.WorkerID = "";
amazonInfo.HITID = "";
result.amazonInfo = amazonInfo;
result.TaskResult = Hidden_Result.Value;
string resultString = JSonUtils.ConvertObjectToJSon <SatyamResult>(result);
SatyamResultsTableAccess resultdb = new SatyamResultsTableAccess();
resultdb.AddEntry(taskEntry.JobTemplateType, taskEntry.UserID, taskEntry.JobGUID, resultString, taskEntry.ID, PageLoadTime, SubmitTime);
resultdb.close();
//SatyamTaskTableAccess taskDB = new SatyamTaskTableAccess();
//taskDB.IncrementDoneScore(taskEntry.ID);
bool NotDone = getNewRandomJob();
if (NotDone == false)
{
Response.Redirect("AllJobsDone.aspx");
}
}
public SatyamResultSaveDataRequester(SatyamResultsTableEntry entry)
{
string status = entry.Status;
if (status == ResultStatus.accepted || status == ResultStatus.accepted_Paid || status == ResultStatus.accepted_NotPaid)
{
resultAccepted = true;
}
else
{
resultAccepted = false;
}
if (status == ResultStatus.rejected_Paid || status == ResultStatus.accepted_Paid)
{
resultPaid = true;
}
else
{
resultPaid = false;
}
SatyamResult result = JSonUtils.ConvertJSonToObject <SatyamResult>(entry.ResultString);
TaskStartTime = result.TaskStartTime;
TaskEndTime = result.TaskEndTime;
JobGUID = entry.JobGUID;
JobTemplateType = entry.JobTemplateType;
SatyamTask task = JSonUtils.ConvertJSonToObject <SatyamTask>(result.TaskParametersString);
//OrignialURI = task.OriginalURI;
SatyamJob job = task.jobEntry;
UserID = job.UserID;
JobSubmitTime = job.JobSubmitTime;
JobParameters = job.JobParameters;
amazonInfo = new AmazonSavingInfo();
amazonInfo.AssignmentID = result.amazonInfo.AssignmentID;
amazonInfo.pricePerHIT = result.amazonInfo.PricePerHIT;
amazonInfo.TasksPerHIT = job.TasksPerJob;
amazonInfo.WorkerID = result.amazonInfo.WorkerID;
ResultString = result.TaskResult;
}
protected void SubmitButton_Click(object sender, EventArgs e)
{
DateTime SubmitTime = DateTime.Now;
DateTime PageLoadTime = Convert.ToDateTime(Hidden_PageLoadTime.Value);
int count;
bool isValidCount = int.TryParse(CountTextBox.Text, out count);
if (!isValidCount)
{
ErrorLabel.Text = "Error : Ivalid Count.";
ErrorLabel.ForeColor = System.Drawing.Color.Red;
ErrorLabel.Font.Bold = true;
}
else if (count < 0)
{
ErrorLabel.Text = "Error : Count cannot be less than zero.";
ErrorLabel.ForeColor = System.Drawing.Color.Red;
ErrorLabel.Font.Bold = true;
}
else
{
ErrorLabel.Text = "";
CountTextBox.Text = "";
SatyamTaskTableEntry taskEntry = JSonUtils.ConvertJSonToObject <SatyamTaskTableEntry>(Hidden_TaskEntryString.Value);
SatyamResult result = new SatyamResult();
result.TaskParametersString = taskEntry.TaskParametersString;
result.TaskStartTime = PageLoadTime;
result.TaskEndTime = SubmitTime;
result.TaskTableEntryID = taskEntry.ID;
AmazonTaskResultInfo amazonInfo = new AmazonTaskResultInfo();
amazonInfo.AssignmentID = Hidden_AmazonAssignmentID.Value;
amazonInfo.WorkerID = Hidden_AmazonWorkerID.Value;
amazonInfo.HITID = Hidden_HITID.Value;
amazonInfo.PricePerHIT = Convert.ToDouble(Hidden_Price.Value);
result.amazonInfo = amazonInfo;
ObjectCountingResult sresult = new ObjectCountingResult();
sresult.Count = count;
string sresultString = JSonUtils.ConvertObjectToJSon <ObjectCountingResult>(sresult);
result.TaskResult = sresultString;
string resultString = JSonUtils.ConvertObjectToJSon <SatyamResult>(result);
SatyamResultsTableAccess resultdb = new SatyamResultsTableAccess();
resultdb.AddEntry(taskEntry.JobTemplateType, taskEntry.UserID, taskEntry.JobGUID, resultString, taskEntry.ID, PageLoadTime, SubmitTime);
resultdb.close();
SatyamTaskTableManagement.UpdateResultNumber(taskEntry.ID);
//SatyamTaskTableAccess taskDB = new SatyamTaskTableAccess();
//taskDB.IncrementDoneScore(taskEntry.ID);
int noDone = Convert.ToInt32(Hidden_NoImagesDone.Value);
noDone++;
Hidden_NoImagesDone.Value = noDone.ToString();
bool NotDone = false;
int noTasksPerJob = Convert.ToInt32(Hidden_TasksPerJob.Value);
if (noDone < noTasksPerJob)
{
NotDone = getNewRandomJob();
}
if (NotDone == false)
{
if (!Testing)
{
SatyamAmazonHITTableAccess HITdb = new SatyamAmazonHITTableAccess();
string HITID = result.amazonInfo.HITID;
HITdb.UpdateStatusByHITID(HITID, HitStatus.submitted);
HITdb.close();
AmazonMTurkNotification.submitAmazonTurkHit(result.amazonInfo.AssignmentID, result.amazonInfo.WorkerID, false);
}
Response.Redirect("MTurkTaskDonePage.aspx");
}
}
}
public static bool IsAcceptable(
SatyamAggregatedResultsTableEntry aggResultEntry,
SatyamResultsTableEntry resultEntry,
double ACCEPTANCE_NUMBER_OF_POLYGONS_THRESHOLD = TaskConstants.IMAGE_SEGMENTATION_MTURK_OBJECT_COVERAGE_THRESHOLD_FOR_PAYMENT, //the person must have made at least 80% of the boxes
double POLYGON_IOU_THRESHOLD = TaskConstants.IMAGE_SEGMENTATION_MTURK_POLYGON_IOU_THRESHOLD_FOR_PAYMENT
)
{
//most boxes should be within limits
//most categories should be right
SatyamAggregatedResult satyamAggResult = JSonUtils.ConvertJSonToObject <SatyamAggregatedResult>(aggResultEntry.ResultString);
ImageSegmentationAggregatedResult_NoHoles aggresult = JSonUtils.ConvertJSonToObject <ImageSegmentationAggregatedResult_NoHoles>(satyamAggResult.AggregatedResultString);
SatyamResult satyamResult = JSonUtils.ConvertJSonToObject <SatyamResult>(resultEntry.ResultString);
ImageSegmentationResult_NoHoles result = JSonUtils.ConvertJSonToObject <ImageSegmentationResult_NoHoles>(satyamResult.TaskResult);
if (result == null)
{
return(false);
}
//first check if the number of boxes are within limit
int boxLimit = (int)Math.Ceiling((double)aggresult.boxesAndCategories.objects.Count * ACCEPTANCE_NUMBER_OF_POLYGONS_THRESHOLD);
if (result.objects.Count < boxLimit)
{
return(false);
}
//We fist do a bipartitte matching to find the best assocaition for the boxes
List <List <GenericPolygon> > allPolygons = new List <List <GenericPolygon> >();
allPolygons.Add(new List <GenericPolygon>());
foreach (ImageSegmentationResultSingleEntry_NoHoles entry in result.objects)
{
allPolygons[0].Add(entry.polygon);
}
allPolygons.Add(new List <GenericPolygon>());
List <bool> tooSmallToIgnore = new List <bool>();
foreach (ImageSegmentationResultSingleEntry_NoHoles entry in aggresult.boxesAndCategories.objects)
{
allPolygons[1].Add(entry.polygon);
}
List <MultipartiteWeightedMatch> polygonAssociation = PolygonAssociation.computeGenericPolygonAssociations(allPolygons);
//now find how many of the results match aggregated results
int noAccepted = 0;
foreach (MultipartiteWeightedMatch match in polygonAssociation)
{
if (match.elementList.ContainsKey(1)) // this contains an aggregated box
{
if (match.elementList.ContainsKey(0)) // a result box has been associated
{
GenericPolygon aggregatedGenericPolygon = allPolygons[1][match.elementList[1]];
GenericPolygon resultGenericPolygon = allPolygons[0][match.elementList[0]];
//double IoU = GenericPolygon.computeIntersectionOverUnion(aggregatedGenericPolygon, resultGenericPolygon);
double IoU = 1;
if (IoU >= POLYGON_IOU_THRESHOLD)
{
//now check category
if (result.objects[match.elementList[0]].Category == aggresult.boxesAndCategories.objects[match.elementList[1]].Category)
{
//both category and bounding box tests have passed
noAccepted++;
}
}
}
}
}
if (noAccepted >= boxLimit)
{
return(true);
}
return(false);
}
protected void SubmitButton_Click(object sender, EventArgs e)
{
DateTime SubmitTime = DateTime.Now;
DateTime PageLoadTime = Convert.ToDateTime(Hidden_PageLoadTime.Value);
if (CategorySelection_RadioButtonList.SelectedIndex != -1)
{
SatyamTaskTableEntry taskEntry = JSonUtils.ConvertJSonToObject <SatyamTaskTableEntry>(Hidden_TaskEntryString.Value);
SatyamResult result = new SatyamResult();
result.TaskParametersString = taskEntry.TaskParametersString;
result.TaskStartTime = PageLoadTime;
result.TaskEndTime = SubmitTime;
result.TaskTableEntryID = taskEntry.ID;
AmazonTaskResultInfo amazonInfo = new AmazonTaskResultInfo();
amazonInfo.AssignmentID = Hidden_AmazonAssignmentID.Value;
amazonInfo.WorkerID = Hidden_AmazonWorkerID.Value;
amazonInfo.HITID = Hidden_HITID.Value;
amazonInfo.PricePerHIT = Convert.ToDouble(Hidden_Price.Value);
result.amazonInfo = amazonInfo;
SingleObjectLabelingResult sresult = new SingleObjectLabelingResult();
sresult.Category = CategorySelection_RadioButtonList.Items[CategorySelection_RadioButtonList.SelectedIndex].Text;
string sresultString = JSonUtils.ConvertObjectToJSon <SingleObjectLabelingResult>(sresult);
result.TaskResult = sresultString;
string resultString = JSonUtils.ConvertObjectToJSon <SatyamResult>(result);
SatyamResultsTableAccess resultdb = new SatyamResultsTableAccess();
resultdb.AddEntry(taskEntry.JobTemplateType, taskEntry.UserID, taskEntry.JobGUID, resultString, taskEntry.ID, PageLoadTime, SubmitTime);
resultdb.close();
SatyamTaskTableManagement.UpdateResultNumber(taskEntry.ID);
//SatyamTaskTableAccess taskDB = new SatyamTaskTableAccess();
//taskDB.IncrementDoneScore(taskEntry.ID);
int noDone = Convert.ToInt32(Hidden_NoImagesDone.Value);
noDone++;
Hidden_NoImagesDone.Value = noDone.ToString();
int noTasksPerJob = Convert.ToInt32(Hidden_TasksPerJob.Value);
bool NotDone = false;
if (noDone < noTasksPerJob)
{
NotDone = getNewRandomJob();
}
if (NotDone == false)
{
if (!Testing)
{
SatyamAmazonHITTableAccess HITdb = new SatyamAmazonHITTableAccess();
string HITID = result.amazonInfo.HITID;
HITdb.UpdateStatusByHITID(HITID, HitStatus.submitted);
HITdb.close();
AmazonMTurkNotification.submitAmazonTurkHit(result.amazonInfo.AssignmentID, result.amazonInfo.WorkerID, false);
}
Response.Redirect("MTurkTaskDonePage.aspx");
}
}
}
// If the results were not good enough, or aggregation method changed, retargeted, users might want to reopen the job
// All we need to do: (IN STRICT ORDER)
// clear all results back to inconclusive,
// remove all aggregated results
// restore the task table as it was for the guid,
// change the job status back to launched, and
// WARNING:
// for safety, this has to be run atomically synchronously, without any parallel process. for now.
public static void reopenJobForMoreResults(string guid)
{
SatyamJobSubmissionsTableAccess jobDB = new SatyamJobSubmissionsTableAccess();
SatyamJobSubmissionsTableAccessEntry jobEntry = jobDB.getEntryByJobGIUD(guid);
jobDB.close();
if (jobEntry.JobStatus != JobStatus.completed)
{
Console.WriteLine("Job not completed yet!");
return;
}
SatyamAggregatedResultsTableAccess aggResultDB = new SatyamAggregatedResultsTableAccess();
bool delSuccess = aggResultDB.DeleteEntriesByGUID(guid);
aggResultDB.close();
if (!delSuccess)
{
Console.WriteLine("Delete Agg Result DB Failed");
return;
}
SatyamResultsTableAccess resultDB = new SatyamResultsTableAccess();
if (!resultDB.UpdateStatusByGUID(guid, ResultStatus.inconclusive))
{
Console.WriteLine("Update Result DB Failed");
//resultDB.close();
//return;
}
List <SatyamResultsTableEntry> results = resultDB.getEntriesByGUID(guid);
resultDB.close();
Dictionary <int, SatyamTask> taskParamsByTaskID = new Dictionary <int, SatyamTask>();
foreach (SatyamResultsTableEntry result in results)
{
if (taskParamsByTaskID.ContainsKey(result.SatyamTaskTableEntryID))
{
continue;
}
SatyamResult satyamRes = JSonUtils.ConvertJSonToObject <SatyamResult>(result.ResultString);
SatyamTask task = JSonUtils.ConvertJSonToObject <SatyamTask>(satyamRes.TaskParametersString);
taskParamsByTaskID.Add(result.SatyamTaskTableEntryID, task);
}
SatyamTaskTableAccess taskDB = new SatyamTaskTableAccess();
foreach (int taskID in taskParamsByTaskID.Keys)
{
SatyamTask task = taskParamsByTaskID[taskID];
SatyamJob job = task.jobEntry;
bool suc = taskDB.AddEntryWithSpecificID(taskID, job.JobTemplateType, job.UserID, job.JobGUIDString, JSonUtils.ConvertObjectToJSon(task),
job.JobSubmitTime, job.amazonHITInformation.Price);
if (!suc)
{
Console.WriteLine("Update Task Table Failed");
taskDB.close();
return;
}
}
taskDB.close();
jobDB = new SatyamJobSubmissionsTableAccess();
bool success = jobDB.UpdateEntryStatus(guid, JobStatus.launched);
jobDB.close();
if (!success)
{
Console.WriteLine("Update Job Entry Failed");
return;
}
}
public static void AggregateWithParameterAndValidateSatyamVideoClassificationResultByGUID(string guid,
int MinResults = TaskConstants.SINGLE_OBJECT_LABLING_MTURK_MIN_RESULTS_TO_AGGREGATE,
int MaxResults = TaskConstants.SINGLE_OBJECT_LABLING_MTURK_MAX_RESULTS_TO_AGGREGATE,
double MajorityThreshold = TaskConstants.SINGLE_OBJECT_LABLING_MTURK_MAJORITY_THRESHOLD,
bool prepareDataForTraining = false, string outputDirectory = null,
string confusingVideoListFilePath = null)
{
string configString = "Min_" + MinResults + "_Max_" + MaxResults + "_Thresh_" + MajorityThreshold;
Console.WriteLine("Aggregating for param set " + configString);
int noTerminatedTasks = 0;
SatyamResultsTableAccess resultsDB = new SatyamResultsTableAccess();
List <SatyamResultsTableEntry> entries = resultsDB.getEntriesByGUIDOrderByID(guid);
resultsDB.close();
SortedDictionary <DateTime, List <SatyamResultsTableEntry> > entriesBySubmitTime = SatyamResultValidationToolKit.SortResultsBySubmitTime(entries);
int noTotalConverged = 0;
int noCorrect = 0;
Dictionary <int, List <SatyamResultsTableEntry> > ResultsPerTask = new Dictionary <int, List <SatyamResultsTableEntry> >();
List <int> aggregatedTasks = new List <int>();
SortedDictionary <string, Dictionary <string, int> > confusionMatrix_res_groundtruth = new SortedDictionary <string, Dictionary <string, int> >();
SortedDictionary <string, Dictionary <string, int> > confusionMatrix_groundtruth_res = new SortedDictionary <string, Dictionary <string, int> >();
List <SatyamAggregatedResultsTableEntry> aggEntries = new List <SatyamAggregatedResultsTableEntry>();
Dictionary <int, int> noResultsNeededForAggregation = SatyamResultsAnalysis.getNoResultsNeededForAggregationFromLog(configString, guid);
Dictionary <int, int> noResultsNeededForAggregation_new = new Dictionary <int, int>();
foreach (DateTime t in entriesBySubmitTime.Keys)
{
//Console.WriteLine("Processing Results of time: {0}", t);
foreach (SatyamResultsTableEntry entry in entriesBySubmitTime[t])
{
SatyamResult satyamResult = JSonUtils.ConvertJSonToObject <SatyamResult>(entry.ResultString);
SatyamTask task = JSonUtils.ConvertJSonToObject <SatyamTask>(satyamResult.TaskParametersString);
SatyamJob job = task.jobEntry;
string fileName = URIUtilities.filenameFromURI(task.SatyamURI);
string VideoCategoryGroundTruth = getVideoCategoryFromFileName(fileName);
int taskEntryID = entry.SatyamTaskTableEntryID;
if (aggregatedTasks.Contains(taskEntryID))
{
continue;
}
if (!ResultsPerTask.ContainsKey(taskEntryID))
{
ResultsPerTask.Add(taskEntryID, new List <SatyamResultsTableEntry>());
}
ResultsPerTask[taskEntryID].Add(entry);
// check log if enough results are collected
if (noResultsNeededForAggregation != null && noResultsNeededForAggregation.ContainsKey(taskEntryID) &&
ResultsPerTask[taskEntryID].Count < noResultsNeededForAggregation[taskEntryID])
{
continue;
}
//SingleObjectLabelingAggregatedResult aggResult = SingleObjectLabelingAggregator.getAggregatedResult(ResultsPerTask[taskEntryID], MinResults, MaxResult, MajorityThreshold);
string aggResultString = SingleObjectLabelingAggregator.GetAggregatedResultString(ResultsPerTask[taskEntryID], MinResults, MaxResults, MajorityThreshold);
if (aggResultString == null)
{
continue;
}
SatyamAggregatedResultsTableEntry aggEntry = new SatyamAggregatedResultsTableEntry();
aggEntry.JobGUID = ResultsPerTask[taskEntryID][0].JobGUID;
aggEntry.JobTemplateType = ResultsPerTask[taskEntryID][0].JobTemplateType;
aggEntry.SatyamTaskTableEntryID = taskEntryID;
aggEntry.UserID = ResultsPerTask[taskEntryID][0].UserID;
aggEntry.ResultString = aggResultString;
/// aggregation happen
// record logs
if (noResultsNeededForAggregation == null || !noResultsNeededForAggregation.ContainsKey(taskEntryID))
{
noResultsNeededForAggregation_new.Add(taskEntryID, ResultsPerTask[taskEntryID].Count);
}
aggEntries.Add(aggEntry);
noTotalConverged++;
if (ResultsPerTask[taskEntryID].Count >= MaxResults)
{
noTerminatedTasks++;
}
aggregatedTasks.Add(taskEntryID);
}
}
SatyamResultsAnalysis.RecordAggregationLog(noResultsNeededForAggregation_new, configString, guid);
string outputmatFile = DirectoryConstants.defaultTempDirectory + guid + "\\" + configString + "_mat.txt";
EvaluateAndPrintConfusionMatrixOfAggregatedResultEntries(aggEntries, outputmatFile, out noCorrect, prepareDataForTraining, outputDirectory);
Console.WriteLine("noTotalConverged {0}", noTotalConverged);
Console.WriteLine("noTerminatedTasks {0}", noTerminatedTasks);
Console.WriteLine("Result: {0}/{1}, precision: {2}", noCorrect, noTotalConverged, (double)noCorrect / noTotalConverged);
/// local file
string outputString = String.Format("{0} {1} {2} {3} {4} {5}\n", configString, noCorrect, noTotalConverged, (double)noCorrect / noTotalConverged, noTerminatedTasks, ResultsPerTask.Count - noTotalConverged);
string outputfile = DirectoryConstants.defaultTempDirectory + guid + "\\resultSummary.txt";
File.AppendAllText(outputfile, outputString);
//for (double prob = 0; prob < 1;prob +=0.2)
//{
// SatyamResultsAnalysis.AnalyzeApprovalRate(aggEntries, entries, guid, configString, anotherChanceProbablity: prob);
//}
//for (double ratio = 0; ratio < 1; ratio += 0.2)
//{
// SatyamResultsAnalysis.AnalyzeApprovalRate(aggEntries, entriesBySubmitTime, noResultsNeededForAggregation, noResultsNeededForAggregation_new, guid, configString, approvalRatioThreshold: ratio);
//}
SatyamResultsAnalysis.AggregationAnalysis(aggEntries, entriesBySubmitTime, noResultsNeededForAggregation, noResultsNeededForAggregation_new, guid, configString);
}
public static void Run([QueueTrigger("judge-result")] string myQueueItem, TraceWriter log)
{
bool logging = false;
string[] fields = myQueueItem.Split('_');
string guid = fields[0];
int taskID = Convert.ToInt32(fields[1]);
string resultID = fields[2];
if (logging)
{
log.Info($"Judge Result: {myQueueItem}");
}
SatyamResultsTableAccess resultsDB = new SatyamResultsTableAccess();
List <SatyamResultsTableEntry> results = resultsDB.getEntriesID(resultID);
resultsDB.close();
if (results.Count == 0)
{
return;
}
SatyamAggregatedResultsTableAccess aggDB = new SatyamAggregatedResultsTableAccess();
SatyamAggregatedResultsTableEntry aggResult = aggDB.getEntryByTaskID(taskID);
aggDB.close();
if (aggResult == null)
{
return;
}
SatyamResultsTableEntry result = results[0];
SatyamResult res = JSonUtils.ConvertJSonToObject <SatyamResult>(result.ResultString);
string taskTemplateType = result.JobTemplateType;
string workerID = res.amazonInfo.WorkerID;
DateTime doneTime = res.TaskEndTime;
if (AcceptanceCriterionChecker.IsAcceptable(aggResult, result))
{
resultsDB = new SatyamResultsTableAccess();
resultsDB.UpdateStatusByID(result.ID, ResultStatus.accepted);
resultsDB.close();
WorkerStatisticsManagement.UpdateWorkerStatistics(workerID, taskTemplateType, true, doneTime);
if (logging)
{
log.Info($"Accepted");
}
}
else
{
resultsDB = new SatyamResultsTableAccess();
resultsDB.UpdateStatusByID(result.ID, ResultStatus.rejected);
resultsDB.close();
WorkerStatisticsManagement.UpdateWorkerStatistics(workerID, taskTemplateType, false, doneTime);
if (logging)
{
log.Info($"Rejected");
}
}
}
protected void SubmitButton_Click(object sender, EventArgs e)
{
DateTime SubmitTime = DateTime.Now;
DateTime PageLoadTime = Convert.ToDateTime(Hidden_PageLoadTime.Value);
int count;
bool isValidCount = int.TryParse(CountTextBox.Text, out count);
if (!isValidCount)
{
ErrorLabel.Text = "Error : Ivalid Count.";
ErrorLabel.ForeColor = System.Drawing.Color.Red;
ErrorLabel.Font.Bold = true;
}
else if (count < 0)
{
ErrorLabel.Text = "Error : Count cannot be less than zero.";
ErrorLabel.ForeColor = System.Drawing.Color.Red;
ErrorLabel.Font.Bold = true;
}
else
{
ErrorLabel.Text = "";
SatyamTaskTableEntry taskEntry = JSonUtils.ConvertJSonToObject <SatyamTaskTableEntry>(Hidden_TaskEntryString.Value);
SatyamResult result = new SatyamResult();
result.TaskParametersString = taskEntry.TaskParametersString;
result.TaskStartTime = PageLoadTime;
result.TaskEndTime = SubmitTime;
result.TaskTableEntryID = taskEntry.ID;
AmazonTaskResultInfo amazonInfo = new AmazonTaskResultInfo();
amazonInfo.AssignmentID = "";
amazonInfo.WorkerID = "";
amazonInfo.HITID = "";
result.amazonInfo = amazonInfo;
ObjectCountingResult sresult = new ObjectCountingResult();
sresult.Count = count;
string sresultString = JSonUtils.ConvertObjectToJSon <ObjectCountingResult>(sresult);
result.TaskResult = sresultString;
string resultString = JSonUtils.ConvertObjectToJSon <SatyamResult>(result);
SatyamResultsTableAccess resultdb = new SatyamResultsTableAccess();
resultdb.AddEntry(taskEntry.JobTemplateType, taskEntry.UserID, taskEntry.JobGUID, resultString, taskEntry.ID, PageLoadTime, SubmitTime);
resultdb.close();
//SatyamTaskTableAccess taskDB = new SatyamTaskTableAccess();
//taskDB.IncrementDoneScore(taskEntry.ID);
bool NotDone = getNewRandomJob();
if (NotDone == false)
{
Response.Redirect("AllJobsDone.aspx");
}
}
}
//protected void SubmitButton_Click(object sender, EventArgs e)
private void storeResult()
{
DateTime SubmitTime = DateTime.Now;
DateTime PageLoadTime = Convert.ToDateTime(Hidden_PageLoadTime.Value);
SatyamTaskTableEntry taskEntry = JSonUtils.ConvertJSonToObject <SatyamTaskTableEntry>(Hidden_TaskEntryString.Value);
string tracksString = TracksOutput_Hidden.Value;
string urlList = Hidden_ImageURLList.Value;
string[] fields = urlList.Split(',');
DateTime start = DateTime.MinValue;
List <DateTime> frameTimes = new List <DateTime>();
//double frameTimeSpanInMiliseconds = (Convert.ToDouble(Hidden_ChunkDuration.Value) / (double)fields.Length) * 1000;
double frameTimeSpanInMiliseconds = (double)(1000) / Convert.ToDouble(fps_Hidden.Value);
for (int i = 0; i < fields.Length; i++)
{
DateTime t;
t = start.AddMilliseconds(frameTimeSpanInMiliseconds * i);
frameTimes.Add(t);
}
string s = Raw_VATIC_DVA_Crowdsourced_Track_Collection.Raw_VATIC_DVA_Crowdsourced_Track_Collection_ToTrackStrings(tracksString, frameTimes);
SatyamResult result = new SatyamResult();
result.TaskParametersString = taskEntry.TaskParametersString;
result.TaskStartTime = PageLoadTime;
result.TaskEndTime = SubmitTime;
result.TaskTableEntryID = taskEntry.ID;
AmazonTaskResultInfo amazonInfo = new AmazonTaskResultInfo();
amazonInfo.AssignmentID = Hidden_AmazonAssignmentID.Value;
amazonInfo.WorkerID = Hidden_AmazonWorkerID.Value;
amazonInfo.HITID = Hidden_HITID.Value;
amazonInfo.PricePerHIT = Convert.ToDouble(Hidden_Price.Value);
result.amazonInfo = amazonInfo;
result.TaskResult = s;
string resultString = JSonUtils.ConvertObjectToJSon <SatyamResult>(result);
SatyamResultsTableAccess resultdb = new SatyamResultsTableAccess();
resultdb.AddEntry(taskEntry.JobTemplateType, taskEntry.UserID, taskEntry.JobGUID, resultString, taskEntry.ID, PageLoadTime, SubmitTime);
resultdb.close();
SatyamTaskTableManagement.UpdateResultNumber(taskEntry.ID);
if (!Testing)
{
SatyamAmazonHITTableAccess HITdb = new SatyamAmazonHITTableAccess();
string HITID = result.amazonInfo.HITID;
HITdb.UpdateStatusByHITID(HITID, HitStatus.submitted);
HITdb.close();
AmazonMTurkNotification.submitAmazonTurkHit(result.amazonInfo.AssignmentID, result.amazonInfo.WorkerID, false);
}
Response.Redirect("AllJobsDone.aspx");
}
