public static string GetAggregatedResultString(List <SatyamResultsTableEntry> results, int MinResults = TaskConstants.SINGLE_OBJECT_LABLING_MTURK_MIN_RESULTS_TO_AGGREGATE, int MaxResults = TaskConstants.SINGLE_OBJECT_LABLING_MTURK_MAX_RESULTS_TO_AGGREGATE, double probabilityThreshold = TaskConstants.SINGLE_OBJECT_LABLING_MTURK_MAJORITY_THRESHOLD) { string resultString = null; List <SingleObjectLabelingResult> resultList = new List <SingleObjectLabelingResult>(); foreach (SatyamResultsTableEntry entry in results) { SatyamResult res = JSonUtils.ConvertJSonToObject <SatyamResult>(entry.ResultString); SingleObjectLabelingResult taskr = JSonUtils.ConvertJSonToObject <SingleObjectLabelingResult>(res.TaskResult); resultList.Add(taskr); } SingleObjectLabelingAggregatedResult r = getAggregatedResult(resultList, MinResults, MaxResults, probabilityThreshold); if (r != null) { string rString = JSonUtils.ConvertObjectToJSon <SingleObjectLabelingAggregatedResult>(r); SatyamAggregatedResult aggResult = new SatyamAggregatedResult(); aggResult.SatyamTaskTableEntryID = results[0].SatyamTaskTableEntryID; aggResult.AggregatedResultString = rString; SatyamResult res = JSonUtils.ConvertJSonToObject <SatyamResult>(results[0].ResultString); aggResult.TaskParameters = res.TaskParametersString; resultString = JSonUtils.ConvertObjectToJSon <SatyamAggregatedResult>(aggResult); } return(resultString); }
public static string GetAggregatedResultString(List <SatyamResultsTableEntry> results, int MinResults = TaskConstants.OBJECT_COUNTING_MTURK_MIN_RESULTS_TO_AGGREGATE, int MaxResults = TaskConstants.OBJECT_COUNTING_MTURK_MAX_RESULTS_TO_AGGREGATE, double MAX_ABSOLUTE_COUNT_DEVIATION_LOWERBOUND = TaskConstants.OBJECT_COUNTING_MTURK_MAX_ABSOLUTE_COUNT_DEVIATION_LOWERBOUND, double MAX_DEVIATION_FRACTION = TaskConstants.OBJECT_COUNTING_MTURK_MAX_DEVIATION_FRACTION, double SUPER_MAJORITY_VALUE = TaskConstants.OBJECT_COUNTING_MTURK_SUPER_MAJORITY_VALUE) { Console.WriteLine("Aggregating"); string resultString = null; List <ObjectCountingResult> resultList = new List <ObjectCountingResult>(); List <SatyamResult> SatyamResultList = new List <SatyamResult>(); foreach (SatyamResultsTableEntry entry in results) { SatyamResult res = JSonUtils.ConvertJSonToObject <SatyamResult>(entry.ResultString); SatyamResultList.Add(res); ObjectCountingResult taskr = JSonUtils.ConvertJSonToObject <ObjectCountingResult>(res.TaskResult); resultList.Add(taskr); } //ObjectCountingAggregatedResult r = getAggregatedResult(resultList, MinResults, MaxResults, MAX_ABSOLUTE_COUNT_DEVIATION_LOWERBOUND, MAX_DEVIATION_FRACTION, SUPER_MAJORITY_VALUE); ObjectCountingAggregatedResult r = getAggregatedResultUsingWorkerStatistics(SatyamResultList, MinResults, MaxResults, MAX_ABSOLUTE_COUNT_DEVIATION_LOWERBOUND, MAX_DEVIATION_FRACTION, SUPER_MAJORITY_VALUE); if (r != null) { string rString = JSonUtils.ConvertObjectToJSon <ObjectCountingAggregatedResult>(r); SatyamAggregatedResult aggResult = new SatyamAggregatedResult(); aggResult.SatyamTaskTableEntryID = results[0].SatyamTaskTableEntryID; aggResult.AggregatedResultString = rString; SatyamResult res = JSonUtils.ConvertJSonToObject <SatyamResult>(results[0].ResultString); aggResult.TaskParameters = res.TaskParametersString; resultString = JSonUtils.ConvertObjectToJSon <SatyamAggregatedResult>(aggResult); } return(resultString); }
public static void SaveResultVideosLocally(List <SatyamResultsTableEntry> entries, string directoryName, int fps = 10) { directoryName = directoryName + "\\Raw"; if (!Directory.Exists(directoryName)) { Directory.CreateDirectory(directoryName); } SatyamJobStorageAccountAccess satyamStorage = new SatyamJobStorageAccountAccess(); for (int i = 0; i < entries.Count; i++) { SatyamResultsTableEntry entry = entries[i]; SatyamResult satyamResult = JSonUtils.ConvertJSonToObject <SatyamResult>(entry.ResultString); SatyamTask task = JSonUtils.ConvertJSonToObject <SatyamTask>(satyamResult.TaskParametersString); MultiObjectTrackingSubmittedJob job = JSonUtils.ConvertJSonToObject <MultiObjectTrackingSubmittedJob>(task.jobEntry.JobParameters); //VATIC_DVA_CrowdsourcedResult taskr = new VATIC_DVA_CrowdsourcedResult(satyamResult.TaskResult, task.SatyamURI, start, end, task.SatyamJobSubmissionsTableEntryID.ToString()); VATIC_DVA_CrowdsourcedResult taskr = MultiObjectTrackingAggregator.createVATIC_DVA_CrowdsourcedResultUsingSatyamBlobImageCount(satyamResult.TaskResult, task.SatyamURI, entry.ID.ToString(), job.FrameRate); MultiObjectTrackingResult res = taskr.getCompressedTracksInTimeSegment(); List <string> ImageURLs = satyamStorage.getURLListOfSubDirectoryByURL(task.SatyamURI); string[] names = task.SatyamURI.Split('/'); string videoName = names[names.Length - 2] + "_" + entry.ID; generateVideoForEvaluation(ImageURLs, res, directoryName, videoName, job.FrameRate); } }
public static string GetAggregatedResultString(List <SatyamResultsTableEntry> results) { if (results.Count == 0) { return(null); } string resultString = null; List <ImageSegmentationResult> resultList = new List <ImageSegmentationResult>(); List <string> WorkersPerTask = new List <string>(); SatyamResult res0 = JSonUtils.ConvertJSonToObject <SatyamResult>(results[0].ResultString); SatyamTask task = JSonUtils.ConvertJSonToObject <SatyamTask>(res0.TaskParametersString); string SatyamURL = task.SatyamURI; string guid = results[0].JobGUID; foreach (SatyamResultsTableEntry entry in results) { SatyamResult res = JSonUtils.ConvertJSonToObject <SatyamResult>(entry.ResultString); // remove duplicate workers result string workerID = res.amazonInfo.WorkerID; if (WorkersPerTask.Contains(workerID)) { continue; } string assignmentID = res.amazonInfo.AssignmentID; if (assignmentID == "" || assignmentID == "ASSIGNMENT_ID_NOT_AVAILABLE") { continue; } //enclose only non-duplicate results, one per each worker. if (workerID != "" && workerID != TaskConstants.AdminID) { // make a pass for test and admins WorkersPerTask.Add(workerID); } ImageSegmentationResult taskr = JSonUtils.ConvertJSonToObject <ImageSegmentationResult>(res.TaskResult); resultList.Add(taskr); } ImageSegmentationAggregatedResult r = getAggregatedResult(resultList, SatyamURL, guid); if (r != null) { string rString = JSonUtils.ConvertObjectToJSon <ImageSegmentationAggregatedResult>(r); SatyamAggregatedResult aggResult = new SatyamAggregatedResult(); aggResult.SatyamTaskTableEntryID = results[0].SatyamTaskTableEntryID; aggResult.AggregatedResultString = rString; SatyamResult res = JSonUtils.ConvertJSonToObject <SatyamResult>(results[0].ResultString); aggResult.TaskParameters = res.TaskParametersString; resultString = JSonUtils.ConvertObjectToJSon <SatyamAggregatedResult>(aggResult); } return(resultString); }
//public static string GetAggregatedResultString(List<SatyamResultsTableEntry> results, // int MinResults = TaskConstants.OBJECT_COUNTING_MTURK_MIN_RESULTS_TO_AGGREGATE, // int MaxResults = TaskConstants.OBJECT_COUNTING_MTURK_MAX_RESULTS_TO_AGGREGATE, // double MAX_ABSOLUTE_COUNT_DEVIATION_LOWERBOUND = TaskConstants.OBJECT_COUNTING_MTURK_MAX_ABSOLUTE_COUNT_DEVIATION_LOWERBOUND, // double MAX_DEVIATION_FRACTION = TaskConstants.OBJECT_COUNTING_MTURK_MAX_DEVIATION_FRACTION, // double SUPER_MAJORITY_VALUE = TaskConstants.OBJECT_COUNTING_MTURK_SUPER_MAJORITY_VALUE) //{ // string resultString = null; // List<ObjectCountingResult> resultList = new List<ObjectCountingResult>(); // foreach (SatyamResultsTableEntry entry in results) // { // SatyamResult res = JSonUtils.ConvertJSonToObject<SatyamResult>(entry.ResultString); // ObjectCountingResult taskr = JSonUtils.ConvertJSonToObject<ObjectCountingResult>(res.TaskResult); // resultList.Add(taskr); // } // ObjectCountingAggregatedResult r = getAggregatedResult(resultList, MinResults, MaxResults, MAX_ABSOLUTE_COUNT_DEVIATION_LOWERBOUND, MAX_DEVIATION_FRACTION, SUPER_MAJORITY_VALUE); // if (r != null) // { // string rString = JSonUtils.ConvertObjectToJSon<ObjectCountingAggregatedResult>(r); // SatyamAggregatedResult aggResult = new SatyamAggregatedResult(); // aggResult.SatyamTaskTableEntryID = results[0].SatyamTaskTableEntryID; // aggResult.AggregatedResultString = rString; // SatyamResult res = JSonUtils.ConvertJSonToObject<SatyamResult>(results[0].ResultString); // aggResult.TaskParameters = res.TaskParametersString; // resultString = JSonUtils.ConvertObjectToJSon<SatyamAggregatedResult>(aggResult); // } // return resultString; //} public static bool IsAcceptable(SatyamAggregatedResultsTableEntry aggResult, SatyamResultsTableEntry result, double MAX_ABSOLUTE_COUNT_DEVIATION = TaskConstants.OBJECT_COUNTING_MTURK_MAX_ABSOLUTE_COUNT_DEVIATION_FOR_PAYMENT, double MAX_DEVIATION_FRACTION = TaskConstants.OBJECT_COUNTING_MTURK_MAX_DEVIATION_FRACTION_FOR_PAYMENT) { SatyamResult res = JSonUtils.ConvertJSonToObject <SatyamResult>(result.ResultString); SatyamAggregatedResult aggres = JSonUtils.ConvertJSonToObject <SatyamAggregatedResult>(aggResult.ResultString); ObjectCountingResult r = JSonUtils.ConvertJSonToObject <ObjectCountingResult>(res.TaskResult); ObjectCountingAggregatedResult ragg = JSonUtils.ConvertJSonToObject <ObjectCountingAggregatedResult>(aggres.AggregatedResultString); double maxdev = MAX_ABSOLUTE_COUNT_DEVIATION; if (ragg.Count * MAX_DEVIATION_FRACTION > maxdev) { maxdev = ragg.Count * MAX_DEVIATION_FRACTION; } if (Math.Abs(ragg.Count - r.Count) <= maxdev) { return(true); } else { return(false); } }
public static void SaveResultVideosLocally(List <SatyamResultsTableEntry> entries, string directoryName, int fps = 10) { directoryName = directoryName + "\\Raw"; if (!Directory.Exists(directoryName)) { Directory.CreateDirectory(directoryName); } SatyamJobStorageAccountAccess satyamStorage = new SatyamJobStorageAccountAccess(); for (int i = 0; i < entries.Count; i++) { SatyamResultsTableEntry entry = entries[i]; SatyamResult satyamResult = JSonUtils.ConvertJSonToObject <SatyamResult>(entry.ResultString); SatyamTask task = JSonUtils.ConvertJSonToObject <SatyamTask>(satyamResult.TaskParametersString); MultiObjectTrackingSubmittedJob job = JSonUtils.ConvertJSonToObject <MultiObjectTrackingSubmittedJob>(task.jobEntry.JobParameters); string blobDir = URIUtilities.localDirectoryFullPathFromURI(task.SatyamURI); //VATIC_DVA_CrowdsourcedResult taskr = new VATIC_DVA_CrowdsourcedResult(satyamResult.TaskResult, task.SatyamURI, start, end, task.SatyamJobSubmissionsTableEntryID.ToString()); VATIC_DVA_CrowdsourcedResult taskr = MultiObjectTrackingAggregator.createVATIC_DVA_CrowdsourcedResultUsingSatyamBlobImageCount(satyamResult.TaskResult, blobDir, entry.ID.ToString(), job.FrameRate); MultiObjectTrackingResult res = taskr.getCompressedTracksInTimeSegment(); List <string> ImageURLs = satyamStorage.getURLListOfSpecificExtensionUnderSubDirectoryByURI(blobDir, new List <string>() { "jpg", "png" }); string videoName = URIUtilities.localDirectoryNameFromURI(blobDir) + "_" + URIUtilities.filenameFromURINoExtension(task.SatyamURI) + "_" + entry.ID; MultiObjectTrackingAnalyzer.generateVideoForEvaluation(ImageURLs, res, directoryName, videoName, job.FrameRate); } }
public static void SaveKITTIResultVideosLocally(List <SatyamResultsTableEntry> entries, string directoryName, int fps = 10) { directoryName = directoryName + "\\Raw"; if (!Directory.Exists(directoryName)) { Directory.CreateDirectory(directoryName); } SatyamJobStorageAccountAccess satyamStorage = new SatyamJobStorageAccountAccess(); for (int i = 0; i < entries.Count; i++) { SatyamResultsTableEntry entry = entries[i]; SatyamResult satyamResult = JSonUtils.ConvertJSonToObject <SatyamResult>(entry.ResultString); SatyamTask task = JSonUtils.ConvertJSonToObject <SatyamTask>(satyamResult.TaskParametersString); MultiObjectTrackingSubmittedJob job = JSonUtils.ConvertJSonToObject <MultiObjectTrackingSubmittedJob>(task.jobEntry.JobParameters); //VATIC_DVA_CrowdsourcedResult taskr = VATIC_DVA_CrowdsourcedResult.createVATIC_DVA_CrowdsourcedResultUsingSatyamBlobImageCount(satyamResult.TaskResult, task.SatyamURI, task.SatyamJobSubmissionsTableEntryID.ToString(), job.FrameRate); //MultiObjectTrackingResult res = taskr.getCompressedTracksInTimeSegment(); string[] names = task.SatyamURI.Split('/'); string videoName = names[names.Length - 2] + "_" + entry.ID; string[] fields = videoName.Split('_'); string videoSequence = fields[fields.Length - 5]; int startingFrame = Convert.ToInt32(fields[fields.Length - 2]); int maxChunkEndFrame = startingFrame + job.ChunkDuration * job.FrameRate; int noFrameOverlap = (int)(job.ChunkOverlap * job.FrameRate); if (startingFrame != 0) { startingFrame -= noFrameOverlap; } if (entry.JobGUID == "1e43a983-548d-4a2e-8161-5537eb985902") { videoSequence = getCorrectSequenceNo(videoSequence); } string videoFrameDir = DirectoryConstants.KITTITrackingImages + videoSequence; List <string> files = Directory.GetFiles(videoFrameDir).ToList(); List <string> ImageURLs = new List <string>(); for (int j = startingFrame; j < files.Count && j < maxChunkEndFrame; j++) { ImageURLs.Add(files[j]); } if (ImageURLs.Count == 0) { continue; } VATIC_DVA_CrowdsourcedResult taskr = new VATIC_DVA_CrowdsourcedResult(satyamResult.TaskResult, videoName, entry.ID.ToString(), ImageURLs.Count, job.FrameRate); MultiObjectTrackingResult res = taskr.getCompressedTracksInTimeSegment(); generateVideoForEvaluation(ImageURLs, res, directoryName, videoName, job.FrameRate); } }
//protected void SubmitButton_Click(object sender, EventArgs e) private void storeResult() { DateTime SubmitTime = DateTime.Now; DateTime PageLoadTime = Convert.ToDateTime(Hidden_PageLoadTime.Value); SatyamTaskTableEntry taskEntry = JSonUtils.ConvertJSonToObject <SatyamTaskTableEntry>(Hidden_TaskEntryString.Value); string tracksString = TracksOutput_Hidden.Value; string urlList = Hidden_ImageURLList.Value; string[] fields = urlList.Split(','); DateTime start = DateTime.MinValue; List <DateTime> frameTimes = new List <DateTime>(); //double frameTimeSpanInMiliseconds = (Convert.ToDouble(Hidden_ChunkDuration.Value) / (double)fields.Length) * 1000; double frameTimeSpanInMiliseconds = (double)(1000) / Convert.ToDouble(fps_Hidden.Value); for (int i = 0; i < fields.Length; i++) { DateTime t; t = start.AddMilliseconds(frameTimeSpanInMiliseconds * i); frameTimes.Add(t); } string s = Raw_VATIC_DVA_Crowdsourced_Track_Collection.Raw_VATIC_DVA_Crowdsourced_Track_Collection_ToTrackStrings(tracksString, frameTimes); SatyamResult result = new SatyamResult(); result.TaskParametersString = taskEntry.TaskParametersString; result.TaskStartTime = PageLoadTime; result.TaskEndTime = SubmitTime; result.TaskTableEntryID = taskEntry.ID; AmazonTaskResultInfo amazonInfo = new AmazonTaskResultInfo(); amazonInfo.AssignmentID = ""; amazonInfo.WorkerID = ""; amazonInfo.HITID = ""; result.amazonInfo = amazonInfo; result.TaskResult = s; string resultString = JSonUtils.ConvertObjectToJSon <SatyamResult>(result); SatyamResultsTableAccess resultdb = new SatyamResultsTableAccess(); resultdb.AddEntry(taskEntry.JobTemplateType, taskEntry.UserID, taskEntry.JobGUID, resultString, taskEntry.ID, PageLoadTime, SubmitTime); resultdb.close(); //SatyamTaskTableAccess taskDB = new SatyamTaskTableAccess(); //taskDB.IncrementDoneScore(taskEntry.ID); bool NotDone = getNewRandomJob(); if (NotDone == false) { Response.Redirect("AllJobsDone.aspx"); } }
public static void SaveResultOnlyText(string jobGUID) { string directoryName = DirectoryConstants.defaultTempDirectory + "\\" + jobGUID + "\\"; SatyamResultsTableAccess resultsDB = new SatyamResultsTableAccess(); List <SatyamResultsTableEntry> entries = resultsDB.getEntriesByGUID(jobGUID); resultsDB.close(); if (!Directory.Exists(directoryName)) { Directory.CreateDirectory(directoryName); } directoryName = directoryName + "\\Raw\\"; if (!Directory.Exists(directoryName)) { Directory.CreateDirectory(directoryName); } for (int i = 0; i < entries.Count; i++) { SatyamResultsTableEntry entry = entries[i]; SatyamResult satyamResult = JSonUtils.ConvertJSonToObject <SatyamResult>(entry.ResultString); SatyamTask task = JSonUtils.ConvertJSonToObject <SatyamTask>(satyamResult.TaskParametersString); SatyamJob job = task.jobEntry; string result = satyamResult.TaskResult; string ofilename = URIUtilities.filenameFromURI(task.SatyamURI); string[] fields = ofilename.Split('.'); string fileName = ""; for (int j = 0; j < fields.Count(); j++) { fileName += fields[j]; if (j == fields.Count() - 1) { break; } fileName += "."; } //string fileName = ofilename; fileName = fileName + "-Result"; fileName = fileName + "-" + entry.ID; Console.WriteLine("Saving " + fileName); string resultFile = directoryName + fileName + ".txt"; StreamWriter f = new System.IO.StreamWriter(resultFile); f.WriteLine(result); f.Close(); } }
public static void SaveResultImagesLocally(List <SatyamResultsTableEntry> entries, string directoryName) { if (!Directory.Exists(directoryName)) { Directory.CreateDirectory(directoryName); } directoryName = directoryName + "\\Raw\\"; if (!Directory.Exists(directoryName)) { Directory.CreateDirectory(directoryName); } for (int i = 0; i < entries.Count; i++) { SatyamResultsTableEntry entry = entries[i]; SatyamResult satyamResult = JSonUtils.ConvertJSonToObject <SatyamResult>(entry.ResultString); SatyamTask task = JSonUtils.ConvertJSonToObject <SatyamTask>(satyamResult.TaskParametersString); SatyamJob job = task.jobEntry; string result = satyamResult.TaskResult; Image originalImage = ImageUtilities.getImageFromURI(task.SatyamURI); Image ResultImage = DrawResultStringOnImage(result, originalImage); string ofilename = URIUtilities.filenameFromURI(task.SatyamURI); string[] fields = ofilename.Split('.'); string fileName = ""; for (int j = 0; j < fields.Count(); j++) { fileName += fields[j]; if (j == fields.Count() - 1) { break; } fileName += "."; } //string fileName = ofilename; fileName = fileName + "-Result"; fileName = fileName + "-" + entry.ID; Console.WriteLine("Saving " + fileName); ImageUtilities.saveImage(ResultImage, directoryName, fileName); string resultFile = directoryName + fileName + ".txt"; StreamWriter f = new System.IO.StreamWriter(resultFile); f.WriteLine(result); f.Close(); } }
//protected void Page_Unload(object sender, EventArgs e) //{ // if (!DoneFlag) // { // SatyamTaskTableEntry taskEntry = JSonUtils.ConvertJSonToObject<SatyamTaskTableEntry>(Hidden_TaskEntryString.Value); // if (taskEntry == null) return; // // Decrement the DoneScore, releasing token after latest results are saved. // SatyamTaskTableAccess taskTableDB = new SatyamTaskTableAccess(); // taskTableDB.DecrementDoneScore(taskEntry.ID); // taskTableDB.close(); // } //} protected void SubmitButton_Click(object sender, EventArgs e) { DateTime SubmitTime = DateTime.Now; DateTime PageLoadTime = Convert.ToDateTime(Hidden_PageLoadTime.Value); SatyamTaskTableEntry taskEntry = JSonUtils.ConvertJSonToObject <SatyamTaskTableEntry>(Hidden_TaskEntryString.Value); SatyamResult result = new SatyamResult(); result.TaskParametersString = taskEntry.TaskParametersString; result.TaskStartTime = PageLoadTime; result.TaskEndTime = SubmitTime; result.TaskTableEntryID = taskEntry.ID; AmazonTaskResultInfo amazonInfo = new AmazonTaskResultInfo(); amazonInfo.AssignmentID = ""; amazonInfo.WorkerID = ""; amazonInfo.HITID = ""; result.amazonInfo = amazonInfo; result.TaskResult = Hidden_Result.Value; result.PrevResultID = Convert.ToInt32(Hidden_PrevResultID.Value); string resultString = JSonUtils.ConvertObjectToJSon <SatyamResult>(result); SatyamResultsTableAccess resultdb = new SatyamResultsTableAccess(); resultdb.AddEntry(taskEntry.JobTemplateType, taskEntry.UserID, taskEntry.JobGUID, resultString, taskEntry.ID, PageLoadTime, SubmitTime); resultdb.close(); SatyamTaskTableManagement.UpdateResultNumber(taskEntry.ID); // Decrement the DoneScore, releasing token after latest results are saved. SatyamTaskTableAccess taskTableDB = new SatyamTaskTableAccess(); //taskTableDB.DecrementDoneScore(taskEntry.ID); taskTableDB.UpdateDoneScore(taskEntry.ID, 0); taskTableDB.close(); //DoneFlag = true; //bool NotDone = getNewRandomJob(); //if (NotDone == false) //{ Response.Redirect("AllJobsDone.aspx"); //} }
public static List <SatyamResultsTableEntry> getAllEntriesByGUIDFilteredByHITID(string guid, string hitid) { List <SatyamResultsTableEntry> ret = new List <SatyamResultsTableEntry>(); SatyamResultsTableAccess resDB = new SatyamResultsTableAccess(); List <SatyamResultsTableEntry> res = resDB.getEntriesByGUID(guid); resDB.close(); foreach (SatyamResultsTableEntry r in res) { SatyamResult rr = JSonUtils.ConvertJSonToObject <SatyamResult>(r.ResultString); if (rr.amazonInfo.HITID == hitid) { ret.Add(r); } } return(ret); }
public static bool IsAcceptable(SatyamAggregatedResultsTableEntry aggResult, SatyamResultsTableEntry result) { SatyamResult res = JSonUtils.ConvertJSonToObject <SatyamResult>(result.ResultString); SatyamAggregatedResult aggres = JSonUtils.ConvertJSonToObject <SatyamAggregatedResult>(aggResult.ResultString); SingleObjectLabelingResult r = JSonUtils.ConvertJSonToObject <SingleObjectLabelingResult>(res.TaskResult); SingleObjectLabelingAggregatedResult ragg = JSonUtils.ConvertJSonToObject <SingleObjectLabelingAggregatedResult>(aggres.AggregatedResultString); if (r.Category == ragg.Category) { return(true); } else { return(false); } }
public List <int> getAllTaskIDsAWorkerHasDoneForAGUID(string WorkerID, string guid) { List <int> ret = new List <int>(); List <SatyamResultsTableEntry> entries = getEntriesByGUID(guid); foreach (SatyamResultsTableEntry entry in entries) { SatyamResult result = JSonUtils.ConvertJSonToObject <SatyamResult>(entry.ResultString); if (result.amazonInfo.WorkerID == WorkerID) { ret.Add(result.TaskTableEntryID); } } return(ret); }
protected void SubmitButton_Click(object sender, EventArgs e) { DateTime SubmitTime = DateTime.Now; DateTime PageLoadTime = Convert.ToDateTime(Hidden_PageLoadTime.Value); SatyamTaskTableEntry taskEntry = JSonUtils.ConvertJSonToObject <SatyamTaskTableEntry>(Hidden_TaskEntryString.Value); SatyamResult result = new SatyamResult(); result.TaskParametersString = taskEntry.TaskParametersString; result.TaskStartTime = PageLoadTime; result.TaskEndTime = SubmitTime; result.TaskTableEntryID = taskEntry.ID; AmazonTaskResultInfo amazonInfo = new AmazonTaskResultInfo(); amazonInfo.AssignmentID = Hidden_AmazonAssignmentID.Value; amazonInfo.WorkerID = Hidden_AmazonWorkerID.Value; amazonInfo.HITID = Hidden_HITID.Value; amazonInfo.PricePerHIT = Convert.ToDouble(Hidden_Price.Value); result.amazonInfo = amazonInfo; result.TaskResult = Hidden_Result.Value; string resultString = JSonUtils.ConvertObjectToJSon <SatyamResult>(result); SatyamResultsTableAccess resultdb = new SatyamResultsTableAccess(); resultdb.AddEntry(taskEntry.JobTemplateType, taskEntry.UserID, taskEntry.JobGUID, resultString, taskEntry.ID, PageLoadTime, SubmitTime); resultdb.close(); SatyamTaskTableManagement.UpdateResultNumber(taskEntry.ID); if (!Testing) { SatyamAmazonHITTableAccess HITdb = new SatyamAmazonHITTableAccess(); string HITID = result.amazonInfo.HITID; HITdb.UpdateStatusByHITID(HITID, HitStatus.submitted); HITdb.close(); AmazonMTurkNotification.submitAmazonTurkHit(result.amazonInfo.AssignmentID, result.amazonInfo.WorkerID, false); } Response.Redirect("AllJobsDone.aspx"); //} }
private MultiObjectLocalizationAndLabelingResult LoadLatestTurkerResult(SatyamTaskTableEntry entry) { SatyamResultsTableAccess resultsDB = new SatyamResultsTableAccess(); List <SatyamResultsTableEntry> entries = resultsDB.getEntriesByGUIDAndTaskID(entry.JobGUID, entry.ID); if (entries.Count == 0) { return(null); } //organized sequentially SatyamResultsTableEntry prevResult = entries[entries.Count - 1]; Hidden_PrevResultID.Value = prevResult.ID.ToString(); SatyamResult satyamResult = JSonUtils.ConvertJSonToObject <SatyamResult>(prevResult.ResultString); MultiObjectLocalizationAndLabelingResult res = JSonUtils.ConvertJSonToObject <MultiObjectLocalizationAndLabelingResult>(satyamResult.TaskResult); return(res); }
protected void SubmitButton_Click(object sender, EventArgs e) { DateTime SubmitTime = DateTime.Now; DateTime PageLoadTime = Convert.ToDateTime(Hidden_PageLoadTime.Value); if (CategorySelection_RadioButtonList.SelectedIndex != -1) { SatyamTaskTableEntry taskEntry = JSonUtils.ConvertJSonToObject <SatyamTaskTableEntry>(Hidden_TaskEntryString.Value); SatyamResult result = new SatyamResult(); result.TaskParametersString = taskEntry.TaskParametersString; result.TaskStartTime = PageLoadTime; result.TaskEndTime = SubmitTime; result.TaskTableEntryID = taskEntry.ID; AmazonTaskResultInfo amazonInfo = new AmazonTaskResultInfo(); amazonInfo.AssignmentID = ""; amazonInfo.WorkerID = ""; amazonInfo.HITID = ""; result.amazonInfo = amazonInfo; SingleObjectLabelingResult sresult = new SingleObjectLabelingResult(); sresult.Category = CategorySelection_RadioButtonList.Items[CategorySelection_RadioButtonList.SelectedIndex].Text; string sresultString = JSonUtils.ConvertObjectToJSon <SingleObjectLabelingResult>(sresult); result.TaskResult = sresultString; string resultString = JSonUtils.ConvertObjectToJSon <SatyamResult>(result); SatyamResultsTableAccess resultdb = new SatyamResultsTableAccess(); resultdb.AddEntry(taskEntry.JobTemplateType, taskEntry.UserID, taskEntry.JobGUID, resultString, taskEntry.ID, PageLoadTime, SubmitTime); resultdb.close(); //SatyamTaskTableAccess taskDB = new SatyamTaskTableAccess(); //taskDB.IncrementDoneScore(taskEntry.ID); bool status = getNewRandomJob(); if (!status) { Response.Redirect("AllJobsDone.aspx"); } } }
public static MultiObjectLabelingAndLocalizationAnalysisPerJob Analyse(List <SatyamResultsTableEntry> entries) { MultiObjectLabelingAndLocalizationAnalysisPerJob ana = new MultiObjectLabelingAndLocalizationAnalysisPerJob(); ana.jobTemplateType = entries[0].JobTemplateType; for (int i = 0; i < entries.Count; i++) { SatyamResultsTableEntry entry = entries[i]; SatyamResult satyamResult = JSonUtils.ConvertJSonToObject <SatyamResult>(entry.ResultString); SatyamTask task = JSonUtils.ConvertJSonToObject <SatyamTask>(satyamResult.TaskParametersString); SatyamJob job = task.jobEntry; MultiObjectLocalizationAndLabelingResult res = JSonUtils.ConvertJSonToObject <MultiObjectLocalizationAndLabelingResult>(satyamResult.TaskResult); } return(ana); }
public static void SaveByGUIDRequester(string guid) { //get all the results of a GUID SatyamResultsTableAccess resultsDB = new SatyamResultsTableAccess(); List <SatyamResultsTableEntry> results = resultsDB.getEntriesByGUID(guid); if (results.Count == 0) { resultsDB.close(); return; } //get the azure info SatyamResult result = JSonUtils.ConvertJSonToObject <SatyamResult>(results[0].ResultString); SatyamTask task = JSonUtils.ConvertJSonToObject <SatyamTask>(result.TaskParametersString); SatyamJob job = task.jobEntry; string ConnectionString = job.azureInformation.AzureBlobStorageConnectionString; string ContainerName = job.azureInformation.AzureBlobStorageContainerName; string DirectoryName = job.azureInformation.AzureBlobStorageContainerDirectoryName; //create the data to be saved List <SatyamResultSaveDataRequester> savingDataList = new List <SatyamResultSaveDataRequester>(); StringBuilder s = new StringBuilder(); for (int i = 0; i < results.Count; i++) { SatyamResultSaveDataRequester data = new SatyamResultSaveDataRequester(results[i]); String jsonString = JSonUtils.ConvertObjectToJSon(data); s.Append(jsonString); if (i == results.Count - 1) { s.Append("\n"); } } string dataToBeSaved = s.ToString(); BlobContainerManager bcm = new BlobContainerManager(); bcm.Connect(ConnectionString); string FileName = "Results-" + results[0].JobGUID + ".txt"; bcm.SaveATextFile(ContainerName, DirectoryName, FileName, dataToBeSaved); resultsDB.close(); }
public static void TestGetGlobalStatistics() { Dictionary <string, int> workerSubmissions = new Dictionary <string, int>(); Dictionary <string, int> HitSubmissions = new Dictionary <string, int>(); Dictionary <int, int> TaskSubmissions = new Dictionary <int, int>(); SatyamResultsTableAccess resultsDB = new SatyamResultsTableAccess(); List <SatyamResultsTableEntry> entries = new List <SatyamResultsTableEntry>(); entries = (resultsDB.getAllEntries()); for (int i = 0; i < entries.Count; i++) { SatyamResultsTableEntry entry = entries[i]; SatyamResult satyamResult = JSonUtils.ConvertJSonToObject <SatyamResult>(entry.ResultString); SatyamTask task = JSonUtils.ConvertJSonToObject <SatyamTask>(satyamResult.TaskParametersString); SatyamJob job = task.jobEntry; string worker = satyamResult.amazonInfo.WorkerID; string hitID = satyamResult.amazonInfo.HITID; int taskID = entry.SatyamTaskTableEntryID; if (!workerSubmissions.ContainsKey(worker)) { workerSubmissions.Add(worker, 0); } workerSubmissions[worker]++; if (!HitSubmissions.ContainsKey(hitID)) { HitSubmissions.Add(hitID, 0); } HitSubmissions[hitID]++; if (!TaskSubmissions.ContainsKey(taskID)) { TaskSubmissions.Add(taskID, 0); } TaskSubmissions[taskID]++; } Console.WriteLine(workerSubmissions.Count); Console.WriteLine(HitSubmissions.Count); Console.WriteLine(TaskSubmissions.Count); }
protected void SubmitButton_Click(object sender, EventArgs e) { DateTime SubmitTime = DateTime.Now; DateTime PageLoadTime = Convert.ToDateTime(Hidden_PageLoadTime.Value); SatyamTaskTableEntry taskEntry = JSonUtils.ConvertJSonToObject <SatyamTaskTableEntry>(Hidden_TaskEntryString.Value); SatyamResult result = new SatyamResult(); result.TaskParametersString = taskEntry.TaskParametersString; result.TaskStartTime = PageLoadTime; result.TaskEndTime = SubmitTime; result.TaskTableEntryID = taskEntry.ID; AmazonTaskResultInfo amazonInfo = new AmazonTaskResultInfo(); amazonInfo.AssignmentID = ""; amazonInfo.WorkerID = ""; amazonInfo.HITID = ""; result.amazonInfo = amazonInfo; result.TaskResult = Hidden_Result.Value; string resultString = JSonUtils.ConvertObjectToJSon <SatyamResult>(result); SatyamResultsTableAccess resultdb = new SatyamResultsTableAccess(); resultdb.AddEntry(taskEntry.JobTemplateType, taskEntry.UserID, taskEntry.JobGUID, resultString, taskEntry.ID, PageLoadTime, SubmitTime); resultdb.close(); //SatyamTaskTableAccess taskDB = new SatyamTaskTableAccess(); //taskDB.IncrementDoneScore(taskEntry.ID); bool NotDone = getNewRandomJob(); if (NotDone == false) { Response.Redirect("AllJobsDone.aspx"); } }
public SatyamResultSaveDataRequester(SatyamResultsTableEntry entry) { string status = entry.Status; if (status == ResultStatus.accepted || status == ResultStatus.accepted_Paid || status == ResultStatus.accepted_NotPaid) { resultAccepted = true; } else { resultAccepted = false; } if (status == ResultStatus.rejected_Paid || status == ResultStatus.accepted_Paid) { resultPaid = true; } else { resultPaid = false; } SatyamResult result = JSonUtils.ConvertJSonToObject <SatyamResult>(entry.ResultString); TaskStartTime = result.TaskStartTime; TaskEndTime = result.TaskEndTime; JobGUID = entry.JobGUID; JobTemplateType = entry.JobTemplateType; SatyamTask task = JSonUtils.ConvertJSonToObject <SatyamTask>(result.TaskParametersString); //OrignialURI = task.OriginalURI; SatyamJob job = task.jobEntry; UserID = job.UserID; JobSubmitTime = job.JobSubmitTime; JobParameters = job.JobParameters; amazonInfo = new AmazonSavingInfo(); amazonInfo.AssignmentID = result.amazonInfo.AssignmentID; amazonInfo.pricePerHIT = result.amazonInfo.PricePerHIT; amazonInfo.TasksPerHIT = job.TasksPerJob; amazonInfo.WorkerID = result.amazonInfo.WorkerID; ResultString = result.TaskResult; }
protected void SubmitButton_Click(object sender, EventArgs e) { DateTime SubmitTime = DateTime.Now; DateTime PageLoadTime = Convert.ToDateTime(Hidden_PageLoadTime.Value); int count; bool isValidCount = int.TryParse(CountTextBox.Text, out count); if (!isValidCount) { ErrorLabel.Text = "Error : Ivalid Count."; ErrorLabel.ForeColor = System.Drawing.Color.Red; ErrorLabel.Font.Bold = true; } else if (count < 0) { ErrorLabel.Text = "Error : Count cannot be less than zero."; ErrorLabel.ForeColor = System.Drawing.Color.Red; ErrorLabel.Font.Bold = true; } else { ErrorLabel.Text = ""; CountTextBox.Text = ""; SatyamTaskTableEntry taskEntry = JSonUtils.ConvertJSonToObject <SatyamTaskTableEntry>(Hidden_TaskEntryString.Value); SatyamResult result = new SatyamResult(); result.TaskParametersString = taskEntry.TaskParametersString; result.TaskStartTime = PageLoadTime; result.TaskEndTime = SubmitTime; result.TaskTableEntryID = taskEntry.ID; AmazonTaskResultInfo amazonInfo = new AmazonTaskResultInfo(); amazonInfo.AssignmentID = Hidden_AmazonAssignmentID.Value; amazonInfo.WorkerID = Hidden_AmazonWorkerID.Value; amazonInfo.HITID = Hidden_HITID.Value; amazonInfo.PricePerHIT = Convert.ToDouble(Hidden_Price.Value); result.amazonInfo = amazonInfo; ObjectCountingResult sresult = new ObjectCountingResult(); sresult.Count = count; string sresultString = JSonUtils.ConvertObjectToJSon <ObjectCountingResult>(sresult); result.TaskResult = sresultString; string resultString = JSonUtils.ConvertObjectToJSon <SatyamResult>(result); SatyamResultsTableAccess resultdb = new SatyamResultsTableAccess(); resultdb.AddEntry(taskEntry.JobTemplateType, taskEntry.UserID, taskEntry.JobGUID, resultString, taskEntry.ID, PageLoadTime, SubmitTime); resultdb.close(); SatyamTaskTableManagement.UpdateResultNumber(taskEntry.ID); //SatyamTaskTableAccess taskDB = new SatyamTaskTableAccess(); //taskDB.IncrementDoneScore(taskEntry.ID); int noDone = Convert.ToInt32(Hidden_NoImagesDone.Value); noDone++; Hidden_NoImagesDone.Value = noDone.ToString(); bool NotDone = false; int noTasksPerJob = Convert.ToInt32(Hidden_TasksPerJob.Value); if (noDone < noTasksPerJob) { NotDone = getNewRandomJob(); } if (NotDone == false) { if (!Testing) { SatyamAmazonHITTableAccess HITdb = new SatyamAmazonHITTableAccess(); string HITID = result.amazonInfo.HITID; HITdb.UpdateStatusByHITID(HITID, HitStatus.submitted); HITdb.close(); AmazonMTurkNotification.submitAmazonTurkHit(result.amazonInfo.AssignmentID, result.amazonInfo.WorkerID, false); } Response.Redirect("MTurkTaskDonePage.aspx"); } } }
public static bool IsAcceptable( SatyamAggregatedResultsTableEntry aggResultEntry, SatyamResultsTableEntry resultEntry, double ACCEPTANCE_NUMBER_OF_POLYGONS_THRESHOLD = TaskConstants.IMAGE_SEGMENTATION_MTURK_OBJECT_COVERAGE_THRESHOLD_FOR_PAYMENT, //the person must have made at least 80% of the boxes double POLYGON_IOU_THRESHOLD = TaskConstants.IMAGE_SEGMENTATION_MTURK_POLYGON_IOU_THRESHOLD_FOR_PAYMENT ) { //most boxes should be within limits //most categories should be right SatyamAggregatedResult satyamAggResult = JSonUtils.ConvertJSonToObject <SatyamAggregatedResult>(aggResultEntry.ResultString); ImageSegmentationAggregatedResult_NoHoles aggresult = JSonUtils.ConvertJSonToObject <ImageSegmentationAggregatedResult_NoHoles>(satyamAggResult.AggregatedResultString); SatyamResult satyamResult = JSonUtils.ConvertJSonToObject <SatyamResult>(resultEntry.ResultString); ImageSegmentationResult_NoHoles result = JSonUtils.ConvertJSonToObject <ImageSegmentationResult_NoHoles>(satyamResult.TaskResult); if (result == null) { return(false); } //first check if the number of boxes are within limit int boxLimit = (int)Math.Ceiling((double)aggresult.boxesAndCategories.objects.Count * ACCEPTANCE_NUMBER_OF_POLYGONS_THRESHOLD); if (result.objects.Count < boxLimit) { return(false); } //We fist do a bipartitte matching to find the best assocaition for the boxes List <List <GenericPolygon> > allPolygons = new List <List <GenericPolygon> >(); allPolygons.Add(new List <GenericPolygon>()); foreach (ImageSegmentationResultSingleEntry_NoHoles entry in result.objects) { allPolygons[0].Add(entry.polygon); } allPolygons.Add(new List <GenericPolygon>()); List <bool> tooSmallToIgnore = new List <bool>(); foreach (ImageSegmentationResultSingleEntry_NoHoles entry in aggresult.boxesAndCategories.objects) { allPolygons[1].Add(entry.polygon); } List <MultipartiteWeightedMatch> polygonAssociation = PolygonAssociation.computeGenericPolygonAssociations(allPolygons); //now find how many of the results match aggregated results int noAccepted = 0; foreach (MultipartiteWeightedMatch match in polygonAssociation) { if (match.elementList.ContainsKey(1)) // this contains an aggregated box { if (match.elementList.ContainsKey(0)) // a result box has been associated { GenericPolygon aggregatedGenericPolygon = allPolygons[1][match.elementList[1]]; GenericPolygon resultGenericPolygon = allPolygons[0][match.elementList[0]]; //double IoU = GenericPolygon.computeIntersectionOverUnion(aggregatedGenericPolygon, resultGenericPolygon); double IoU = 1; if (IoU >= POLYGON_IOU_THRESHOLD) { //now check category if (result.objects[match.elementList[0]].Category == aggresult.boxesAndCategories.objects[match.elementList[1]].Category) { //both category and bounding box tests have passed noAccepted++; } } } } } if (noAccepted >= boxLimit) { return(true); } return(false); }
protected void SubmitButton_Click(object sender, EventArgs e) { DateTime SubmitTime = DateTime.Now; DateTime PageLoadTime = Convert.ToDateTime(Hidden_PageLoadTime.Value); if (CategorySelection_RadioButtonList.SelectedIndex != -1) { SatyamTaskTableEntry taskEntry = JSonUtils.ConvertJSonToObject <SatyamTaskTableEntry>(Hidden_TaskEntryString.Value); SatyamResult result = new SatyamResult(); result.TaskParametersString = taskEntry.TaskParametersString; result.TaskStartTime = PageLoadTime; result.TaskEndTime = SubmitTime; result.TaskTableEntryID = taskEntry.ID; AmazonTaskResultInfo amazonInfo = new AmazonTaskResultInfo(); amazonInfo.AssignmentID = Hidden_AmazonAssignmentID.Value; amazonInfo.WorkerID = Hidden_AmazonWorkerID.Value; amazonInfo.HITID = Hidden_HITID.Value; amazonInfo.PricePerHIT = Convert.ToDouble(Hidden_Price.Value); result.amazonInfo = amazonInfo; SingleObjectLabelingResult sresult = new SingleObjectLabelingResult(); sresult.Category = CategorySelection_RadioButtonList.Items[CategorySelection_RadioButtonList.SelectedIndex].Text; string sresultString = JSonUtils.ConvertObjectToJSon <SingleObjectLabelingResult>(sresult); result.TaskResult = sresultString; string resultString = JSonUtils.ConvertObjectToJSon <SatyamResult>(result); SatyamResultsTableAccess resultdb = new SatyamResultsTableAccess(); resultdb.AddEntry(taskEntry.JobTemplateType, taskEntry.UserID, taskEntry.JobGUID, resultString, taskEntry.ID, PageLoadTime, SubmitTime); resultdb.close(); SatyamTaskTableManagement.UpdateResultNumber(taskEntry.ID); //SatyamTaskTableAccess taskDB = new SatyamTaskTableAccess(); //taskDB.IncrementDoneScore(taskEntry.ID); int noDone = Convert.ToInt32(Hidden_NoImagesDone.Value); noDone++; Hidden_NoImagesDone.Value = noDone.ToString(); int noTasksPerJob = Convert.ToInt32(Hidden_TasksPerJob.Value); bool NotDone = false; if (noDone < noTasksPerJob) { NotDone = getNewRandomJob(); } if (NotDone == false) { if (!Testing) { SatyamAmazonHITTableAccess HITdb = new SatyamAmazonHITTableAccess(); string HITID = result.amazonInfo.HITID; HITdb.UpdateStatusByHITID(HITID, HitStatus.submitted); HITdb.close(); AmazonMTurkNotification.submitAmazonTurkHit(result.amazonInfo.AssignmentID, result.amazonInfo.WorkerID, false); } Response.Redirect("MTurkTaskDonePage.aspx"); } } }
// If the results were not good enough, or aggregation method changed, retargeted, users might want to reopen the job // All we need to do: (IN STRICT ORDER) // clear all results back to inconclusive, // remove all aggregated results // restore the task table as it was for the guid, // change the job status back to launched, and // WARNING: // for safety, this has to be run atomically synchronously, without any parallel process. for now. public static void reopenJobForMoreResults(string guid) { SatyamJobSubmissionsTableAccess jobDB = new SatyamJobSubmissionsTableAccess(); SatyamJobSubmissionsTableAccessEntry jobEntry = jobDB.getEntryByJobGIUD(guid); jobDB.close(); if (jobEntry.JobStatus != JobStatus.completed) { Console.WriteLine("Job not completed yet!"); return; } SatyamAggregatedResultsTableAccess aggResultDB = new SatyamAggregatedResultsTableAccess(); bool delSuccess = aggResultDB.DeleteEntriesByGUID(guid); aggResultDB.close(); if (!delSuccess) { Console.WriteLine("Delete Agg Result DB Failed"); return; } SatyamResultsTableAccess resultDB = new SatyamResultsTableAccess(); if (!resultDB.UpdateStatusByGUID(guid, ResultStatus.inconclusive)) { Console.WriteLine("Update Result DB Failed"); //resultDB.close(); //return; } List <SatyamResultsTableEntry> results = resultDB.getEntriesByGUID(guid); resultDB.close(); Dictionary <int, SatyamTask> taskParamsByTaskID = new Dictionary <int, SatyamTask>(); foreach (SatyamResultsTableEntry result in results) { if (taskParamsByTaskID.ContainsKey(result.SatyamTaskTableEntryID)) { continue; } SatyamResult satyamRes = JSonUtils.ConvertJSonToObject <SatyamResult>(result.ResultString); SatyamTask task = JSonUtils.ConvertJSonToObject <SatyamTask>(satyamRes.TaskParametersString); taskParamsByTaskID.Add(result.SatyamTaskTableEntryID, task); } SatyamTaskTableAccess taskDB = new SatyamTaskTableAccess(); foreach (int taskID in taskParamsByTaskID.Keys) { SatyamTask task = taskParamsByTaskID[taskID]; SatyamJob job = task.jobEntry; bool suc = taskDB.AddEntryWithSpecificID(taskID, job.JobTemplateType, job.UserID, job.JobGUIDString, JSonUtils.ConvertObjectToJSon(task), job.JobSubmitTime, job.amazonHITInformation.Price); if (!suc) { Console.WriteLine("Update Task Table Failed"); taskDB.close(); return; } } taskDB.close(); jobDB = new SatyamJobSubmissionsTableAccess(); bool success = jobDB.UpdateEntryStatus(guid, JobStatus.launched); jobDB.close(); if (!success) { Console.WriteLine("Update Job Entry Failed"); return; } }
public static void AggregateWithParameterAndValidateSatyamVideoClassificationResultByGUID(string guid, int MinResults = TaskConstants.SINGLE_OBJECT_LABLING_MTURK_MIN_RESULTS_TO_AGGREGATE, int MaxResults = TaskConstants.SINGLE_OBJECT_LABLING_MTURK_MAX_RESULTS_TO_AGGREGATE, double MajorityThreshold = TaskConstants.SINGLE_OBJECT_LABLING_MTURK_MAJORITY_THRESHOLD, bool prepareDataForTraining = false, string outputDirectory = null, string confusingVideoListFilePath = null) { string configString = "Min_" + MinResults + "_Max_" + MaxResults + "_Thresh_" + MajorityThreshold; Console.WriteLine("Aggregating for param set " + configString); int noTerminatedTasks = 0; SatyamResultsTableAccess resultsDB = new SatyamResultsTableAccess(); List <SatyamResultsTableEntry> entries = resultsDB.getEntriesByGUIDOrderByID(guid); resultsDB.close(); SortedDictionary <DateTime, List <SatyamResultsTableEntry> > entriesBySubmitTime = SatyamResultValidationToolKit.SortResultsBySubmitTime(entries); int noTotalConverged = 0; int noCorrect = 0; Dictionary <int, List <SatyamResultsTableEntry> > ResultsPerTask = new Dictionary <int, List <SatyamResultsTableEntry> >(); List <int> aggregatedTasks = new List <int>(); SortedDictionary <string, Dictionary <string, int> > confusionMatrix_res_groundtruth = new SortedDictionary <string, Dictionary <string, int> >(); SortedDictionary <string, Dictionary <string, int> > confusionMatrix_groundtruth_res = new SortedDictionary <string, Dictionary <string, int> >(); List <SatyamAggregatedResultsTableEntry> aggEntries = new List <SatyamAggregatedResultsTableEntry>(); Dictionary <int, int> noResultsNeededForAggregation = SatyamResultsAnalysis.getNoResultsNeededForAggregationFromLog(configString, guid); Dictionary <int, int> noResultsNeededForAggregation_new = new Dictionary <int, int>(); foreach (DateTime t in entriesBySubmitTime.Keys) { //Console.WriteLine("Processing Results of time: {0}", t); foreach (SatyamResultsTableEntry entry in entriesBySubmitTime[t]) { SatyamResult satyamResult = JSonUtils.ConvertJSonToObject <SatyamResult>(entry.ResultString); SatyamTask task = JSonUtils.ConvertJSonToObject <SatyamTask>(satyamResult.TaskParametersString); SatyamJob job = task.jobEntry; string fileName = URIUtilities.filenameFromURI(task.SatyamURI); string VideoCategoryGroundTruth = getVideoCategoryFromFileName(fileName); int taskEntryID = entry.SatyamTaskTableEntryID; if (aggregatedTasks.Contains(taskEntryID)) { continue; } if (!ResultsPerTask.ContainsKey(taskEntryID)) { ResultsPerTask.Add(taskEntryID, new List <SatyamResultsTableEntry>()); } ResultsPerTask[taskEntryID].Add(entry); // check log if enough results are collected if (noResultsNeededForAggregation != null && noResultsNeededForAggregation.ContainsKey(taskEntryID) && ResultsPerTask[taskEntryID].Count < noResultsNeededForAggregation[taskEntryID]) { continue; } //SingleObjectLabelingAggregatedResult aggResult = SingleObjectLabelingAggregator.getAggregatedResult(ResultsPerTask[taskEntryID], MinResults, MaxResult, MajorityThreshold); string aggResultString = SingleObjectLabelingAggregator.GetAggregatedResultString(ResultsPerTask[taskEntryID], MinResults, MaxResults, MajorityThreshold); if (aggResultString == null) { continue; } SatyamAggregatedResultsTableEntry aggEntry = new SatyamAggregatedResultsTableEntry(); aggEntry.JobGUID = ResultsPerTask[taskEntryID][0].JobGUID; aggEntry.JobTemplateType = ResultsPerTask[taskEntryID][0].JobTemplateType; aggEntry.SatyamTaskTableEntryID = taskEntryID; aggEntry.UserID = ResultsPerTask[taskEntryID][0].UserID; aggEntry.ResultString = aggResultString; /// aggregation happen // record logs if (noResultsNeededForAggregation == null || !noResultsNeededForAggregation.ContainsKey(taskEntryID)) { noResultsNeededForAggregation_new.Add(taskEntryID, ResultsPerTask[taskEntryID].Count); } aggEntries.Add(aggEntry); noTotalConverged++; if (ResultsPerTask[taskEntryID].Count >= MaxResults) { noTerminatedTasks++; } aggregatedTasks.Add(taskEntryID); } } SatyamResultsAnalysis.RecordAggregationLog(noResultsNeededForAggregation_new, configString, guid); string outputmatFile = DirectoryConstants.defaultTempDirectory + guid + "\\" + configString + "_mat.txt"; EvaluateAndPrintConfusionMatrixOfAggregatedResultEntries(aggEntries, outputmatFile, out noCorrect, prepareDataForTraining, outputDirectory); Console.WriteLine("noTotalConverged {0}", noTotalConverged); Console.WriteLine("noTerminatedTasks {0}", noTerminatedTasks); Console.WriteLine("Result: {0}/{1}, precision: {2}", noCorrect, noTotalConverged, (double)noCorrect / noTotalConverged); /// local file string outputString = String.Format("{0} {1} {2} {3} {4} {5}\n", configString, noCorrect, noTotalConverged, (double)noCorrect / noTotalConverged, noTerminatedTasks, ResultsPerTask.Count - noTotalConverged); string outputfile = DirectoryConstants.defaultTempDirectory + guid + "\\resultSummary.txt"; File.AppendAllText(outputfile, outputString); //for (double prob = 0; prob < 1;prob +=0.2) //{ // SatyamResultsAnalysis.AnalyzeApprovalRate(aggEntries, entries, guid, configString, anotherChanceProbablity: prob); //} //for (double ratio = 0; ratio < 1; ratio += 0.2) //{ // SatyamResultsAnalysis.AnalyzeApprovalRate(aggEntries, entriesBySubmitTime, noResultsNeededForAggregation, noResultsNeededForAggregation_new, guid, configString, approvalRatioThreshold: ratio); //} SatyamResultsAnalysis.AggregationAnalysis(aggEntries, entriesBySubmitTime, noResultsNeededForAggregation, noResultsNeededForAggregation_new, guid, configString); }
public static void Run([QueueTrigger("judge-result")] string myQueueItem, TraceWriter log) { bool logging = false; string[] fields = myQueueItem.Split('_'); string guid = fields[0]; int taskID = Convert.ToInt32(fields[1]); string resultID = fields[2]; if (logging) { log.Info($"Judge Result: {myQueueItem}"); } SatyamResultsTableAccess resultsDB = new SatyamResultsTableAccess(); List <SatyamResultsTableEntry> results = resultsDB.getEntriesID(resultID); resultsDB.close(); if (results.Count == 0) { return; } SatyamAggregatedResultsTableAccess aggDB = new SatyamAggregatedResultsTableAccess(); SatyamAggregatedResultsTableEntry aggResult = aggDB.getEntryByTaskID(taskID); aggDB.close(); if (aggResult == null) { return; } SatyamResultsTableEntry result = results[0]; SatyamResult res = JSonUtils.ConvertJSonToObject <SatyamResult>(result.ResultString); string taskTemplateType = result.JobTemplateType; string workerID = res.amazonInfo.WorkerID; DateTime doneTime = res.TaskEndTime; if (AcceptanceCriterionChecker.IsAcceptable(aggResult, result)) { resultsDB = new SatyamResultsTableAccess(); resultsDB.UpdateStatusByID(result.ID, ResultStatus.accepted); resultsDB.close(); WorkerStatisticsManagement.UpdateWorkerStatistics(workerID, taskTemplateType, true, doneTime); if (logging) { log.Info($"Accepted"); } } else { resultsDB = new SatyamResultsTableAccess(); resultsDB.UpdateStatusByID(result.ID, ResultStatus.rejected); resultsDB.close(); WorkerStatisticsManagement.UpdateWorkerStatistics(workerID, taskTemplateType, false, doneTime); if (logging) { log.Info($"Rejected"); } } }
protected void SubmitButton_Click(object sender, EventArgs e) { DateTime SubmitTime = DateTime.Now; DateTime PageLoadTime = Convert.ToDateTime(Hidden_PageLoadTime.Value); int count; bool isValidCount = int.TryParse(CountTextBox.Text, out count); if (!isValidCount) { ErrorLabel.Text = "Error : Ivalid Count."; ErrorLabel.ForeColor = System.Drawing.Color.Red; ErrorLabel.Font.Bold = true; } else if (count < 0) { ErrorLabel.Text = "Error : Count cannot be less than zero."; ErrorLabel.ForeColor = System.Drawing.Color.Red; ErrorLabel.Font.Bold = true; } else { ErrorLabel.Text = ""; SatyamTaskTableEntry taskEntry = JSonUtils.ConvertJSonToObject <SatyamTaskTableEntry>(Hidden_TaskEntryString.Value); SatyamResult result = new SatyamResult(); result.TaskParametersString = taskEntry.TaskParametersString; result.TaskStartTime = PageLoadTime; result.TaskEndTime = SubmitTime; result.TaskTableEntryID = taskEntry.ID; AmazonTaskResultInfo amazonInfo = new AmazonTaskResultInfo(); amazonInfo.AssignmentID = ""; amazonInfo.WorkerID = ""; amazonInfo.HITID = ""; result.amazonInfo = amazonInfo; ObjectCountingResult sresult = new ObjectCountingResult(); sresult.Count = count; string sresultString = JSonUtils.ConvertObjectToJSon <ObjectCountingResult>(sresult); result.TaskResult = sresultString; string resultString = JSonUtils.ConvertObjectToJSon <SatyamResult>(result); SatyamResultsTableAccess resultdb = new SatyamResultsTableAccess(); resultdb.AddEntry(taskEntry.JobTemplateType, taskEntry.UserID, taskEntry.JobGUID, resultString, taskEntry.ID, PageLoadTime, SubmitTime); resultdb.close(); //SatyamTaskTableAccess taskDB = new SatyamTaskTableAccess(); //taskDB.IncrementDoneScore(taskEntry.ID); bool NotDone = getNewRandomJob(); if (NotDone == false) { Response.Redirect("AllJobsDone.aspx"); } } }
//protected void SubmitButton_Click(object sender, EventArgs e) private void storeResult() { DateTime SubmitTime = DateTime.Now; DateTime PageLoadTime = Convert.ToDateTime(Hidden_PageLoadTime.Value); SatyamTaskTableEntry taskEntry = JSonUtils.ConvertJSonToObject <SatyamTaskTableEntry>(Hidden_TaskEntryString.Value); string tracksString = TracksOutput_Hidden.Value; string urlList = Hidden_ImageURLList.Value; string[] fields = urlList.Split(','); DateTime start = DateTime.MinValue; List <DateTime> frameTimes = new List <DateTime>(); //double frameTimeSpanInMiliseconds = (Convert.ToDouble(Hidden_ChunkDuration.Value) / (double)fields.Length) * 1000; double frameTimeSpanInMiliseconds = (double)(1000) / Convert.ToDouble(fps_Hidden.Value); for (int i = 0; i < fields.Length; i++) { DateTime t; t = start.AddMilliseconds(frameTimeSpanInMiliseconds * i); frameTimes.Add(t); } string s = Raw_VATIC_DVA_Crowdsourced_Track_Collection.Raw_VATIC_DVA_Crowdsourced_Track_Collection_ToTrackStrings(tracksString, frameTimes); SatyamResult result = new SatyamResult(); result.TaskParametersString = taskEntry.TaskParametersString; result.TaskStartTime = PageLoadTime; result.TaskEndTime = SubmitTime; result.TaskTableEntryID = taskEntry.ID; AmazonTaskResultInfo amazonInfo = new AmazonTaskResultInfo(); amazonInfo.AssignmentID = Hidden_AmazonAssignmentID.Value; amazonInfo.WorkerID = Hidden_AmazonWorkerID.Value; amazonInfo.HITID = Hidden_HITID.Value; amazonInfo.PricePerHIT = Convert.ToDouble(Hidden_Price.Value); result.amazonInfo = amazonInfo; result.TaskResult = s; string resultString = JSonUtils.ConvertObjectToJSon <SatyamResult>(result); SatyamResultsTableAccess resultdb = new SatyamResultsTableAccess(); resultdb.AddEntry(taskEntry.JobTemplateType, taskEntry.UserID, taskEntry.JobGUID, resultString, taskEntry.ID, PageLoadTime, SubmitTime); resultdb.close(); SatyamTaskTableManagement.UpdateResultNumber(taskEntry.ID); if (!Testing) { SatyamAmazonHITTableAccess HITdb = new SatyamAmazonHITTableAccess(); string HITID = result.amazonInfo.HITID; HITdb.UpdateStatusByHITID(HITID, HitStatus.submitted); HITdb.close(); AmazonMTurkNotification.submitAmazonTurkHit(result.amazonInfo.AssignmentID, result.amazonInfo.WorkerID, false); } Response.Redirect("AllJobsDone.aspx"); }