public static void SaveResultImagesLocally(string jobGUID, string directoryName) { SatyamResultsTableAccess resultsDB = new SatyamResultsTableAccess(); List <SatyamResultsTableEntry> entries = resultsDB.getEntriesByGUID(jobGUID); resultsDB.close(); SaveResultImagesLocally(entries, directoryName); }
//protected void SubmitButton_Click(object sender, EventArgs e) private void storeResult() { DateTime SubmitTime = DateTime.Now; DateTime PageLoadTime = Convert.ToDateTime(Hidden_PageLoadTime.Value); SatyamTaskTableEntry taskEntry = JSonUtils.ConvertJSonToObject <SatyamTaskTableEntry>(Hidden_TaskEntryString.Value); string tracksString = TracksOutput_Hidden.Value; string urlList = Hidden_ImageURLList.Value; string[] fields = urlList.Split(','); DateTime start = DateTime.MinValue; List <DateTime> frameTimes = new List <DateTime>(); //double frameTimeSpanInMiliseconds = (Convert.ToDouble(Hidden_ChunkDuration.Value) / (double)fields.Length) * 1000; double frameTimeSpanInMiliseconds = (double)(1000) / Convert.ToDouble(fps_Hidden.Value); for (int i = 0; i < fields.Length; i++) { DateTime t; t = start.AddMilliseconds(frameTimeSpanInMiliseconds * i); frameTimes.Add(t); } string s = Raw_VATIC_DVA_Crowdsourced_Track_Collection.Raw_VATIC_DVA_Crowdsourced_Track_Collection_ToTrackStrings(tracksString, frameTimes); SatyamResult result = new SatyamResult(); result.TaskParametersString = taskEntry.TaskParametersString; result.TaskStartTime = PageLoadTime; result.TaskEndTime = SubmitTime; result.TaskTableEntryID = taskEntry.ID; AmazonTaskResultInfo amazonInfo = new AmazonTaskResultInfo(); amazonInfo.AssignmentID = ""; amazonInfo.WorkerID = ""; amazonInfo.HITID = ""; result.amazonInfo = amazonInfo; result.TaskResult = s; string resultString = JSonUtils.ConvertObjectToJSon <SatyamResult>(result); SatyamResultsTableAccess resultdb = new SatyamResultsTableAccess(); resultdb.AddEntry(taskEntry.JobTemplateType, taskEntry.UserID, taskEntry.JobGUID, resultString, taskEntry.ID, PageLoadTime, SubmitTime); resultdb.close(); //SatyamTaskTableAccess taskDB = new SatyamTaskTableAccess(); //taskDB.IncrementDoneScore(taskEntry.ID); bool NotDone = getNewRandomJob(); if (NotDone == false) { Response.Redirect("AllJobsDone.aspx"); } }
public static void SaveResultVideosLocally(string guid) { string outputDirectory = DirectoryConstants.defaultTempDirectory + guid; SatyamResultsTableAccess resultsDB = new SatyamResultsTableAccess(); List <SatyamResultsTableEntry> entries = resultsDB.getEntriesByGUID(guid); resultsDB.close(); SaveResultVideosLocally(entries, outputDirectory); }
public static void SaveResultImagesLocally(string jobGUID) { string directoryName = DirectoryConstants.defaultTempDirectory + "\\" + jobGUID + "\\"; SatyamResultsTableAccess resultsDB = new SatyamResultsTableAccess(); List <SatyamResultsTableEntry> entries = resultsDB.getEntriesByGUID(jobGUID); SaveResultImagesLocally(entries, directoryName); resultsDB.close(); }
public static void SaveResultOnlyText(string jobGUID) { string directoryName = DirectoryConstants.defaultTempDirectory + "\\" + jobGUID + "\\"; SatyamResultsTableAccess resultsDB = new SatyamResultsTableAccess(); List <SatyamResultsTableEntry> entries = resultsDB.getEntriesByGUID(jobGUID); resultsDB.close(); if (!Directory.Exists(directoryName)) { Directory.CreateDirectory(directoryName); } directoryName = directoryName + "\\Raw\\"; if (!Directory.Exists(directoryName)) { Directory.CreateDirectory(directoryName); } for (int i = 0; i < entries.Count; i++) { SatyamResultsTableEntry entry = entries[i]; SatyamResult satyamResult = JSonUtils.ConvertJSonToObject <SatyamResult>(entry.ResultString); SatyamTask task = JSonUtils.ConvertJSonToObject <SatyamTask>(satyamResult.TaskParametersString); SatyamJob job = task.jobEntry; string result = satyamResult.TaskResult; string ofilename = URIUtilities.filenameFromURI(task.SatyamURI); string[] fields = ofilename.Split('.'); string fileName = ""; for (int j = 0; j < fields.Count(); j++) { fileName += fields[j]; if (j == fields.Count() - 1) { break; } fileName += "."; } //string fileName = ofilename; fileName = fileName + "-Result"; fileName = fileName + "-" + entry.ID; Console.WriteLine("Saving " + fileName); string resultFile = directoryName + fileName + ".txt"; StreamWriter f = new System.IO.StreamWriter(resultFile); f.WriteLine(result); f.Close(); } }
public static void SaveByGUIDRequester(string guid) { //get all the results of a GUID SatyamResultsTableAccess resultsDB = new SatyamResultsTableAccess(); List <SatyamResultsTableEntry> results = resultsDB.getEntriesByGUID(guid); if (results.Count == 0) { resultsDB.close(); return; } //get the azure info SatyamResult result = JSonUtils.ConvertJSonToObject <SatyamResult>(results[0].ResultString); SatyamTask task = JSonUtils.ConvertJSonToObject <SatyamTask>(result.TaskParametersString); SatyamJob job = task.jobEntry; string ConnectionString = job.azureInformation.AzureBlobStorageConnectionString; string ContainerName = job.azureInformation.AzureBlobStorageContainerName; string DirectoryName = job.azureInformation.AzureBlobStorageContainerDirectoryName; //create the data to be saved List <SatyamResultSaveDataRequester> savingDataList = new List <SatyamResultSaveDataRequester>(); StringBuilder s = new StringBuilder(); for (int i = 0; i < results.Count; i++) { SatyamResultSaveDataRequester data = new SatyamResultSaveDataRequester(results[i]); String jsonString = JSonUtils.ConvertObjectToJSon(data); s.Append(jsonString); if (i == results.Count - 1) { s.Append("\n"); } } string dataToBeSaved = s.ToString(); BlobContainerManager bcm = new BlobContainerManager(); bcm.Connect(ConnectionString); string FileName = "Results-" + results[0].JobGUID + ".txt"; bcm.SaveATextFile(ContainerName, DirectoryName, FileName, dataToBeSaved); resultsDB.close(); }
public static void UpdateResultNumber(int taskID) { SatyamResultsTableAccess resultdb = new SatyamResultsTableAccess(); List <SatyamResultsTableEntry> res = resultdb.getEntriesByTaskID(taskID); resultdb.close(); SatyamTaskTableAccess taskDB = new SatyamTaskTableAccess(); taskDB.UpdateResultNumber(taskID, res.Count); taskDB.close(); }
public static void SaveByGUIDSatyam(string guid) { //get all the results of a GUID SatyamResultsTableAccess resultsDB = new SatyamResultsTableAccess(); List <SatyamResultsTableEntry> results = resultsDB.getEntriesByGUID(guid); if (results.Count == 0) { resultsDB.close(); return; } //create the data to be saved List <SatyamResultSaveDataSatyam> savingDataList = new List <SatyamResultSaveDataSatyam>(); StringBuilder s = new StringBuilder(); for (int i = 0; i < results.Count; i++) { SatyamResultSaveDataSatyam data = new SatyamResultSaveDataSatyam(results[i]); String jsonString = JSonUtils.ConvertObjectToJSon(data); s.Append(jsonString); if (i == results.Count - 1) { s.Append("\n"); } } string dataToBeSaved = s.ToString(); SatyamJobStorageAccountAccess storage = new SatyamJobStorageAccountAccess(); string FileName = "Results-" + results[0].JobGUID + ".txt"; string satyamDirectoryName = SatyamTaskGenerator.JobTemplateToSatyamContainerNameMap[results[0].JobTemplateType]; storage.SaveATextFile(satyamDirectoryName, results[0].JobGUID, FileName, dataToBeSaved); resultsDB.close(); }
//protected void Page_Unload(object sender, EventArgs e) //{ // if (!DoneFlag) // { // SatyamTaskTableEntry taskEntry = JSonUtils.ConvertJSonToObject<SatyamTaskTableEntry>(Hidden_TaskEntryString.Value); // if (taskEntry == null) return; // // Decrement the DoneScore, releasing token after latest results are saved. // SatyamTaskTableAccess taskTableDB = new SatyamTaskTableAccess(); // taskTableDB.DecrementDoneScore(taskEntry.ID); // taskTableDB.close(); // } //} protected void SubmitButton_Click(object sender, EventArgs e) { DateTime SubmitTime = DateTime.Now; DateTime PageLoadTime = Convert.ToDateTime(Hidden_PageLoadTime.Value); SatyamTaskTableEntry taskEntry = JSonUtils.ConvertJSonToObject <SatyamTaskTableEntry>(Hidden_TaskEntryString.Value); SatyamResult result = new SatyamResult(); result.TaskParametersString = taskEntry.TaskParametersString; result.TaskStartTime = PageLoadTime; result.TaskEndTime = SubmitTime; result.TaskTableEntryID = taskEntry.ID; AmazonTaskResultInfo amazonInfo = new AmazonTaskResultInfo(); amazonInfo.AssignmentID = ""; amazonInfo.WorkerID = ""; amazonInfo.HITID = ""; result.amazonInfo = amazonInfo; result.TaskResult = Hidden_Result.Value; result.PrevResultID = Convert.ToInt32(Hidden_PrevResultID.Value); string resultString = JSonUtils.ConvertObjectToJSon <SatyamResult>(result); SatyamResultsTableAccess resultdb = new SatyamResultsTableAccess(); resultdb.AddEntry(taskEntry.JobTemplateType, taskEntry.UserID, taskEntry.JobGUID, resultString, taskEntry.ID, PageLoadTime, SubmitTime); resultdb.close(); SatyamTaskTableManagement.UpdateResultNumber(taskEntry.ID); // Decrement the DoneScore, releasing token after latest results are saved. SatyamTaskTableAccess taskTableDB = new SatyamTaskTableAccess(); //taskTableDB.DecrementDoneScore(taskEntry.ID); taskTableDB.UpdateDoneScore(taskEntry.ID, 0); taskTableDB.close(); //DoneFlag = true; //bool NotDone = getNewRandomJob(); //if (NotDone == false) //{ Response.Redirect("AllJobsDone.aspx"); //} }
public static List <SatyamResultsTableEntry> getAllEntriesByGUIDFilteredByHITID(string guid, string hitid) { List <SatyamResultsTableEntry> ret = new List <SatyamResultsTableEntry>(); SatyamResultsTableAccess resDB = new SatyamResultsTableAccess(); List <SatyamResultsTableEntry> res = resDB.getEntriesByGUID(guid); resDB.close(); foreach (SatyamResultsTableEntry r in res) { SatyamResult rr = JSonUtils.ConvertJSonToObject <SatyamResult>(r.ResultString); if (rr.amazonInfo.HITID == hitid) { ret.Add(r); } } return(ret); }
protected void SubmitButton_Click(object sender, EventArgs e) { DateTime SubmitTime = DateTime.Now; DateTime PageLoadTime = Convert.ToDateTime(Hidden_PageLoadTime.Value); SatyamTaskTableEntry taskEntry = JSonUtils.ConvertJSonToObject <SatyamTaskTableEntry>(Hidden_TaskEntryString.Value); SatyamResult result = new SatyamResult(); result.TaskParametersString = taskEntry.TaskParametersString; result.TaskStartTime = PageLoadTime; result.TaskEndTime = SubmitTime; result.TaskTableEntryID = taskEntry.ID; AmazonTaskResultInfo amazonInfo = new AmazonTaskResultInfo(); amazonInfo.AssignmentID = Hidden_AmazonAssignmentID.Value; amazonInfo.WorkerID = Hidden_AmazonWorkerID.Value; amazonInfo.HITID = Hidden_HITID.Value; amazonInfo.PricePerHIT = Convert.ToDouble(Hidden_Price.Value); result.amazonInfo = amazonInfo; result.TaskResult = Hidden_Result.Value; string resultString = JSonUtils.ConvertObjectToJSon <SatyamResult>(result); SatyamResultsTableAccess resultdb = new SatyamResultsTableAccess(); resultdb.AddEntry(taskEntry.JobTemplateType, taskEntry.UserID, taskEntry.JobGUID, resultString, taskEntry.ID, PageLoadTime, SubmitTime); resultdb.close(); SatyamTaskTableManagement.UpdateResultNumber(taskEntry.ID); if (!Testing) { SatyamAmazonHITTableAccess HITdb = new SatyamAmazonHITTableAccess(); string HITID = result.amazonInfo.HITID; HITdb.UpdateStatusByHITID(HITID, HitStatus.submitted); HITdb.close(); AmazonMTurkNotification.submitAmazonTurkHit(result.amazonInfo.AssignmentID, result.amazonInfo.WorkerID, false); } Response.Redirect("AllJobsDone.aspx"); //} }
public static void TestAggregationAnalysis() { string guid = "86cbfe43-175a-4a5a-ba89-073f3004e7df";//20 class mturk all pascal //string guid = "61bf62b5-d796-4b3c-9de9-9c67f04cf26f";// changed taskpage string configString = "Min_" + 10 + "_Max_" + 10 + "_Majority_" + 0.5 + "_Ratio_" + 0.5; SatyamAggregatedResultsTableAccess aggDB = new SatyamAggregatedResultsTableAccess(); List <SatyamAggregatedResultsTableEntry> aggEntries = aggDB.getEntriesByGUID(guid); aggDB.close(); SatyamResultsTableAccess resDB = new SatyamResultsTableAccess(); List <SatyamResultsTableEntry> entries = resDB.getEntriesByGUID(guid); resDB.close(); List <SatyamResultsTableEntry> filteredResults = new List <SatyamResultsTableEntry>(); //foreach (SatyamResultsTableEntry entry in entries) //{ // SatyamResult res = JSonUtils.ConvertJSonToObject<SatyamResult>(entry.ResultString); // string workerID = res.amazonInfo.WorkerID; // string assignmentID = res.amazonInfo.AssignmentID; // if (assignmentID == "" || assignmentID == "ASSIGNMENT_ID_NOT_AVAILABLE") continue; // if (workerID == "") continue; // /// im seg only // ImageSegmentationResult res_imseg = JSonUtils.ConvertJSonToObject<ImageSegmentationResult>(res.TaskResult); // if (res_imseg == null) continue; // filteredResults.Add(entry); //} filteredResults = entries; SortedDictionary <DateTime, List <SatyamResultsTableEntry> > entriesBySubmitTime = SatyamResultValidationToolKit.SortResultsBySubmitTime_OneResultPerTurkerPerTask(filteredResults); SatyamResultsAnalysis.AggregationAnalysis(aggEntries, entriesBySubmitTime, null, null, guid, configString, usePreApprovalResult: false, approvalRatioThreshold: 0.5); }
public static void Run([QueueTrigger("aggregation")] string guidAndTaskID, TraceWriter log) { bool logging = false; string[] fields = guidAndTaskID.Split('_'); string guid = fields[0]; int taskID = Convert.ToInt32(fields[1]); SatyamResultsTableAccess resultsDB = new SatyamResultsTableAccess(); List <SatyamResultsTableEntry> collectedResults = resultsDB.getEntriesByGUIDAndTaskID(guid, taskID); resultsDB.close(); if (collectedResults.Count == 0) { return; } if (logging) { log.Info($"Task Type: {collectedResults[0].JobTemplateType}({guid}), Task ID: {taskID}, # of Results: {collectedResults.Count}"); } //SatyamAggregatedResultsTableAccess aggDB = new SatyamAggregatedResultsTableAccess(); //int LatestResultsAggregated = aggDB.getLatestNoResultsAggregatedByTaskID(taskID); //aggDB.close(); //if (collectedResults.Count != LatestResultsAggregated) //{ SatyamAggregatedResultsTableEntry aggEntry = ResultsTableAggregator.GetAggregatedResultString(taskID, collectedResults); if (aggEntry != null) { //aggEntries.Add(aggEntry); SatyamAggregatedResultsTableAccess aggDB = new SatyamAggregatedResultsTableAccess(); aggDB.AddEntry(aggEntry, collectedResults.Count); aggDB.close(); if (logging) { log.Info($"Aggregated"); } } //} }
protected void SubmitButton_Click(object sender, EventArgs e) { DateTime SubmitTime = DateTime.Now; DateTime PageLoadTime = Convert.ToDateTime(Hidden_PageLoadTime.Value); if (CategorySelection_RadioButtonList.SelectedIndex != -1) { SatyamTaskTableEntry taskEntry = JSonUtils.ConvertJSonToObject <SatyamTaskTableEntry>(Hidden_TaskEntryString.Value); SatyamResult result = new SatyamResult(); result.TaskParametersString = taskEntry.TaskParametersString; result.TaskStartTime = PageLoadTime; result.TaskEndTime = SubmitTime; result.TaskTableEntryID = taskEntry.ID; AmazonTaskResultInfo amazonInfo = new AmazonTaskResultInfo(); amazonInfo.AssignmentID = ""; amazonInfo.WorkerID = ""; amazonInfo.HITID = ""; result.amazonInfo = amazonInfo; SingleObjectLabelingResult sresult = new SingleObjectLabelingResult(); sresult.Category = CategorySelection_RadioButtonList.Items[CategorySelection_RadioButtonList.SelectedIndex].Text; string sresultString = JSonUtils.ConvertObjectToJSon <SingleObjectLabelingResult>(sresult); result.TaskResult = sresultString; string resultString = JSonUtils.ConvertObjectToJSon <SatyamResult>(result); SatyamResultsTableAccess resultdb = new SatyamResultsTableAccess(); resultdb.AddEntry(taskEntry.JobTemplateType, taskEntry.UserID, taskEntry.JobGUID, resultString, taskEntry.ID, PageLoadTime, SubmitTime); resultdb.close(); //SatyamTaskTableAccess taskDB = new SatyamTaskTableAccess(); //taskDB.IncrementDoneScore(taskEntry.ID); bool status = getNewRandomJob(); if (!status) { Response.Redirect("AllJobsDone.aspx"); } } }
protected void SubmitButton_Click(object sender, EventArgs e) { DateTime SubmitTime = DateTime.Now; DateTime PageLoadTime = Convert.ToDateTime(Hidden_PageLoadTime.Value); SatyamTaskTableEntry taskEntry = JSonUtils.ConvertJSonToObject <SatyamTaskTableEntry>(Hidden_TaskEntryString.Value); SatyamResult result = new SatyamResult(); result.TaskParametersString = taskEntry.TaskParametersString; result.TaskStartTime = PageLoadTime; result.TaskEndTime = SubmitTime; result.TaskTableEntryID = taskEntry.ID; AmazonTaskResultInfo amazonInfo = new AmazonTaskResultInfo(); amazonInfo.AssignmentID = ""; amazonInfo.WorkerID = ""; amazonInfo.HITID = ""; result.amazonInfo = amazonInfo; result.TaskResult = Hidden_Result.Value; string resultString = JSonUtils.ConvertObjectToJSon <SatyamResult>(result); SatyamResultsTableAccess resultdb = new SatyamResultsTableAccess(); resultdb.AddEntry(taskEntry.JobTemplateType, taskEntry.UserID, taskEntry.JobGUID, resultString, taskEntry.ID, PageLoadTime, SubmitTime); resultdb.close(); //SatyamTaskTableAccess taskDB = new SatyamTaskTableAccess(); //taskDB.IncrementDoneScore(taskEntry.ID); bool NotDone = getNewRandomJob(); if (NotDone == false) { Response.Redirect("AllJobsDone.aspx"); } }
//go through all the inconclusive results and aggregate them if they are not aggregated already public static void AggregateResults() { //first get all the results that are not aggregated SatyamResultsTableAccess resultsDB = new SatyamResultsTableAccess(); List <SatyamResultsTableEntry> results = resultsDB.getEntriesByStatus(ResultStatus.inconclusive); if (results.Count == 0) //there is nothing to do { resultsDB.close(); return; } //first group all the results by the tasks since wach task may have multiple entries Dictionary <int, List <SatyamResultsTableEntry> > collectedResults = new Dictionary <int, List <SatyamResultsTableEntry> >(); List <string> pendingGUID = new List <string>(); foreach (SatyamResultsTableEntry entry in results) { int taskID = entry.SatyamTaskTableEntryID; if (!collectedResults.ContainsKey(taskID)) { collectedResults.Add(taskID, new List <SatyamResultsTableEntry>()); } collectedResults[taskID].Add(entry); if (!pendingGUID.Contains(entry.JobGUID)) { pendingGUID.Add(entry.JobGUID); } } List <int> taskIDList = collectedResults.Keys.ToList(); taskIDList.Sort(); //check if guid is still pending SatyamJobSubmissionsTableAccess jobDB = new SatyamJobSubmissionsTableAccess(); List <string> completedGUIDs = jobDB.getAllJobGUIDSByStatus(JobStatus.completed); jobDB.close(); //check if this taskID has already been aggregated and exists int he aggregationDB Dictionary <string, Dictionary <int, SatyamAggregatedResultsTableEntry> > ExistingAggEntriesPerTaskPerGUID = SatyamAggregatedResultManagement.getAggregatedEntriesPerTaskByGuidList(pendingGUID); Console.WriteLine("Aggregating {0} Tasks", taskIDList.Count); List <SatyamAggregatedResultsTableEntry> aggEntries = new List <SatyamAggregatedResultsTableEntry>(); //foreach (int taskId in taskIDList) for (int i = 0; i < taskIDList.Count; i++) { //int taskId = taskIDList[taskIDList.Count - i - 1]; int taskId = taskIDList[i]; string guid = collectedResults[taskId][0].JobGUID; if (completedGUIDs.Contains(guid)) { // Hit Completed, mark results outdated. resultsDB.UpdateStatusByTaskID(taskId, ResultStatus.outdated); continue; } //SatyamAggregatedResultsTableAccess aggDB = new SatyamAggregatedResultsTableAccess(); //int LatestResultsAggregated = aggDB.getLatestNoResultsAggregatedByTaskID(taskId); //aggDB.close(); if (ExistingAggEntriesPerTaskPerGUID.ContainsKey(guid) && ExistingAggEntriesPerTaskPerGUID[guid].ContainsKey(taskId)) { continue; //int MinResults = TaskConstants.getMinResultsByTemplate(ExistingAggEntriesPerTaskPerGUID[guid][taskId].JobTemplateType); //if (LatestResultsAggregated >= MinResults) //{ // continue; //} // already aggregated to MinResult Request, but leftover results will be judged. So do nothing } //if it does not exist only then aggregate Console.WriteLine("Task Type: {2}({3}), Task ID: {0}, # of Results: {1}", taskId, collectedResults[taskId].Count, collectedResults[taskId][0].JobTemplateType, collectedResults[taskId][0].JobGUID); //if (collectedResults[taskId].Count != LatestResultsAggregated) //{ SatyamAggregatedResultsTableEntry aggEntry = ResultsTableAggregator.GetAggregatedResultString(taskId, collectedResults[taskId]); if (aggEntry != null) { //aggEntries.Add(aggEntry); SatyamAggregatedResultsTableAccess aggDB = new SatyamAggregatedResultsTableAccess(); aggDB.AddEntry(aggEntry, collectedResults[taskId].Count); aggDB.close(); } //} //else //{ // Console.WriteLine("No New Results"); //} } resultsDB.close(); }
protected void SubmitButton_Click(object sender, EventArgs e) { DateTime SubmitTime = DateTime.Now; DateTime PageLoadTime = Convert.ToDateTime(Hidden_PageLoadTime.Value); int count; bool isValidCount = int.TryParse(CountTextBox.Text, out count); if (!isValidCount) { ErrorLabel.Text = "Error : Ivalid Count."; ErrorLabel.ForeColor = System.Drawing.Color.Red; ErrorLabel.Font.Bold = true; } else if (count < 0) { ErrorLabel.Text = "Error : Count cannot be less than zero."; ErrorLabel.ForeColor = System.Drawing.Color.Red; ErrorLabel.Font.Bold = true; } else { ErrorLabel.Text = ""; SatyamTaskTableEntry taskEntry = JSonUtils.ConvertJSonToObject <SatyamTaskTableEntry>(Hidden_TaskEntryString.Value); SatyamResult result = new SatyamResult(); result.TaskParametersString = taskEntry.TaskParametersString; result.TaskStartTime = PageLoadTime; result.TaskEndTime = SubmitTime; result.TaskTableEntryID = taskEntry.ID; AmazonTaskResultInfo amazonInfo = new AmazonTaskResultInfo(); amazonInfo.AssignmentID = ""; amazonInfo.WorkerID = ""; amazonInfo.HITID = ""; result.amazonInfo = amazonInfo; ObjectCountingResult sresult = new ObjectCountingResult(); sresult.Count = count; string sresultString = JSonUtils.ConvertObjectToJSon <ObjectCountingResult>(sresult); result.TaskResult = sresultString; string resultString = JSonUtils.ConvertObjectToJSon <SatyamResult>(result); SatyamResultsTableAccess resultdb = new SatyamResultsTableAccess(); resultdb.AddEntry(taskEntry.JobTemplateType, taskEntry.UserID, taskEntry.JobGUID, resultString, taskEntry.ID, PageLoadTime, SubmitTime); resultdb.close(); //SatyamTaskTableAccess taskDB = new SatyamTaskTableAccess(); //taskDB.IncrementDoneScore(taskEntry.ID); bool NotDone = getNewRandomJob(); if (NotDone == false) { Response.Redirect("AllJobsDone.aspx"); } } }
//protected void SubmitButton_Click(object sender, EventArgs e) private void storeResult() { DateTime SubmitTime = DateTime.Now; DateTime PageLoadTime = Convert.ToDateTime(Hidden_PageLoadTime.Value); SatyamTaskTableEntry taskEntry = JSonUtils.ConvertJSonToObject <SatyamTaskTableEntry>(Hidden_TaskEntryString.Value); string tracksString = TracksOutput_Hidden.Value; string urlList = Hidden_ImageURLList.Value; string[] fields = urlList.Split(','); DateTime start = DateTime.MinValue; List <DateTime> frameTimes = new List <DateTime>(); //double frameTimeSpanInMiliseconds = (Convert.ToDouble(Hidden_ChunkDuration.Value) / (double)fields.Length) * 1000; double frameTimeSpanInMiliseconds = (double)(1000) / Convert.ToDouble(fps_Hidden.Value); for (int i = 0; i < fields.Length; i++) { DateTime t; t = start.AddMilliseconds(frameTimeSpanInMiliseconds * i); frameTimes.Add(t); } string s = Raw_VATIC_DVA_Crowdsourced_Track_Collection.Raw_VATIC_DVA_Crowdsourced_Track_Collection_ToTrackStrings(tracksString, frameTimes); SatyamResult result = new SatyamResult(); result.TaskParametersString = taskEntry.TaskParametersString; result.TaskStartTime = PageLoadTime; result.TaskEndTime = SubmitTime; result.TaskTableEntryID = taskEntry.ID; AmazonTaskResultInfo amazonInfo = new AmazonTaskResultInfo(); amazonInfo.AssignmentID = Hidden_AmazonAssignmentID.Value; amazonInfo.WorkerID = Hidden_AmazonWorkerID.Value; amazonInfo.HITID = Hidden_HITID.Value; amazonInfo.PricePerHIT = Convert.ToDouble(Hidden_Price.Value); result.amazonInfo = amazonInfo; result.TaskResult = s; string resultString = JSonUtils.ConvertObjectToJSon <SatyamResult>(result); SatyamResultsTableAccess resultdb = new SatyamResultsTableAccess(); resultdb.AddEntry(taskEntry.JobTemplateType, taskEntry.UserID, taskEntry.JobGUID, resultString, taskEntry.ID, PageLoadTime, SubmitTime); resultdb.close(); SatyamTaskTableManagement.UpdateResultNumber(taskEntry.ID); if (!Testing) { SatyamAmazonHITTableAccess HITdb = new SatyamAmazonHITTableAccess(); string HITID = result.amazonInfo.HITID; HITdb.UpdateStatusByHITID(HITID, HitStatus.submitted); HITdb.close(); AmazonMTurkNotification.submitAmazonTurkHit(result.amazonInfo.AssignmentID, result.amazonInfo.WorkerID, false); } Response.Redirect("AllJobsDone.aspx"); }
public static void Run([QueueTrigger("judge-result")] string myQueueItem, TraceWriter log) { bool logging = false; string[] fields = myQueueItem.Split('_'); string guid = fields[0]; int taskID = Convert.ToInt32(fields[1]); string resultID = fields[2]; if (logging) { log.Info($"Judge Result: {myQueueItem}"); } SatyamResultsTableAccess resultsDB = new SatyamResultsTableAccess(); List <SatyamResultsTableEntry> results = resultsDB.getEntriesID(resultID); resultsDB.close(); if (results.Count == 0) { return; } SatyamAggregatedResultsTableAccess aggDB = new SatyamAggregatedResultsTableAccess(); SatyamAggregatedResultsTableEntry aggResult = aggDB.getEntryByTaskID(taskID); aggDB.close(); if (aggResult == null) { return; } SatyamResultsTableEntry result = results[0]; SatyamResult res = JSonUtils.ConvertJSonToObject <SatyamResult>(result.ResultString); string taskTemplateType = result.JobTemplateType; string workerID = res.amazonInfo.WorkerID; DateTime doneTime = res.TaskEndTime; if (AcceptanceCriterionChecker.IsAcceptable(aggResult, result)) { resultsDB = new SatyamResultsTableAccess(); resultsDB.UpdateStatusByID(result.ID, ResultStatus.accepted); resultsDB.close(); WorkerStatisticsManagement.UpdateWorkerStatistics(workerID, taskTemplateType, true, doneTime); if (logging) { log.Info($"Accepted"); } } else { resultsDB = new SatyamResultsTableAccess(); resultsDB.UpdateStatusByID(result.ID, ResultStatus.rejected); resultsDB.close(); WorkerStatisticsManagement.UpdateWorkerStatistics(workerID, taskTemplateType, false, doneTime); if (logging) { log.Info($"Rejected"); } } }
public static void AggregateWithParameter(string guid, int MinResults = TaskConstants.TRACKLET_LABELING_MTURK_MIN_RESULTS_TO_AGGREGATE, int MaxResults = TaskConstants.TRACKLET_LABELING_MTURK_MAX_RESULTS_TO_AGGREGATE, double boxToleranceThreshold = TaskConstants.TRACKLET_LABELING_BOX_DEVIATION_THRESHOLD, double ObjectCoverageApprovalThresholdPerVideo = TaskConstants.TRACKLET_LABELING_APPROVALRATIO_PER_VIDEO, double BoxCoverageApprovalThresholdPerTrack = TaskConstants.TRACKLET_LABELING_APPROVALRATIO_PER_TRACK, int consensusNumber = TaskConstants.TRACKLET_LABELING_MIN_RESULTS_FOR_CONSENSUS, double minTubeletIoUSimilarityThreshold = TaskConstants.TRACKLET_LABELING_MIN_TUBELET_SIMILARITY_THRESHOLD, double attributeMajority = TaskConstants.TRACKLET_LABELING_MTURK_ATTRIBUTE_MAJORITY_THRESHOLD, bool allFalseAttributeInvalid = false ) { string configString = "Min_" + MinResults + "_Max_" + MaxResults + "_IoU_" + minTubeletIoUSimilarityThreshold + "_Ratio_" + ObjectCoverageApprovalThresholdPerVideo; Console.WriteLine("Aggregating for " + guid + " with param set " + configString); SatyamResultsTableAccess resultsDB = new SatyamResultsTableAccess(); List <SatyamResultsTableEntry> entries = resultsDB.getEntriesByGUID(guid); resultsDB.close(); SortedDictionary <DateTime, List <SatyamResultsTableEntry> > entriesBySubmitTime = SatyamResultValidationToolKit.SortResultsBySubmitTime_OneResultPerTurkerPerTask(entries); Dictionary <int, List <MultiObjectTrackingResult> > ResultsPerTask = new Dictionary <int, List <MultiObjectTrackingResult> >(); List <int> aggregatedTasks = new List <int>(); int noTotalConverged = 0; //int noCorrect = 0; int noTerminatedTasks = 0; List <SatyamAggregatedResultsTableEntry> aggEntries = new List <SatyamAggregatedResultsTableEntry>(); Dictionary <int, int> noResultsNeededForAggregation = SatyamResultsAnalysis.getNoResultsNeededForAggregationFromLog(configString, guid); Dictionary <int, int> noResultsNeededForAggregation_new = new Dictionary <int, int>(); // play back by time Dictionary <int, List <string> > WorkersPerTask = new Dictionary <int, List <string> >(); foreach (DateTime t in entriesBySubmitTime.Keys) { //Console.WriteLine("Processing Results of time: {0}", t); List <SatyamResultsTableEntry> ResultEntries = entriesBySubmitTime[t]; foreach (SatyamResultsTableEntry entry in ResultEntries) { SatyamResult satyamResult = JSonUtils.ConvertJSonToObject <SatyamResult>(entry.ResultString); SatyamTask task = JSonUtils.ConvertJSonToObject <SatyamTask>(satyamResult.TaskParametersString); MultiObjectTrackingSubmittedJob job = JSonUtils.ConvertJSonToObject <MultiObjectTrackingSubmittedJob>(task.jobEntry.JobParameters); string fileName = URIUtilities.filenameFromURINoExtension(task.SatyamURI); int taskEntryID = entry.SatyamTaskTableEntryID; if (aggregatedTasks.Contains(taskEntryID)) { continue; } if (!ResultsPerTask.ContainsKey(taskEntryID)) { ResultsPerTask.Add(taskEntryID, new List <MultiObjectTrackingResult>()); WorkersPerTask.Add(taskEntryID, new List <string>()); } // remove duplicate workers result string workerID = satyamResult.amazonInfo.WorkerID; if (WorkersPerTask[taskEntryID].Contains(workerID)) { continue; } //enclose only non-duplicate results, one per each worker. WorkersPerTask[taskEntryID].Add(workerID); string videoName = URIUtilities.localDirectoryNameFromURI(task.SatyamURI); string[] fields = videoName.Split('_'); int startingFrame = Convert.ToInt32(fields[fields.Length - 1]); int maxChunkEndFrame = startingFrame + job.ChunkDuration * job.FrameRate; int noFrameOverlap = (int)(job.ChunkOverlap * job.FrameRate); if (startingFrame != 0) { startingFrame -= noFrameOverlap; } string blobDir = URIUtilities.localDirectoryFullPathFromURI(task.SatyamURI); VATIC_DVA_CrowdsourcedResult taskr = TrackletLabelingAggregator.createVATIC_DVA_CrowdsourcedResultUsingSatyamBlobImageCount(satyamResult.TaskResult, blobDir, entry.SatyamTaskTableEntryID.ToString(), job.FrameRate); //MultiObjectTrackingResult result_tmp = vatic_tmp.getCompressedTracksInTimeSegment(); //VATIC_DVA_CrowdsourcedResult taskr = new VATIC_DVA_CrowdsourcedResult(satyamResult.TaskResult, videoName, entry.ID.ToString(), ImageURLs.Count, job.FrameRate); MultiObjectTrackingResult res = taskr.getCompressedTracksInTimeSegment(); if (allFalseAttributeInvalid) { if (TrackletLabelingAggregator.AllAttributeAllFalse(res)) { continue; } } ResultsPerTask[taskEntryID].Add(res); // check log if enough results are collected if (noResultsNeededForAggregation != null && noResultsNeededForAggregation.ContainsKey(taskEntryID) && ResultsPerTask[taskEntryID].Count < noResultsNeededForAggregation[taskEntryID]) { continue; } // hack for masters int tempMin = MinResults; if (TaskConstants.masterGUIDs.Contains(entry.JobGUID)) { tempMin = 1; } TrackletLabelingAggregatedResult aggResult = TrackletLabelingAggregator.getAggregatedResult(ResultsPerTask[taskEntryID], guid, task.SatyamURI, job.FrameRate, job.BoundaryLines, MinResults, MaxResults, boxToleranceThreshold, ObjectCoverageApprovalThresholdPerVideo, BoxCoverageApprovalThresholdPerTrack, consensusNumber, minTubeletIoUSimilarityThreshold, attributeMajority); if (aggResult == null) { continue; } //////////////// aggregation happen // record logs if (noResultsNeededForAggregation == null || !noResultsNeededForAggregation.ContainsKey(taskEntryID)) { noResultsNeededForAggregation_new.Add(taskEntryID, ResultsPerTask[taskEntryID].Count); } aggregatedTasks.Add(taskEntryID); noTotalConverged++; if (ResultsPerTask[taskEntryID].Count >= MaxResults) { noTerminatedTasks++; } SatyamAggregatedResult SatyamAggResult = new SatyamAggregatedResult(); SatyamAggResult.SatyamTaskTableEntryID = taskEntryID; SatyamAggResult.AggregatedResultString = JSonUtils.ConvertObjectToJSon <TrackletLabelingAggregatedResult>(aggResult); SatyamAggResult.TaskParameters = JSonUtils.ConvertJSonToObject <SatyamResult>(entry.ResultString).TaskParametersString; SatyamAggregatedResultsTableEntry aggEntry = new SatyamAggregatedResultsTableEntry(); aggEntry.SatyamTaskTableEntryID = taskEntryID; aggEntry.JobGUID = entry.JobGUID; aggEntry.ResultString = JSonUtils.ConvertObjectToJSon <SatyamAggregatedResult>(SatyamAggResult); aggEntries.Add(aggEntry); } } Console.WriteLine("Total_Aggregated_Tasks: {0}", noTotalConverged); Console.WriteLine("Total_Terminated_Tasks: {0}", noTerminatedTasks); SatyamResultsAnalysis.RecordAggregationLog(noResultsNeededForAggregation_new, configString, guid); TrackletLabelingAnalyzer.GroupEntriesByVideoNameAndStitchAndSaveAggregatedResultVideosLocally(aggEntries, DirectoryConstants.defaultTempDirectory + guid); }
//go through all the inconclusive results and check with aggregated table, if aggreagted, compare and see if it is acceptable and then //accept/reject public static void AcceptRejectResults() { //get all inconclusive results and group them by taskIDs SatyamResultsTableAccess resultsDB = new SatyamResultsTableAccess(); List <SatyamResultsTableEntry> results = resultsDB.getAllEntriesByStatus(ResultStatus.inconclusive); if (results.Count == 0) { resultsDB.close(); return; } Dictionary <int, List <SatyamResultsTableEntry> > resultsByTaskID = new Dictionary <int, List <SatyamResultsTableEntry> >(); List <string> pendingGUID = new List <string>(); foreach (SatyamResultsTableEntry result in results) { if (!resultsByTaskID.ContainsKey(result.SatyamTaskTableEntryID)) { resultsByTaskID.Add(result.SatyamTaskTableEntryID, new List <SatyamResultsTableEntry>()); } resultsByTaskID[result.SatyamTaskTableEntryID].Add(result); if (!pendingGUID.Contains(result.JobGUID)) { pendingGUID.Add(result.JobGUID); } } //now check against the aggregated result to see if the result is aceptable Dictionary <string, Dictionary <int, SatyamAggregatedResultsTableEntry> > aggEntriesPerTaskPerGUID = SatyamAggregatedResultManagement.getAggregatedEntriesPerTaskByGuidList(pendingGUID); foreach (KeyValuePair <int, List <SatyamResultsTableEntry> > entry in resultsByTaskID) { if (entry.Value.Count == 0) { continue; } int taskEntryID = entry.Key; string taskGUID = entry.Value[0].JobGUID; if (aggEntriesPerTaskPerGUID.ContainsKey(taskGUID) && aggEntriesPerTaskPerGUID[taskGUID].ContainsKey(taskEntryID)) { // this task has been aggregated foreach (SatyamResultsTableEntry result in resultsByTaskID[taskEntryID]) //now got through each task to see if they satify pass criterion { SatyamResult res = JSonUtils.ConvertJSonToObject <SatyamResult>(result.ResultString); string taskTemplateType = result.JobTemplateType; string workerID = res.amazonInfo.WorkerID; DateTime doneTime = res.TaskEndTime; if (AcceptanceCriterionChecker.IsAcceptable(aggEntriesPerTaskPerGUID[taskGUID][taskEntryID], result)) { resultsDB.UpdateStatusByID(result.ID, ResultStatus.accepted); WorkerStatisticsManagement.UpdateWorkerStatistics(workerID, taskTemplateType, true, doneTime); } else { resultsDB.UpdateStatusByID(result.ID, ResultStatus.rejected); WorkerStatisticsManagement.UpdateWorkerStatistics(workerID, taskTemplateType, false, doneTime); } } } } resultsDB.close(); }
protected void SubmitButton_Click(object sender, EventArgs e) { DateTime SubmitTime = DateTime.Now; DateTime PageLoadTime = Convert.ToDateTime(Hidden_PageLoadTime.Value); int count; bool isValidCount = int.TryParse(CountTextBox.Text, out count); if (!isValidCount) { ErrorLabel.Text = "Error : Ivalid Count."; ErrorLabel.ForeColor = System.Drawing.Color.Red; ErrorLabel.Font.Bold = true; } else if (count < 0) { ErrorLabel.Text = "Error : Count cannot be less than zero."; ErrorLabel.ForeColor = System.Drawing.Color.Red; ErrorLabel.Font.Bold = true; } else { ErrorLabel.Text = ""; CountTextBox.Text = ""; SatyamTaskTableEntry taskEntry = JSonUtils.ConvertJSonToObject <SatyamTaskTableEntry>(Hidden_TaskEntryString.Value); SatyamResult result = new SatyamResult(); result.TaskParametersString = taskEntry.TaskParametersString; result.TaskStartTime = PageLoadTime; result.TaskEndTime = SubmitTime; result.TaskTableEntryID = taskEntry.ID; AmazonTaskResultInfo amazonInfo = new AmazonTaskResultInfo(); amazonInfo.AssignmentID = Hidden_AmazonAssignmentID.Value; amazonInfo.WorkerID = Hidden_AmazonWorkerID.Value; amazonInfo.HITID = Hidden_HITID.Value; amazonInfo.PricePerHIT = Convert.ToDouble(Hidden_Price.Value); result.amazonInfo = amazonInfo; ObjectCountingResult sresult = new ObjectCountingResult(); sresult.Count = count; string sresultString = JSonUtils.ConvertObjectToJSon <ObjectCountingResult>(sresult); result.TaskResult = sresultString; string resultString = JSonUtils.ConvertObjectToJSon <SatyamResult>(result); SatyamResultsTableAccess resultdb = new SatyamResultsTableAccess(); resultdb.AddEntry(taskEntry.JobTemplateType, taskEntry.UserID, taskEntry.JobGUID, resultString, taskEntry.ID, PageLoadTime, SubmitTime); resultdb.close(); SatyamTaskTableManagement.UpdateResultNumber(taskEntry.ID); //SatyamTaskTableAccess taskDB = new SatyamTaskTableAccess(); //taskDB.IncrementDoneScore(taskEntry.ID); int noDone = Convert.ToInt32(Hidden_NoImagesDone.Value); noDone++; Hidden_NoImagesDone.Value = noDone.ToString(); bool NotDone = false; int noTasksPerJob = Convert.ToInt32(Hidden_TasksPerJob.Value); if (noDone < noTasksPerJob) { NotDone = getNewRandomJob(); } if (NotDone == false) { if (!Testing) { SatyamAmazonHITTableAccess HITdb = new SatyamAmazonHITTableAccess(); string HITID = result.amazonInfo.HITID; HITdb.UpdateStatusByHITID(HITID, HitStatus.submitted); HITdb.close(); AmazonMTurkNotification.submitAmazonTurkHit(result.amazonInfo.AssignmentID, result.amazonInfo.WorkerID, false); } Response.Redirect("MTurkTaskDonePage.aspx"); } } }
public static void AggregateWithParameterAndValidateObjectCountingResultByGUID(string guid, //bool saveImage = false, bool prepareTrainingSet = false, string outputDirectory = null, int MinHeight = TaskConstants.OBJECT_COUNTING_VALIDATION_MIN_HEIGHT, int MaxOcclusion = TaskConstants.OBJECT_COUNTING_VALIDATION_MAX_OCCLUSION, double Max_Truncation = TaskConstants.OBJECT_COUNTING_VALIDATION_MIN_TRUNCATION, int MinResults = TaskConstants.OBJECT_COUNTING_MTURK_MIN_RESULTS_TO_AGGREGATE, int MaxResults = TaskConstants.OBJECT_COUNTING_MTURK_MAX_RESULTS_TO_AGGREGATE, double MAX_ABSOLUTE_COUNT_DEVIATION_LOWERBOUND = TaskConstants.OBJECT_COUNTING_MTURK_MAX_ABSOLUTE_COUNT_DEVIATION_LOWERBOUND, double MAX_DEVIATION_FRACTION = TaskConstants.OBJECT_COUNTING_MTURK_MAX_DEVIATION_FRACTION, double SUPER_MAJORITY_VALUE = TaskConstants.OBJECT_COUNTING_MTURK_SUPER_MAJORITY_VALUE ) { if (!Directory.Exists(DirectoryConstants.defaultTempDirectory + guid)) { Directory.CreateDirectory(DirectoryConstants.defaultTempDirectory + guid); } string configString = "Min_" + MinResults + "_Max_" + MaxResults + "_Dev_" + MAX_DEVIATION_FRACTION + "_Majority_" + SUPER_MAJORITY_VALUE; Console.WriteLine("Aggregating for param set " + configString); int noTerminatedTasks = 0; SatyamResultsTableAccess resultsDB = new SatyamResultsTableAccess(); List <SatyamResultsTableEntry> entries = resultsDB.getEntriesByGUIDOrderByID(guid); resultsDB.close(); SortedDictionary <DateTime, List <SatyamResultsTableEntry> > entriesBySubmitTime = SatyamResultValidationToolKit.SortResultsBySubmitTime(entries); int noTotalConverged = 0; //int noCorrect = 0; Dictionary <int, List <SatyamResultsTableEntry> > ResultsPerTask = new Dictionary <int, List <SatyamResultsTableEntry> >(); List <int> aggregatedTasks = new List <int>(); List <SatyamAggregatedResultsTableEntry> aggEntries = new List <SatyamAggregatedResultsTableEntry>(); Dictionary <int, int> noResultsNeededForAggregation = SatyamResultsAnalysis.getNoResultsNeededForAggregationFromLog(configString, guid); Dictionary <int, int> noResultsNeededForAggregation_new = new Dictionary <int, int>(); // play back by time foreach (DateTime t in entriesBySubmitTime.Keys) { //Console.WriteLine("Processing Results of time: {0}", t); foreach (SatyamResultsTableEntry entry in entriesBySubmitTime[t]) { SatyamResult satyamResult = JSonUtils.ConvertJSonToObject <SatyamResult>(entry.ResultString); SatyamTask task = JSonUtils.ConvertJSonToObject <SatyamTask>(satyamResult.TaskParametersString); SatyamJob job = task.jobEntry; string fileName = URIUtilities.filenameFromURI(task.SatyamURI); int taskEntryID = entry.SatyamTaskTableEntryID; if (aggregatedTasks.Contains(taskEntryID)) { continue; } if (!ResultsPerTask.ContainsKey(taskEntryID)) { ResultsPerTask.Add(taskEntryID, new List <SatyamResultsTableEntry>()); } ResultsPerTask[taskEntryID].Add(entry); // check log if enough results are collected if (noResultsNeededForAggregation != null && noResultsNeededForAggregation.ContainsKey(taskEntryID) && ResultsPerTask[taskEntryID].Count < noResultsNeededForAggregation[taskEntryID]) { continue; } string aggResultString = ObjectCountingAggregator.GetAggregatedResultString(ResultsPerTask[taskEntryID], MinResults, MaxResults, MAX_ABSOLUTE_COUNT_DEVIATION_LOWERBOUND, MAX_DEVIATION_FRACTION, SUPER_MAJORITY_VALUE); if (aggResultString == null) { continue; } SatyamAggregatedResultsTableEntry aggEntry = new SatyamAggregatedResultsTableEntry(); aggEntry.JobGUID = ResultsPerTask[taskEntryID][0].JobGUID; aggEntry.JobTemplateType = ResultsPerTask[taskEntryID][0].JobTemplateType; aggEntry.SatyamTaskTableEntryID = taskEntryID; aggEntry.UserID = ResultsPerTask[taskEntryID][0].UserID; aggEntry.ResultString = aggResultString; /// aggregation happen /// // record logs // record logs if (noResultsNeededForAggregation == null || !noResultsNeededForAggregation.ContainsKey(taskEntryID)) { noResultsNeededForAggregation_new.Add(taskEntryID, ResultsPerTask[taskEntryID].Count); } aggEntries.Add(aggEntry); noTotalConverged++; if (ResultsPerTask[taskEntryID].Count >= MaxResults) { noTerminatedTasks++; } aggregatedTasks.Add(taskEntryID); } } SatyamResultsAnalysis.RecordAggregationLog(noResultsNeededForAggregation_new, configString, guid); double totalError, totalGroundtruth; //ValidateSatyamKITTIObjectCountingAggregationResult(aggEntries, out totalError, out totalGroundtruth, // MinHeight, MaxOcclusion, Max_Truncation); ValidateSatyamCARPKObjectCountingAggregationResult(aggEntries, out totalError, out totalGroundtruth); Console.WriteLine("noTotalConverged {0}", noTotalConverged); Console.WriteLine("noTerminatedTasks {0}", noTerminatedTasks); /// local file string outputString = String.Format("{0} {1} {2} {3} {4} {5} {6}\n", configString, totalError, totalGroundtruth, totalError / totalGroundtruth, noTotalConverged, noTerminatedTasks, ResultsPerTask.Count - noTotalConverged); string outputfile = DirectoryConstants.defaultTempDirectory + guid + "\\resultSummary.txt"; File.AppendAllText(outputfile, outputString); string approvalString = configString + "_PayDev_" + TaskConstants.OBJECT_COUNTING_MTURK_MAX_ABSOLUTE_COUNT_DEVIATION_FOR_PAYMENT + "_PayFrac_" + TaskConstants.OBJECT_COUNTING_MTURK_MAX_DEVIATION_FRACTION_FOR_PAYMENT; //for (double prob = 0; prob < 1;prob +=0.2) //{ // SatyamResultsAnalysis.AnalyzeApprovalRate(aggEntries, entries, guid, approvalString, anotherChanceProbablity: prob); //} //for (double ratio = 0; ratio < 1; ratio += 0.2) //{ // SatyamResultsAnalysis.AnalyzeApprovalRate(aggEntries, entriesBySubmitTime, noResultsNeededForAggregation, noResultsNeededForAggregation_new, guid, configString, approvalRatioThreshold: ratio); //} SatyamResultsAnalysis.AggregationAnalysis(aggEntries, entriesBySubmitTime, noResultsNeededForAggregation, noResultsNeededForAggregation_new, guid, configString); }
public static bool SafeDeleteHit(SatyamAmazonHITTableAccessEntry hitEntryToBeRemoved) { // first expire AmazonMTurkHIT hit = new AmazonMTurkHIT(); hit.setAccount(hitEntryToBeRemoved.AmazonAccessKeyID, hitEntryToBeRemoved.AmazonSecretAccessKeyID, false); hit.expireHIT(hitEntryToBeRemoved.HITID); SatyamAmazonHITTableAccess hitDB = new SatyamAmazonHITTableAccess(); hitDB.UpdateStatus(hitEntryToBeRemoved.ID, HitStatus.expired); hitDB.close(); // then try dispose if (!hit.DeleteHIT(hitEntryToBeRemoved.HITID)) { // can't dispose due to invalid status: reviewable but payment not cleared, or disposed. // because payment not cleared // find all results of this hitID and mark for rejection if (!resultCache.ContainsKey(hitEntryToBeRemoved.JobGUID)) { List <SatyamResultsTableEntry> ret = new List <SatyamResultsTableEntry>(); SatyamResultsTableAccess resDB = new SatyamResultsTableAccess(); List <SatyamResultsTableEntry> tempres = resDB.getEntriesByGUID(hitEntryToBeRemoved.JobGUID); resDB.close(); resultCache.Add(hitEntryToBeRemoved.JobGUID, tempres); } List <SatyamResultsTableEntry> results = resultCache[hitEntryToBeRemoved.JobGUID]; List <SatyamResultsTableEntry> res = new List <SatyamResultsTableEntry>(); foreach (SatyamResultsTableEntry r in results) { SatyamResult rr = JSonUtils.ConvertJSonToObject <SatyamResult>(r.ResultString); if (rr.amazonInfo.HITID == hitEntryToBeRemoved.HITID) { res.Add(r); } } if (res.Count == 0) { // the hit can't be disposed because there are pending assignments, yet we don't have any on records, // so delete it. AWS will automatically handle it after 120 days of inactivity. hitDB = new SatyamAmazonHITTableAccess(); hitDB.DeleteEntry(hitEntryToBeRemoved.ID); hitDB.close(); Console.WriteLine("Delete Success\n"); return(true); } //bool HITDone = true; foreach (SatyamResultsTableEntry r in res) { //SatyamResult rr = JSonUtils.ConvertJSonToObject<SatyamResult>(r.ResultString); //string assignmentID = rr.amazonInfo.AssignmentID; //hit.RejectAssignment(assignmentID, "Sorry! Your work was not within acceptable parameters!"); if (r.Status == ResultStatus.inconclusive || r.Status == ResultStatus.outdated) { // reject all for now... TODO: check if there is an aggregated results. //Console.WriteLine("Rejecting result ID {0}", r.ID); //SatyamResultsTableAccess resultsDB = new SatyamResultsTableAccess(); //resultsDB.UpdateStatusByID(r.ID, ResultStatus.rejected); //resultsDB.close(); //HITDone = false; } else { Console.WriteLine("Result ID {0} is of status {1}", r.ID, r.Status); if (r.Status == ResultStatus.accepted || r.Status == ResultStatus.rejected) { // need to be handled by payment function //HITDone = false; } // just in case last payment was not made successfully, // mark it to be paid again.... SatyamResultsTableAccess resultsDB = new SatyamResultsTableAccess(); if (ResultStatus.acceptedStatusList.Contains(r.Status)) { Console.WriteLine("Marking as Accepted so Next Cycle will Pay"); resultsDB.UpdateStatusByID(r.ID, ResultStatus.accepted); } if (ResultStatus.rejectedStatusList.Contains(r.Status)) { Console.WriteLine("Marking as Rejected so Next Cycle will Reject"); resultsDB.UpdateStatusByID(r.ID, ResultStatus.rejected); } resultsDB.close(); } } // Commented out cuz the case where the inconsistency status happen is rare, // so don't delete the hit too aggressively //if (HITDone) //{ // hitDB = new SatyamAmazonHITTableAccess(); // hitDB.DeleteEntry(hitEntryToBeRemoved.ID); // hitDB.close(); // Console.WriteLine("Delete Success\n"); // return true; //} Console.WriteLine("Delete Failure\n"); return(false); } else { hitDB = new SatyamAmazonHITTableAccess(); hitDB.DeleteEntry(hitEntryToBeRemoved.ID); hitDB.close(); Console.WriteLine("Delete Success\n"); return(true); } }
public static SatyamResultPerJobDataAnalysis AnalyzeFromResultsTable(List <string> jobGUIDs) { Console.WriteLine("Analyzing Results Statistics of {0}", jobGUIDs[0]); SatyamResultsTableAccess resultsDB = new SatyamResultsTableAccess(); List <SatyamResultsTableEntry> entries = new List <SatyamResultsTableEntry>(); foreach (string guid in jobGUIDs) { entries.AddRange(resultsDB.getEntriesByGUIDOrderByID(guid)); } resultsDB.close(); if (entries.Count == 0) { return(null); } SatyamResultPerJobDataAnalysis ana = new SatyamResultPerJobDataAnalysis(); ana.JobGUID = entries[0].JobGUID; ana.totalResults = entries.Count; for (int i = 0; i < entries.Count; i++) { SatyamResultsTableEntry entry = entries[i]; SatyamResult satyamResult = JSonUtils.ConvertJSonToObject <SatyamResult>(entry.ResultString); SatyamTask task = JSonUtils.ConvertJSonToObject <SatyamTask>(satyamResult.TaskParametersString); SatyamJob job = task.jobEntry; if (ana.job == null) { ana.job = job; } ana.jobSubmitTime = job.JobSubmitTime; double moneyPerResult = 0; if (job.TasksPerJob > 0) { moneyPerResult = (double)satyamResult.amazonInfo.PricePerHIT / (double)job.TasksPerJob; } ana.PricePerResult.Add(entry.ID, moneyPerResult); int taskEntryID = entry.SatyamTaskTableEntryID; if (!ana.noResultsPerTask.ContainsKey(taskEntryID)) { ana.noResultsPerTask.Add(taskEntryID, 0); ana.noAcceptedResultsPerTask.Add(taskEntryID, 0); ana.noPaidResultsPerTask.Add(taskEntryID, 0); ana.moneyPaidPerTask.Add(taskEntryID, 0); ana.finalTaskEndTime.Add(taskEntryID, entry.SubmitTime); ana.ResultsPerWorkerPerTask.Add(taskEntryID, new Dictionary <string, List <string> >()); ana.noDifferentDuplicateResultsPerWorkerPerTask.Add(taskEntryID, new Dictionary <string, int>()); ana.taskSatyamUri.Add(taskEntryID, task.SatyamURI); } ana.noResultsPerTask[taskEntryID]++; ana.SequenceNumberOfResultPerTask.Add(ana.noResultsPerTask[taskEntryID]); if (entry.Status == ResultStatus.accepted || entry.Status == ResultStatus.accepted_Paid || entry.Status == ResultStatus.accepted_NotPaid) { ana.noAcceptedResultsPerTask[taskEntryID]++; ana.totalAcceptedResults++; } if (entry.Status == ResultStatus.accepted_Paid || entry.Status == ResultStatus.rejected_Paid) { ana.noPaidResultsPerTask[taskEntryID]++; ana.totalpaidResults++; ana.moneyPaidPerTask[taskEntryID] += moneyPerResult; ana.totalMoneyPaid += moneyPerResult; } if (ana.finalTaskEndTime[taskEntryID] < entry.SubmitTime) { ana.finalTaskEndTime[taskEntryID] = entry.SubmitTime; } if (ana.lastTaskEndTime < entry.SubmitTime) { ana.lastTaskEndTime = entry.SubmitTime; } TimeSpan dt = satyamResult.TaskEndTime - satyamResult.TaskStartTime; ana.timeTakenPerResult.Add(dt.TotalSeconds); if (!ana.AcceptedTimePerSequenceNumber.ContainsKey(ana.noResultsPerTask[taskEntryID])) { ana.AcceptedTimePerSequenceNumber.Add(ana.noResultsPerTask[taskEntryID], new List <double>()); ana.TimePerSequenceNumber.Add(ana.noResultsPerTask[taskEntryID], new List <double>()); } //ana.acceptedTimeTakenPerResultOrganizedByResultSequencePerTask[ana.noResultsPerTask[taskEntryID]].Add(dt.TotalSeconds); ana.TimePerSequenceNumber[ana.noResultsPerTask[taskEntryID]].Add(dt.TotalSeconds); if (!ana.AcceptedTimePerTask.ContainsKey(taskEntryID)) { ana.AcceptedTimePerTask.Add(taskEntryID, new List <double>()); ana.TimePerTask.Add(taskEntryID, new List <double>()); } //ana.AcceptedTimePerTask[taskEntryID].Add(dt.TotalSeconds); ana.TimePerTask[taskEntryID].Add(dt.TotalSeconds); if (ana.lastTaskEndTime < satyamResult.TaskEndTime) { ana.lastTaskEndTime = satyamResult.TaskEndTime; } if (!ana.finalHITEndTime.ContainsKey(satyamResult.amazonInfo.HITID)) { ana.finalHITEndTime.Add(satyamResult.amazonInfo.HITID, (entry.SubmitTime - job.JobSubmitTime).TotalSeconds); } if (ana.finalHITEndTime[satyamResult.amazonInfo.HITID] < (entry.SubmitTime - job.JobSubmitTime).TotalSeconds) { ana.finalHITEndTime[satyamResult.amazonInfo.HITID] = (entry.SubmitTime - job.JobSubmitTime).TotalSeconds; } if (entry.Status == ResultStatus.accepted || entry.Status == ResultStatus.accepted_Paid || entry.Status == ResultStatus.accepted_NotPaid) { ana.acceptedTimeTakenPerResult.Add(dt.TotalSeconds); ana.AcceptedTimePerTask[taskEntryID].Add(dt.TotalSeconds); ana.AcceptedTimePerSequenceNumber[ana.noResultsPerTask[taskEntryID]].Add(dt.TotalSeconds); } ana.resultArrivalTimes.Add((entry.SubmitTime - job.JobSubmitTime).TotalSeconds); if (!ana.noJobsPerWorker.ContainsKey(satyamResult.amazonInfo.WorkerID)) { ana.noJobsPerWorker.Add(satyamResult.amazonInfo.WorkerID, 0); ana.noAcceptedJobsPerWorker.Add(satyamResult.amazonInfo.WorkerID, 0); ana.noPaidJobsPerWorker.Add(satyamResult.amazonInfo.WorkerID, 0); } ana.noJobsPerWorker[satyamResult.amazonInfo.WorkerID]++; ana.SequenceNumberOfResultPerWorker.Add(ana.noJobsPerWorker[satyamResult.amazonInfo.WorkerID]); if (entry.Status == ResultStatus.accepted || entry.Status == ResultStatus.accepted_Paid || entry.Status == ResultStatus.accepted_NotPaid) { ana.noAcceptedJobsPerWorker[satyamResult.amazonInfo.WorkerID]++; } if (entry.Status == ResultStatus.accepted_Paid || entry.Status == ResultStatus.rejected_Paid) { ana.noPaidJobsPerWorker[satyamResult.amazonInfo.WorkerID]++; } if (!ana.ResultsPerWorkerPerTask[taskEntryID].ContainsKey(satyamResult.amazonInfo.WorkerID)) { ana.ResultsPerWorkerPerTask[taskEntryID].Add(satyamResult.amazonInfo.WorkerID, new List <string>()); ana.noDifferentDuplicateResultsPerWorkerPerTask[taskEntryID].Add(satyamResult.amazonInfo.WorkerID, 0); } else { // this worker has done this task before ana.noDuplicateResultsFromSameWorker++; } if (!ana.ResultsPerWorkerPerTask[taskEntryID][satyamResult.amazonInfo.WorkerID].Contains(satyamResult.TaskResult)) { ana.noDifferentDuplicateResultsPerWorkerPerTask[taskEntryID][satyamResult.amazonInfo.WorkerID]++; if (ana.noDifferentDuplicateResultsPerWorkerPerTask[taskEntryID][satyamResult.amazonInfo.WorkerID] > 1) { ana.noDecisionChangeAmongDuplicateResultsOfSameWorkerSameTask++; } } ana.ResultsPerWorkerPerTask[taskEntryID][satyamResult.amazonInfo.WorkerID].Add(satyamResult.TaskResult); ///////////////// imagenet only //List<SingleObjectLabelingResult> CurrentResultsOfThisTask = new List<SingleObjectLabelingResult>(); //foreach(string worker in ana.ResultsPerWorkerPerTask[taskEntryID].Keys) //{ // foreach(string workerResult in ana.ResultsPerWorkerPerTask[taskEntryID][worker]) // { // CurrentResultsOfThisTask.Add(JSonUtils.ConvertJSonToObject<SingleObjectLabelingResult>(workerResult)); // } //} //if (SingleObjectLabelingAggregator.getAggregatedResult(CurrentResultsOfThisTask) != null) //{ // // aggregation happened // if (!ana.timeTakenTillAggregationPerTask.ContainsKey(taskEntryID)) // { // // for the first time aggregated // ana.timeTakenTillAggregationPerTask.Add(taskEntryID, (entry.SubmitTime - job.JobSubmitTime).TotalSeconds); // } //} /////////////////////// } ana.resultArrivalTimes.Sort(); ana.noTasks = ana.noResultsPerTask.Count; ana.timeTakenPerResult.Sort(); ana.medianTimeTakenPerResultInSeconds = ana.timeTakenPerResult[ana.timeTakenPerResult.Count / 2]; if (ana.acceptedTimeTakenPerResult.Count != 0) { ana.acceptedTimeTakenPerResult.Sort(); ana.medianAcceptedTimeTakenPerResultInSeconds = ana.acceptedTimeTakenPerResult[ana.acceptedTimeTakenPerResult.Count / 2]; } ana.totalTimeTakenForJob = ana.lastTaskEndTime - ana.jobSubmitTime; ///////////////////////////////////// Per Task Analysis ////////////////////////////////////// ana.resultsPerTaskHistogram = new SortedDictionary <int, int>(); ana.resultsAcceptedPerTaskHistogram = new SortedDictionary <int, int>(); ana.resultsPaidPerTaskHistogram = new SortedDictionary <int, int>(); ana.moneyPaidPerTaskHistogram = new SortedDictionary <int, int>(); //cents ana.ResultsPerWorkerPerTaskHistogram = new SortedDictionary <int, int>(); int noAggregatedTasks = 0; List <int> taskIDs = ana.noResultsPerTask.Keys.ToList(); foreach (int taskID in taskIDs) { if (!ana.resultsPerTaskHistogram.ContainsKey(ana.noResultsPerTask[taskID])) { ana.resultsPerTaskHistogram.Add(ana.noResultsPerTask[taskID], 0); } ana.resultsPerTaskHistogram[ana.noResultsPerTask[taskID]]++; if (!ana.resultsAcceptedPerTaskHistogram.ContainsKey(ana.noAcceptedResultsPerTask[taskID])) { ana.resultsAcceptedPerTaskHistogram.Add(ana.noAcceptedResultsPerTask[taskID], 0); } ana.resultsAcceptedPerTaskHistogram[ana.noAcceptedResultsPerTask[taskID]]++; if (!ana.resultsPaidPerTaskHistogram.ContainsKey(ana.noPaidResultsPerTask[taskID])) { ana.resultsPaidPerTaskHistogram.Add(ana.noPaidResultsPerTask[taskID], 0); } ana.resultsPaidPerTaskHistogram[ana.noPaidResultsPerTask[taskID]]++; int moneyPaid = (int)Math.Floor(ana.moneyPaidPerTask[taskID] * 100); if (!ana.moneyPaidPerTaskHistogram.ContainsKey(moneyPaid)) { ana.moneyPaidPerTaskHistogram.Add(moneyPaid, 0); } ana.moneyPaidPerTaskHistogram[moneyPaid]++; if (ana.ResultsPerWorkerPerTask[taskID].Count != ana.noResultsPerTask[taskID]) { //has multiple results from same turker ana.noTasksWithDuplicateResultsFromSameWorker++; } if (ana.noPaidResultsPerTask[taskID] != 0) { noAggregatedTasks++; } } ana.avgNoPaidResultsPerTask = (double)ana.totalpaidResults / (double)noAggregatedTasks; //////////// duplicate analysis //Console.WriteLine("DuplicateResultsHistogram"); //foreach (int no in ana.ResultsPerWorkerPerTaskHistogram.Keys) //{ // Console.WriteLine("{0}, {1}", no, ana.ResultsPerWorkerPerTaskHistogram[no]); //} //Console.WriteLine("{0} images (in total {1}({2})) has duplicate results from same turker, \n" + // "\t{3} images have >=1 workers making more than majority number of results.\n" + // "\t{11} images aggregation is changed ({12} of which incorrectly) by duplicate results from same turker.\n" + // "\t{4} images(In total {5} times) with duplicate results include mixed(>= 2) decisions from same worker.\n" + // //"\t\t{6} images has correct decision among duplicate decisions.\n" + // //"\t\t{7} images({8} times) a worker has switched from incorrect to correct choice\n" + // //"\t\t\t{9} images({10} times) of which maintained the correct choice till the last time of their job on that image.", // ana.noTasksWithDuplicateResultsFromSameWorker, ana.noDuplicateResultsFromSameWorker, 0, // ana.noTaskWithDuplicateResultsAsMajority, // ana.noTasksWithMixedResultsFromSameWorker,ana.noDecisionChangeAmongDuplicateResultsOfSameWorkerSameTask, // ana.noTasksWithCorrectDuplicateResultsOfSameWorkerSameTask // //ana.noTasksWithWorkerSwitchedToCorrect, ana.noWorkerSwitchedToCorrect, // //ana.noTasksWithWorkerSwitchedToCorrectAndMaintained, ana.noWorkerSwitchedToCorrectAndMaintained, // //ana.noTaskWhoseDuplicateResultsChangedAggregation, ana.noTaskWhoseDuplicateResultsChangedAggregationIncorrectly //); ////////////////////////////// string outputDir = DirectoryConstants.defaultTempDirectory + jobGUIDs[0] + "\\ResultStatistics\\"; if (!Directory.Exists(outputDir)) { Directory.CreateDirectory(outputDir); } //Console.WriteLine("timePerResult"); TextWriter tw = new StreamWriter(outputDir + "timePerResult.txt"); foreach (double time in ana.timeTakenPerResult) { tw.WriteLine(time); } tw.Close(); tw = new StreamWriter(outputDir + "AcceptedTimePerSequenceNumber.txt"); foreach (int seq in ana.AcceptedTimePerSequenceNumber.Keys) { foreach (double time in ana.AcceptedTimePerSequenceNumber[seq]) { tw.WriteLine(time); } tw.WriteLine(""); } tw.Close(); tw = new StreamWriter(outputDir + "AcceptedTimePerTask.txt"); foreach (int taskid in ana.AcceptedTimePerTask.Keys) { foreach (double time in ana.AcceptedTimePerTask[taskid]) { tw.WriteLine(time); } tw.WriteLine(""); } tw.Close(); tw = new StreamWriter(outputDir + "TimePerSequenceNumber.txt"); foreach (int seq in ana.TimePerSequenceNumber.Keys) { foreach (double time in ana.TimePerSequenceNumber[seq]) { tw.WriteLine(time); } tw.WriteLine(""); } tw.Close(); tw = new StreamWriter(outputDir + "TimePerTask.txt"); foreach (int taskid in ana.TimePerTask.Keys) { foreach (double time in ana.TimePerTask[taskid]) { tw.WriteLine(time); } tw.WriteLine(""); } tw.Close(); //Console.WriteLine("resultArrivalTime"); TextWriter tw0 = new StreamWriter(outputDir + "resultArrivalTime.txt"); foreach (double time in ana.resultArrivalTimes) { tw0.WriteLine(time); } tw0.Close(); //Console.WriteLine("HITFinishTime"); TextWriter tw2 = new StreamWriter(outputDir + "HITFinishTime.txt"); foreach (double time in ana.finalHITEndTime.Values) { tw2.WriteLine(time); } tw2.Close(); //Console.WriteLine("SequenceNumberOfResultsPerTask"); TextWriter tw3 = new StreamWriter(outputDir + "SequenceNumberOfResultsPerTask.txt"); foreach (int seq in ana.SequenceNumberOfResultPerTask) { tw3.WriteLine(seq); } tw3.Close(); //Console.WriteLine("SequenceNumberOfResultsPerWorker"); TextWriter tw4 = new StreamWriter(outputDir + "SequenceNumberOfResultsPerWorker.txt"); foreach (int seq in ana.SequenceNumberOfResultPerWorker) { tw4.WriteLine(seq); } tw4.Close(); //Console.WriteLine("MoneyPaidPerResult"); TextWriter tw5 = new StreamWriter(outputDir + "MoneyPaidPerResult.txt"); foreach (int seq in ana.PricePerResult.Keys) { tw5.WriteLine(ana.PricePerResult[seq]); } tw5.Close(); TextWriter tw6 = new StreamWriter(outputDir + "acceptedTimePerResult.txt"); foreach (double time in ana.acceptedTimeTakenPerResult) { tw6.WriteLine(time); } tw6.Close(); ///////////////////// aggregation time histogram, out of scope //Console.WriteLine("resultArrivalTime", // TaskConstants.SINGLE_OBJECT_LABLING_MTURK_MIN_RESULTS_TO_AGGREGATE, TaskConstants.SINGLE_OBJECT_LABLING_MTURK_MAX_RESULTS_TO_AGGREGATE); //string filepath = String.Format(@"C:\research\MSRDVA\SatyamResearch\aggregationTimePerTask_MinNoResults_{0}_MaxNoResults_{1}.txt", // TaskConstants.SINGLE_OBJECT_LABLING_MTURK_MIN_RESULTS_TO_AGGREGATE, TaskConstants.SINGLE_OBJECT_LABLING_MTURK_MAX_RESULTS_TO_AGGREGATE); //TextWriter tw1 = new StreamWriter(filepath); //foreach (double time in ana.timeTakenTillAggregationPerTask.Values) //{ // tw1.WriteLine(time); //} //tw1.Close(); Console.WriteLine("Median Time For Accepted Task: {0}", ana.medianAcceptedTimeTakenPerResultInSeconds); Console.WriteLine("Median Time For All Task: {0}", ana.medianTimeTakenPerResultInSeconds); Console.WriteLine("Total Tasks: {0}", ana.noTasks); Console.WriteLine("Total Accepted Results: {0}", ana.totalAcceptedResults); Console.WriteLine("Total Results: {0}", ana.totalResults); Console.WriteLine("Avg Paid Results Per Task: {0}", ana.avgNoPaidResultsPerTask); return(ana); }
public static void AggregateWithParameterAndValidateSatyamVideoClassificationResultByGUID(string guid, int MinResults = TaskConstants.SINGLE_OBJECT_LABLING_MTURK_MIN_RESULTS_TO_AGGREGATE, int MaxResults = TaskConstants.SINGLE_OBJECT_LABLING_MTURK_MAX_RESULTS_TO_AGGREGATE, double MajorityThreshold = TaskConstants.SINGLE_OBJECT_LABLING_MTURK_MAJORITY_THRESHOLD, bool prepareDataForTraining = false, string outputDirectory = null, string confusingVideoListFilePath = null) { string configString = "Min_" + MinResults + "_Max_" + MaxResults + "_Thresh_" + MajorityThreshold; Console.WriteLine("Aggregating for param set " + configString); int noTerminatedTasks = 0; SatyamResultsTableAccess resultsDB = new SatyamResultsTableAccess(); List <SatyamResultsTableEntry> entries = resultsDB.getEntriesByGUIDOrderByID(guid); resultsDB.close(); SortedDictionary <DateTime, List <SatyamResultsTableEntry> > entriesBySubmitTime = SatyamResultValidationToolKit.SortResultsBySubmitTime(entries); int noTotalConverged = 0; int noCorrect = 0; Dictionary <int, List <SatyamResultsTableEntry> > ResultsPerTask = new Dictionary <int, List <SatyamResultsTableEntry> >(); List <int> aggregatedTasks = new List <int>(); SortedDictionary <string, Dictionary <string, int> > confusionMatrix_res_groundtruth = new SortedDictionary <string, Dictionary <string, int> >(); SortedDictionary <string, Dictionary <string, int> > confusionMatrix_groundtruth_res = new SortedDictionary <string, Dictionary <string, int> >(); List <SatyamAggregatedResultsTableEntry> aggEntries = new List <SatyamAggregatedResultsTableEntry>(); Dictionary <int, int> noResultsNeededForAggregation = SatyamResultsAnalysis.getNoResultsNeededForAggregationFromLog(configString, guid); Dictionary <int, int> noResultsNeededForAggregation_new = new Dictionary <int, int>(); foreach (DateTime t in entriesBySubmitTime.Keys) { //Console.WriteLine("Processing Results of time: {0}", t); foreach (SatyamResultsTableEntry entry in entriesBySubmitTime[t]) { SatyamResult satyamResult = JSonUtils.ConvertJSonToObject <SatyamResult>(entry.ResultString); SatyamTask task = JSonUtils.ConvertJSonToObject <SatyamTask>(satyamResult.TaskParametersString); SatyamJob job = task.jobEntry; string fileName = URIUtilities.filenameFromURI(task.SatyamURI); string VideoCategoryGroundTruth = getVideoCategoryFromFileName(fileName); int taskEntryID = entry.SatyamTaskTableEntryID; if (aggregatedTasks.Contains(taskEntryID)) { continue; } if (!ResultsPerTask.ContainsKey(taskEntryID)) { ResultsPerTask.Add(taskEntryID, new List <SatyamResultsTableEntry>()); } ResultsPerTask[taskEntryID].Add(entry); // check log if enough results are collected if (noResultsNeededForAggregation != null && noResultsNeededForAggregation.ContainsKey(taskEntryID) && ResultsPerTask[taskEntryID].Count < noResultsNeededForAggregation[taskEntryID]) { continue; } //SingleObjectLabelingAggregatedResult aggResult = SingleObjectLabelingAggregator.getAggregatedResult(ResultsPerTask[taskEntryID], MinResults, MaxResult, MajorityThreshold); string aggResultString = SingleObjectLabelingAggregator.GetAggregatedResultString(ResultsPerTask[taskEntryID], MinResults, MaxResults, MajorityThreshold); if (aggResultString == null) { continue; } SatyamAggregatedResultsTableEntry aggEntry = new SatyamAggregatedResultsTableEntry(); aggEntry.JobGUID = ResultsPerTask[taskEntryID][0].JobGUID; aggEntry.JobTemplateType = ResultsPerTask[taskEntryID][0].JobTemplateType; aggEntry.SatyamTaskTableEntryID = taskEntryID; aggEntry.UserID = ResultsPerTask[taskEntryID][0].UserID; aggEntry.ResultString = aggResultString; /// aggregation happen // record logs if (noResultsNeededForAggregation == null || !noResultsNeededForAggregation.ContainsKey(taskEntryID)) { noResultsNeededForAggregation_new.Add(taskEntryID, ResultsPerTask[taskEntryID].Count); } aggEntries.Add(aggEntry); noTotalConverged++; if (ResultsPerTask[taskEntryID].Count >= MaxResults) { noTerminatedTasks++; } aggregatedTasks.Add(taskEntryID); } } SatyamResultsAnalysis.RecordAggregationLog(noResultsNeededForAggregation_new, configString, guid); string outputmatFile = DirectoryConstants.defaultTempDirectory + guid + "\\" + configString + "_mat.txt"; EvaluateAndPrintConfusionMatrixOfAggregatedResultEntries(aggEntries, outputmatFile, out noCorrect, prepareDataForTraining, outputDirectory); Console.WriteLine("noTotalConverged {0}", noTotalConverged); Console.WriteLine("noTerminatedTasks {0}", noTerminatedTasks); Console.WriteLine("Result: {0}/{1}, precision: {2}", noCorrect, noTotalConverged, (double)noCorrect / noTotalConverged); /// local file string outputString = String.Format("{0} {1} {2} {3} {4} {5}\n", configString, noCorrect, noTotalConverged, (double)noCorrect / noTotalConverged, noTerminatedTasks, ResultsPerTask.Count - noTotalConverged); string outputfile = DirectoryConstants.defaultTempDirectory + guid + "\\resultSummary.txt"; File.AppendAllText(outputfile, outputString); //for (double prob = 0; prob < 1;prob +=0.2) //{ // SatyamResultsAnalysis.AnalyzeApprovalRate(aggEntries, entries, guid, configString, anotherChanceProbablity: prob); //} //for (double ratio = 0; ratio < 1; ratio += 0.2) //{ // SatyamResultsAnalysis.AnalyzeApprovalRate(aggEntries, entriesBySubmitTime, noResultsNeededForAggregation, noResultsNeededForAggregation_new, guid, configString, approvalRatioThreshold: ratio); //} SatyamResultsAnalysis.AggregationAnalysis(aggEntries, entriesBySubmitTime, noResultsNeededForAggregation, noResultsNeededForAggregation_new, guid, configString); }
// If the results were not good enough, or aggregation method changed, retargeted, users might want to reopen the job // All we need to do: (IN STRICT ORDER) // clear all results back to inconclusive, // remove all aggregated results // restore the task table as it was for the guid, // change the job status back to launched, and // WARNING: // for safety, this has to be run atomically synchronously, without any parallel process. for now. public static void reopenJobForMoreResults(string guid) { SatyamJobSubmissionsTableAccess jobDB = new SatyamJobSubmissionsTableAccess(); SatyamJobSubmissionsTableAccessEntry jobEntry = jobDB.getEntryByJobGIUD(guid); jobDB.close(); if (jobEntry.JobStatus != JobStatus.completed) { Console.WriteLine("Job not completed yet!"); return; } SatyamAggregatedResultsTableAccess aggResultDB = new SatyamAggregatedResultsTableAccess(); bool delSuccess = aggResultDB.DeleteEntriesByGUID(guid); aggResultDB.close(); if (!delSuccess) { Console.WriteLine("Delete Agg Result DB Failed"); return; } SatyamResultsTableAccess resultDB = new SatyamResultsTableAccess(); if (!resultDB.UpdateStatusByGUID(guid, ResultStatus.inconclusive)) { Console.WriteLine("Update Result DB Failed"); //resultDB.close(); //return; } List <SatyamResultsTableEntry> results = resultDB.getEntriesByGUID(guid); resultDB.close(); Dictionary <int, SatyamTask> taskParamsByTaskID = new Dictionary <int, SatyamTask>(); foreach (SatyamResultsTableEntry result in results) { if (taskParamsByTaskID.ContainsKey(result.SatyamTaskTableEntryID)) { continue; } SatyamResult satyamRes = JSonUtils.ConvertJSonToObject <SatyamResult>(result.ResultString); SatyamTask task = JSonUtils.ConvertJSonToObject <SatyamTask>(satyamRes.TaskParametersString); taskParamsByTaskID.Add(result.SatyamTaskTableEntryID, task); } SatyamTaskTableAccess taskDB = new SatyamTaskTableAccess(); foreach (int taskID in taskParamsByTaskID.Keys) { SatyamTask task = taskParamsByTaskID[taskID]; SatyamJob job = task.jobEntry; bool suc = taskDB.AddEntryWithSpecificID(taskID, job.JobTemplateType, job.UserID, job.JobGUIDString, JSonUtils.ConvertObjectToJSon(task), job.JobSubmitTime, job.amazonHITInformation.Price); if (!suc) { Console.WriteLine("Update Task Table Failed"); taskDB.close(); return; } } taskDB.close(); jobDB = new SatyamJobSubmissionsTableAccess(); bool success = jobDB.UpdateEntryStatus(guid, JobStatus.launched); jobDB.close(); if (!success) { Console.WriteLine("Update Job Entry Failed"); return; } }
//go through all unpaid results //check if aggregated results has an entry for this //check if it satisfies acceptance criterion //pay if passes else fail! public static void MakePayments() { //get IDs of all accepted and rejected results related to MTurk SatyamResultsTableAccess resultsDB = new SatyamResultsTableAccess(); Dictionary <string, string> AssignmentIDToHITIDMap = new Dictionary <string, string>(); List <SatyamResultsTableEntry> acceptedEntries = new List <SatyamResultsTableEntry>(); foreach (string taskTemplate in TaskConstants.MTurkTaskTemplates) { List <SatyamResultsTableEntry> entries = resultsDB.getAllEntriesByJobtemplateTypeAndStatus(taskTemplate, ResultStatus.accepted); acceptedEntries.AddRange(entries); } List <SatyamResultsTableEntry> rejectedEntries = new List <SatyamResultsTableEntry>(); foreach (string taskTemplate in TaskConstants.MTurkTaskTemplates) { List <SatyamResultsTableEntry> entries = resultsDB.getAllEntriesByJobtemplateTypeAndStatus(taskTemplate, ResultStatus.rejected); rejectedEntries.AddRange(entries); } resultsDB.close(); if (acceptedEntries.Count == 0 && rejectedEntries.Count == 0) //nothing to do { return; } // a single assignment may have multiple accepted results Dictionary <string, List <SatyamResultsTableEntry> > acceptedResultsByAssignmentID = new Dictionary <string, List <SatyamResultsTableEntry> >(); foreach (SatyamResultsTableEntry entry in acceptedEntries) { SatyamResult sr = JSonUtils.ConvertJSonToObject <SatyamResult>(entry.ResultString); string AssignmentID = sr.amazonInfo.AssignmentID; if (!AssignmentIDToHITIDMap.ContainsKey(AssignmentID)) { string HITID = sr.amazonInfo.HITID; AssignmentIDToHITIDMap.Add(AssignmentID, HITID); } if (!acceptedResultsByAssignmentID.ContainsKey(AssignmentID)) { acceptedResultsByAssignmentID.Add(AssignmentID, new List <SatyamResultsTableEntry>()); } acceptedResultsByAssignmentID[AssignmentID].Add(entry); } List <string> assignmentIDList = acceptedResultsByAssignmentID.Keys.ToList(); // a single assignment may have multiple rejected results Dictionary <string, List <SatyamResultsTableEntry> > rejectedResultsByAssignmentID = new Dictionary <string, List <SatyamResultsTableEntry> >(); foreach (SatyamResultsTableEntry entry in rejectedEntries) { SatyamResult sr = JSonUtils.ConvertJSonToObject <SatyamResult>(entry.ResultString); string AssignmentID = sr.amazonInfo.AssignmentID; if (!AssignmentIDToHITIDMap.ContainsKey(AssignmentID)) { string HITID = sr.amazonInfo.HITID; AssignmentIDToHITIDMap.Add(AssignmentID, HITID); } if (!rejectedResultsByAssignmentID.ContainsKey(AssignmentID)) { rejectedResultsByAssignmentID.Add(AssignmentID, new List <SatyamResultsTableEntry>()); if (!assignmentIDList.Contains(AssignmentID)) { assignmentIDList.Add(AssignmentID); } } rejectedResultsByAssignmentID[AssignmentID].Add(entry); } List <string> rejectedHITs = new List <string>(); List <string> acceptedHITs = new List <string>(); //for each assignment ID we check if it should be paid or not and then pay or reject them. //pay all accepted entries AmazonMTurkHIT hit = new AmazonMTurkHIT(); foreach (string assignmentID in assignmentIDList) { int noAccepted = 0; SatyamResult r = null; SatyamResultsTableEntry entry = null; if (acceptedResultsByAssignmentID.ContainsKey(assignmentID)) { noAccepted = acceptedResultsByAssignmentID[assignmentID].Count; entry = acceptedResultsByAssignmentID[assignmentID][0]; r = JSonUtils.ConvertJSonToObject <SatyamResult>(entry.ResultString); } int noRejected = 0; if (rejectedResultsByAssignmentID.ContainsKey(assignmentID)) { noRejected = rejectedResultsByAssignmentID[assignmentID].Count; if (r == null) { entry = rejectedResultsByAssignmentID[assignmentID][0]; r = JSonUtils.ConvertJSonToObject <SatyamResult>(entry.ResultString); } } double ratio = (double)noAccepted / ((double)noAccepted + (double)noRejected); SatyamTask task = JSonUtils.ConvertJSonToObject <SatyamTask>(r.TaskParametersString); string AmazonAccessKeyID = task.jobEntry.amazonHITInformation.AmazonAccessKeyID; string AmazonSecretAccessKeyID = task.jobEntry.amazonHITInformation.AmazonSecretAccessKeyID; hit.setAccount(AmazonAccessKeyID, AmazonSecretAccessKeyID, false); resultsDB = new SatyamResultsTableAccess(); if (ratio >= assignement_acceptance_threshold) //this is acceptable { hit.ApproveAssignment(assignmentID, "Great Job! Your work was within acceptable parameters!"); if (acceptedResultsByAssignmentID.ContainsKey(assignmentID)) { foreach (SatyamResultsTableEntry result in acceptedResultsByAssignmentID[assignmentID]) { resultsDB.UpdateStatusByID(result.ID, ResultStatus.accepted_Paid); } } if (rejectedResultsByAssignmentID.ContainsKey(assignmentID)) { foreach (SatyamResultsTableEntry result in rejectedResultsByAssignmentID[assignmentID]) { resultsDB.UpdateStatusByID(result.ID, ResultStatus.rejected_Paid); } } if (AssignmentIDToHITIDMap.ContainsKey(assignmentID)) { if (!acceptedHITs.Contains(AssignmentIDToHITIDMap[assignmentID])) { acceptedHITs.Add(AssignmentIDToHITIDMap[assignmentID]); } } } else { hit.RejectAssignment(assignmentID, "Sorry! Your work was not within acceptable parameters!"); if (acceptedResultsByAssignmentID.ContainsKey(assignmentID)) { foreach (SatyamResultsTableEntry result in acceptedResultsByAssignmentID[assignmentID]) { resultsDB.UpdateStatusByID(result.ID, ResultStatus.accepted_NotPaid); } } if (rejectedResultsByAssignmentID.ContainsKey(assignmentID)) { foreach (SatyamResultsTableEntry result in rejectedResultsByAssignmentID[assignmentID]) { resultsDB.UpdateStatusByID(result.ID, ResultStatus.rejected_NotPaid); } } if (AssignmentIDToHITIDMap.ContainsKey(assignmentID)) { if (!rejectedHITs.Contains(AssignmentIDToHITIDMap[assignmentID])) { rejectedHITs.Add(AssignmentIDToHITIDMap[assignmentID]); } } } resultsDB.close(); } //update the status of all finished HITs in the hittable SatyamAmazonHITTableAccess hitDB = new SatyamAmazonHITTableAccess(); foreach (string HITID in acceptedHITs) { hitDB.UpdateStatusByHITID(HITID, HitStatus.accepted); } foreach (string HITID in rejectedHITs) { hitDB.UpdateStatusByHITID(HITID, HitStatus.rejected); } hitDB.close(); }
protected void SubmitButton_Click(object sender, EventArgs e) { DateTime SubmitTime = DateTime.Now; DateTime PageLoadTime = Convert.ToDateTime(Hidden_PageLoadTime.Value); if (CategorySelection_RadioButtonList.SelectedIndex != -1) { SatyamTaskTableEntry taskEntry = JSonUtils.ConvertJSonToObject <SatyamTaskTableEntry>(Hidden_TaskEntryString.Value); SatyamResult result = new SatyamResult(); result.TaskParametersString = taskEntry.TaskParametersString; result.TaskStartTime = PageLoadTime; result.TaskEndTime = SubmitTime; result.TaskTableEntryID = taskEntry.ID; AmazonTaskResultInfo amazonInfo = new AmazonTaskResultInfo(); amazonInfo.AssignmentID = Hidden_AmazonAssignmentID.Value; amazonInfo.WorkerID = Hidden_AmazonWorkerID.Value; amazonInfo.HITID = Hidden_HITID.Value; amazonInfo.PricePerHIT = Convert.ToDouble(Hidden_Price.Value); result.amazonInfo = amazonInfo; SingleObjectLabelingResult sresult = new SingleObjectLabelingResult(); sresult.Category = CategorySelection_RadioButtonList.Items[CategorySelection_RadioButtonList.SelectedIndex].Text; string sresultString = JSonUtils.ConvertObjectToJSon <SingleObjectLabelingResult>(sresult); result.TaskResult = sresultString; string resultString = JSonUtils.ConvertObjectToJSon <SatyamResult>(result); SatyamResultsTableAccess resultdb = new SatyamResultsTableAccess(); resultdb.AddEntry(taskEntry.JobTemplateType, taskEntry.UserID, taskEntry.JobGUID, resultString, taskEntry.ID, PageLoadTime, SubmitTime); resultdb.close(); SatyamTaskTableManagement.UpdateResultNumber(taskEntry.ID); //SatyamTaskTableAccess taskDB = new SatyamTaskTableAccess(); //taskDB.IncrementDoneScore(taskEntry.ID); int noDone = Convert.ToInt32(Hidden_NoImagesDone.Value); noDone++; Hidden_NoImagesDone.Value = noDone.ToString(); int noTasksPerJob = Convert.ToInt32(Hidden_TasksPerJob.Value); bool NotDone = false; if (noDone < noTasksPerJob) { NotDone = getNewRandomJob(); } if (NotDone == false) { if (!Testing) { SatyamAmazonHITTableAccess HITdb = new SatyamAmazonHITTableAccess(); string HITID = result.amazonInfo.HITID; HITdb.UpdateStatusByHITID(HITID, HitStatus.submitted); HITdb.close(); AmazonMTurkNotification.submitAmazonTurkHit(result.amazonInfo.AssignmentID, result.amazonInfo.WorkerID, false); } Response.Redirect("MTurkTaskDonePage.aspx"); } } }
public static void Run([TimerTrigger("0 */5 * * * *")] TimerInfo myTimer, TraceWriter log) { DateTime start = DateTime.Now; bool logging = false; if (logging) { log.Info($"Judge Result Dispatch executed at: {DateTime.Now}"); } //get all inconclusive results and group them by taskIDs SatyamResultsTableAccess resultsDB = new SatyamResultsTableAccess(); List <SatyamResultsTableEntry> results = resultsDB.getAllEntriesByStatus(ResultStatus.inconclusive); resultsDB.close(); if (results.Count == 0) { return; } Dictionary <int, List <SatyamResultsTableEntry> > resultsByTaskID = new Dictionary <int, List <SatyamResultsTableEntry> >(); List <string> pendingGUID = new List <string>(); foreach (SatyamResultsTableEntry result in results) { if (!resultsByTaskID.ContainsKey(result.SatyamTaskTableEntryID)) { resultsByTaskID.Add(result.SatyamTaskTableEntryID, new List <SatyamResultsTableEntry>()); } resultsByTaskID[result.SatyamTaskTableEntryID].Add(result); if (!pendingGUID.Contains(result.JobGUID)) { pendingGUID.Add(result.JobGUID); } } //now check against the aggregated result to see if the result is aceptable SatyamDispatchStorageAccountAccess satyamQueue = new SatyamDispatchStorageAccountAccess(); Dictionary <string, Dictionary <int, SatyamAggregatedResultsTableEntry> > aggEntriesPerTaskPerGUID = SatyamAggregatedResultManagement.getAggregatedEntriesPerTaskByGuidList(pendingGUID); foreach (KeyValuePair <int, List <SatyamResultsTableEntry> > entry in resultsByTaskID) { if (entry.Value.Count == 0) { continue; } int taskEntryID = entry.Key; string taskGUID = entry.Value[0].JobGUID; if (aggEntriesPerTaskPerGUID.ContainsKey(taskGUID) && aggEntriesPerTaskPerGUID[taskGUID].ContainsKey(taskEntryID)) { // this task has been aggregated foreach (SatyamResultsTableEntry result in resultsByTaskID[taskEntryID]) //now got through each task to see if they satify pass criterion { if (logging) { log.Info($"Dispatching Judgement for {taskEntryID}"); } string queueName = "judge-result"; string m = taskGUID + "_" + taskEntryID + "_" + result.ID; satyamQueue.Enqueue(queueName, m); if ((DateTime.Now - start).TotalSeconds > 280) { break; } } } if ((DateTime.Now - start).TotalSeconds > 280) { break; } } if (logging) { log.Info($"Judge Result Dispatch finished at: {DateTime.Now}"); } }