public static double getPaymentRateByGUID(string guid, out int totalResults, out int totalAggregated)
{
SatyamResultsTableAccess resDB = new SatyamResultsTableAccess();
List <SatyamResultsTableEntry> results = resDB.getEntriesByGUID(guid);
int paid = 0;
int notPaid = 0;
totalResults = results.Count;
totalAggregated = 0;
foreach (SatyamResultsTableEntry entry in results)
{
if (ResultStatus.paidStatusList.Contains(entry.Status))
{
paid++;
totalAggregated++;
}
if (ResultStatus.notPaidStatusList.Contains(entry.Status))
{
notPaid++;
totalAggregated++;
}
}
double ratio = (double)paid / (double)(paid + notPaid);
return(ratio);
}
public static void SaveResultImagesLocally(string jobGUID, string directoryName)
{
SatyamResultsTableAccess resultsDB = new SatyamResultsTableAccess();
List <SatyamResultsTableEntry> entries = resultsDB.getEntriesByGUID(jobGUID);
resultsDB.close();
SaveResultImagesLocally(entries, directoryName);
}
//protected void SubmitButton_Click(object sender, EventArgs e)
private void storeResult()
{
DateTime SubmitTime = DateTime.Now;
DateTime PageLoadTime = Convert.ToDateTime(Hidden_PageLoadTime.Value);
SatyamTaskTableEntry taskEntry = JSonUtils.ConvertJSonToObject <SatyamTaskTableEntry>(Hidden_TaskEntryString.Value);
string tracksString = TracksOutput_Hidden.Value;
string urlList = Hidden_ImageURLList.Value;
string[] fields = urlList.Split(',');
DateTime start = DateTime.MinValue;
List <DateTime> frameTimes = new List <DateTime>();
//double frameTimeSpanInMiliseconds = (Convert.ToDouble(Hidden_ChunkDuration.Value) / (double)fields.Length) * 1000;
double frameTimeSpanInMiliseconds = (double)(1000) / Convert.ToDouble(fps_Hidden.Value);
for (int i = 0; i < fields.Length; i++)
{
DateTime t;
t = start.AddMilliseconds(frameTimeSpanInMiliseconds * i);
frameTimes.Add(t);
}
string s = Raw_VATIC_DVA_Crowdsourced_Track_Collection.Raw_VATIC_DVA_Crowdsourced_Track_Collection_ToTrackStrings(tracksString, frameTimes);
SatyamResult result = new SatyamResult();
result.TaskParametersString = taskEntry.TaskParametersString;
result.TaskStartTime = PageLoadTime;
result.TaskEndTime = SubmitTime;
result.TaskTableEntryID = taskEntry.ID;
AmazonTaskResultInfo amazonInfo = new AmazonTaskResultInfo();
amazonInfo.AssignmentID = "";
amazonInfo.WorkerID = "";
amazonInfo.HITID = "";
result.amazonInfo = amazonInfo;
result.TaskResult = s;
string resultString = JSonUtils.ConvertObjectToJSon <SatyamResult>(result);
SatyamResultsTableAccess resultdb = new SatyamResultsTableAccess();
resultdb.AddEntry(taskEntry.JobTemplateType, taskEntry.UserID, taskEntry.JobGUID, resultString, taskEntry.ID, PageLoadTime, SubmitTime);
resultdb.close();
//SatyamTaskTableAccess taskDB = new SatyamTaskTableAccess();
//taskDB.IncrementDoneScore(taskEntry.ID);
bool NotDone = getNewRandomJob();
if (NotDone == false)
{
Response.Redirect("AllJobsDone.aspx");
}
}
public static void SaveResultVideosLocally(string guid)
{
string outputDirectory = DirectoryConstants.defaultTempDirectory + guid;
SatyamResultsTableAccess resultsDB = new SatyamResultsTableAccess();
List <SatyamResultsTableEntry> entries = resultsDB.getEntriesByGUID(guid);
resultsDB.close();
SaveResultVideosLocally(entries, outputDirectory);
}
public static void SaveResultOnlyText(string jobGUID)
{
string directoryName = DirectoryConstants.defaultTempDirectory + "\\" + jobGUID + "\\";
SatyamResultsTableAccess resultsDB = new SatyamResultsTableAccess();
List <SatyamResultsTableEntry> entries = resultsDB.getEntriesByGUID(jobGUID);
resultsDB.close();
if (!Directory.Exists(directoryName))
{
Directory.CreateDirectory(directoryName);
}
directoryName = directoryName + "\\Raw\\";
if (!Directory.Exists(directoryName))
{
Directory.CreateDirectory(directoryName);
}
for (int i = 0; i < entries.Count; i++)
{
SatyamResultsTableEntry entry = entries[i];
SatyamResult satyamResult = JSonUtils.ConvertJSonToObject <SatyamResult>(entry.ResultString);
SatyamTask task = JSonUtils.ConvertJSonToObject <SatyamTask>(satyamResult.TaskParametersString);
SatyamJob job = task.jobEntry;
string result = satyamResult.TaskResult;
string ofilename = URIUtilities.filenameFromURI(task.SatyamURI);
string[] fields = ofilename.Split('.');
string fileName = "";
for (int j = 0; j < fields.Count(); j++)
{
fileName += fields[j];
if (j == fields.Count() - 1)
{
break;
}
fileName += ".";
}
//string fileName = ofilename;
fileName = fileName + "-Result";
fileName = fileName + "-" + entry.ID;
Console.WriteLine("Saving " + fileName);
string resultFile = directoryName + fileName + ".txt";
StreamWriter f = new System.IO.StreamWriter(resultFile);
f.WriteLine(result);
f.Close();
}
}
public static void SaveResultImagesLocally(string jobGUID)
{
string directoryName = DirectoryConstants.defaultTempDirectory + "\\" + jobGUID + "\\";
SatyamResultsTableAccess resultsDB = new SatyamResultsTableAccess();
List <SatyamResultsTableEntry> entries = resultsDB.getEntriesByGUID(jobGUID);
SaveResultImagesLocally(entries, directoryName);
resultsDB.close();
}
public static void UpdateResultNumber(int taskID)
{
SatyamResultsTableAccess resultdb = new SatyamResultsTableAccess();
List <SatyamResultsTableEntry> res = resultdb.getEntriesByTaskID(taskID);
resultdb.close();
SatyamTaskTableAccess taskDB = new SatyamTaskTableAccess();
taskDB.UpdateResultNumber(taskID, res.Count);
taskDB.close();
}
//protected void Page_Unload(object sender, EventArgs e)
//{
// if (!DoneFlag)
// {
// SatyamTaskTableEntry taskEntry = JSonUtils.ConvertJSonToObject<SatyamTaskTableEntry>(Hidden_TaskEntryString.Value);
// if (taskEntry == null) return;
// // Decrement the DoneScore, releasing token after latest results are saved.
// SatyamTaskTableAccess taskTableDB = new SatyamTaskTableAccess();
// taskTableDB.DecrementDoneScore(taskEntry.ID);
// taskTableDB.close();
// }
//}
protected void SubmitButton_Click(object sender, EventArgs e)
{
DateTime SubmitTime = DateTime.Now;
DateTime PageLoadTime = Convert.ToDateTime(Hidden_PageLoadTime.Value);
SatyamTaskTableEntry taskEntry = JSonUtils.ConvertJSonToObject <SatyamTaskTableEntry>(Hidden_TaskEntryString.Value);
SatyamResult result = new SatyamResult();
result.TaskParametersString = taskEntry.TaskParametersString;
result.TaskStartTime = PageLoadTime;
result.TaskEndTime = SubmitTime;
result.TaskTableEntryID = taskEntry.ID;
AmazonTaskResultInfo amazonInfo = new AmazonTaskResultInfo();
amazonInfo.AssignmentID = "";
amazonInfo.WorkerID = "";
amazonInfo.HITID = "";
result.amazonInfo = amazonInfo;
result.TaskResult = Hidden_Result.Value;
result.PrevResultID = Convert.ToInt32(Hidden_PrevResultID.Value);
string resultString = JSonUtils.ConvertObjectToJSon <SatyamResult>(result);
SatyamResultsTableAccess resultdb = new SatyamResultsTableAccess();
resultdb.AddEntry(taskEntry.JobTemplateType, taskEntry.UserID, taskEntry.JobGUID, resultString, taskEntry.ID, PageLoadTime, SubmitTime);
resultdb.close();
SatyamTaskTableManagement.UpdateResultNumber(taskEntry.ID);
// Decrement the DoneScore, releasing token after latest results are saved.
SatyamTaskTableAccess taskTableDB = new SatyamTaskTableAccess();
//taskTableDB.DecrementDoneScore(taskEntry.ID);
taskTableDB.UpdateDoneScore(taskEntry.ID, 0);
taskTableDB.close();
//DoneFlag = true;
//bool NotDone = getNewRandomJob();
//if (NotDone == false)
//{
Response.Redirect("AllJobsDone.aspx");
//}
}
public static List <SatyamResultsTableEntry> getAllEntriesByGUIDFilteredByHITID(string guid, string hitid)
{
List <SatyamResultsTableEntry> ret = new List <SatyamResultsTableEntry>();
SatyamResultsTableAccess resDB = new SatyamResultsTableAccess();
List <SatyamResultsTableEntry> res = resDB.getEntriesByGUID(guid);
resDB.close();
foreach (SatyamResultsTableEntry r in res)
{
SatyamResult rr = JSonUtils.ConvertJSonToObject <SatyamResult>(r.ResultString);
if (rr.amazonInfo.HITID == hitid)
{
ret.Add(r);
}
}
return(ret);
}
protected void SubmitButton_Click(object sender, EventArgs e)
{
DateTime SubmitTime = DateTime.Now;
DateTime PageLoadTime = Convert.ToDateTime(Hidden_PageLoadTime.Value);
SatyamTaskTableEntry taskEntry = JSonUtils.ConvertJSonToObject <SatyamTaskTableEntry>(Hidden_TaskEntryString.Value);
SatyamResult result = new SatyamResult();
result.TaskParametersString = taskEntry.TaskParametersString;
result.TaskStartTime = PageLoadTime;
result.TaskEndTime = SubmitTime;
result.TaskTableEntryID = taskEntry.ID;
AmazonTaskResultInfo amazonInfo = new AmazonTaskResultInfo();
amazonInfo.AssignmentID = Hidden_AmazonAssignmentID.Value;
amazonInfo.WorkerID = Hidden_AmazonWorkerID.Value;
amazonInfo.HITID = Hidden_HITID.Value;
amazonInfo.PricePerHIT = Convert.ToDouble(Hidden_Price.Value);
result.amazonInfo = amazonInfo;
result.TaskResult = Hidden_Result.Value;
string resultString = JSonUtils.ConvertObjectToJSon <SatyamResult>(result);
SatyamResultsTableAccess resultdb = new SatyamResultsTableAccess();
resultdb.AddEntry(taskEntry.JobTemplateType, taskEntry.UserID, taskEntry.JobGUID, resultString, taskEntry.ID, PageLoadTime, SubmitTime);
resultdb.close();
SatyamTaskTableManagement.UpdateResultNumber(taskEntry.ID);
if (!Testing)
{
SatyamAmazonHITTableAccess HITdb = new SatyamAmazonHITTableAccess();
string HITID = result.amazonInfo.HITID;
HITdb.UpdateStatusByHITID(HITID, HitStatus.submitted);
HITdb.close();
AmazonMTurkNotification.submitAmazonTurkHit(result.amazonInfo.AssignmentID, result.amazonInfo.WorkerID, false);
}
Response.Redirect("AllJobsDone.aspx");
//}
}
public static void TestAggregationAnalysis()
{
string guid = "86cbfe43-175a-4a5a-ba89-073f3004e7df";//20 class mturk all pascal
//string guid = "61bf62b5-d796-4b3c-9de9-9c67f04cf26f";// changed taskpage
string configString = "Min_" + 10 + "_Max_" + 10 + "_Majority_" + 0.5 + "_Ratio_" + 0.5;
SatyamAggregatedResultsTableAccess aggDB = new SatyamAggregatedResultsTableAccess();
List <SatyamAggregatedResultsTableEntry> aggEntries = aggDB.getEntriesByGUID(guid);
aggDB.close();
SatyamResultsTableAccess resDB = new SatyamResultsTableAccess();
List <SatyamResultsTableEntry> entries = resDB.getEntriesByGUID(guid);
resDB.close();
List <SatyamResultsTableEntry> filteredResults = new List <SatyamResultsTableEntry>();
//foreach (SatyamResultsTableEntry entry in entries)
//{
// SatyamResult res = JSonUtils.ConvertJSonToObject<SatyamResult>(entry.ResultString);
// string workerID = res.amazonInfo.WorkerID;
// string assignmentID = res.amazonInfo.AssignmentID;
// if (assignmentID == "" || assignmentID == "ASSIGNMENT_ID_NOT_AVAILABLE") continue;
// if (workerID == "") continue;
// /// im seg only
// ImageSegmentationResult res_imseg = JSonUtils.ConvertJSonToObject<ImageSegmentationResult>(res.TaskResult);
// if (res_imseg == null) continue;
// filteredResults.Add(entry);
//}
filteredResults = entries;
SortedDictionary <DateTime, List <SatyamResultsTableEntry> > entriesBySubmitTime =
SatyamResultValidationToolKit.SortResultsBySubmitTime_OneResultPerTurkerPerTask(filteredResults);
SatyamResultsAnalysis.AggregationAnalysis(aggEntries, entriesBySubmitTime, null, null, guid, configString,
usePreApprovalResult: false,
approvalRatioThreshold: 0.5);
}
public static void TestManualAcceptReject()
{
//string path = @"C:\research\listturker.txt";
string path = @"C:\research\topay.txt";
string guid = "dff5415b-b278-43db-8e32-4f654ca4b3a1";
SatyamResultsTableAccess resultsDB = new SatyamResultsTableAccess();
List <SatyamResultsTableEntry> results = resultsDB.getEntriesByGUID(guid);
foreach (string line in File.ReadLines(path))
{
string[] cs = line.Split('-');
string id = cs[cs.Length - 1].Split('.')[0];
//Console.WriteLine(id);
resultsDB.UpdateStatusByID(Convert.ToInt32(id), ResultStatus.accepted);
}
}
private MultiObjectLocalizationAndLabelingResult LoadLatestTurkerResult(SatyamTaskTableEntry entry)
{
SatyamResultsTableAccess resultsDB = new SatyamResultsTableAccess();
List <SatyamResultsTableEntry> entries = resultsDB.getEntriesByGUIDAndTaskID(entry.JobGUID, entry.ID);
if (entries.Count == 0)
{
return(null);
}
//organized sequentially
SatyamResultsTableEntry prevResult = entries[entries.Count - 1];
Hidden_PrevResultID.Value = prevResult.ID.ToString();
SatyamResult satyamResult = JSonUtils.ConvertJSonToObject <SatyamResult>(prevResult.ResultString);
MultiObjectLocalizationAndLabelingResult res = JSonUtils.ConvertJSonToObject <MultiObjectLocalizationAndLabelingResult>(satyamResult.TaskResult);
return(res);
}
protected void SubmitButton_Click(object sender, EventArgs e)
{
DateTime SubmitTime = DateTime.Now;
DateTime PageLoadTime = Convert.ToDateTime(Hidden_PageLoadTime.Value);
if (CategorySelection_RadioButtonList.SelectedIndex != -1)
{
SatyamTaskTableEntry taskEntry = JSonUtils.ConvertJSonToObject <SatyamTaskTableEntry>(Hidden_TaskEntryString.Value);
SatyamResult result = new SatyamResult();
result.TaskParametersString = taskEntry.TaskParametersString;
result.TaskStartTime = PageLoadTime;
result.TaskEndTime = SubmitTime;
result.TaskTableEntryID = taskEntry.ID;
AmazonTaskResultInfo amazonInfo = new AmazonTaskResultInfo();
amazonInfo.AssignmentID = "";
amazonInfo.WorkerID = "";
amazonInfo.HITID = "";
result.amazonInfo = amazonInfo;
SingleObjectLabelingResult sresult = new SingleObjectLabelingResult();
sresult.Category = CategorySelection_RadioButtonList.Items[CategorySelection_RadioButtonList.SelectedIndex].Text;
string sresultString = JSonUtils.ConvertObjectToJSon <SingleObjectLabelingResult>(sresult);
result.TaskResult = sresultString;
string resultString = JSonUtils.ConvertObjectToJSon <SatyamResult>(result);
SatyamResultsTableAccess resultdb = new SatyamResultsTableAccess();
resultdb.AddEntry(taskEntry.JobTemplateType, taskEntry.UserID, taskEntry.JobGUID, resultString, taskEntry.ID, PageLoadTime, SubmitTime);
resultdb.close();
//SatyamTaskTableAccess taskDB = new SatyamTaskTableAccess();
//taskDB.IncrementDoneScore(taskEntry.ID);
bool status = getNewRandomJob();
if (!status)
{
Response.Redirect("AllJobsDone.aspx");
}
}
}
public static void Run([QueueTrigger("aggregation")] string guidAndTaskID, TraceWriter log)
{
bool logging = false;
string[] fields = guidAndTaskID.Split('_');
string guid = fields[0];
int taskID = Convert.ToInt32(fields[1]);
SatyamResultsTableAccess resultsDB = new SatyamResultsTableAccess();
List <SatyamResultsTableEntry> collectedResults = resultsDB.getEntriesByGUIDAndTaskID(guid, taskID);
resultsDB.close();
if (collectedResults.Count == 0)
{
return;
}
if (logging)
{
log.Info($"Task Type: {collectedResults[0].JobTemplateType}({guid}), Task ID: {taskID}, # of Results: {collectedResults.Count}");
}
//SatyamAggregatedResultsTableAccess aggDB = new SatyamAggregatedResultsTableAccess();
//int LatestResultsAggregated = aggDB.getLatestNoResultsAggregatedByTaskID(taskID);
//aggDB.close();
//if (collectedResults.Count != LatestResultsAggregated)
//{
SatyamAggregatedResultsTableEntry aggEntry = ResultsTableAggregator.GetAggregatedResultString(taskID, collectedResults);
if (aggEntry != null)
{
//aggEntries.Add(aggEntry);
SatyamAggregatedResultsTableAccess aggDB = new SatyamAggregatedResultsTableAccess();
aggDB.AddEntry(aggEntry, collectedResults.Count);
aggDB.close();
if (logging)
{
log.Info($"Aggregated");
}
}
//}
}
public static void SaveByGUIDRequester(string guid)
{
//get all the results of a GUID
SatyamResultsTableAccess resultsDB = new SatyamResultsTableAccess();
List <SatyamResultsTableEntry> results = resultsDB.getEntriesByGUID(guid);
if (results.Count == 0)
{
resultsDB.close();
return;
}
//get the azure info
SatyamResult result = JSonUtils.ConvertJSonToObject <SatyamResult>(results[0].ResultString);
SatyamTask task = JSonUtils.ConvertJSonToObject <SatyamTask>(result.TaskParametersString);
SatyamJob job = task.jobEntry;
string ConnectionString = job.azureInformation.AzureBlobStorageConnectionString;
string ContainerName = job.azureInformation.AzureBlobStorageContainerName;
string DirectoryName = job.azureInformation.AzureBlobStorageContainerDirectoryName;
//create the data to be saved
List <SatyamResultSaveDataRequester> savingDataList = new List <SatyamResultSaveDataRequester>();
StringBuilder s = new StringBuilder();
for (int i = 0; i < results.Count; i++)
{
SatyamResultSaveDataRequester data = new SatyamResultSaveDataRequester(results[i]);
String jsonString = JSonUtils.ConvertObjectToJSon(data);
s.Append(jsonString);
if (i == results.Count - 1)
{
s.Append("\n");
}
}
string dataToBeSaved = s.ToString();
BlobContainerManager bcm = new BlobContainerManager();
bcm.Connect(ConnectionString);
string FileName = "Results-" + results[0].JobGUID + ".txt";
bcm.SaveATextFile(ContainerName, DirectoryName, FileName, dataToBeSaved);
resultsDB.close();
}
public static void TestGetGlobalStatistics()
{
Dictionary <string, int> workerSubmissions = new Dictionary <string, int>();
Dictionary <string, int> HitSubmissions = new Dictionary <string, int>();
Dictionary <int, int> TaskSubmissions = new Dictionary <int, int>();
SatyamResultsTableAccess resultsDB = new SatyamResultsTableAccess();
List <SatyamResultsTableEntry> entries = new List <SatyamResultsTableEntry>();
entries = (resultsDB.getAllEntries());
for (int i = 0; i < entries.Count; i++)
{
SatyamResultsTableEntry entry = entries[i];
SatyamResult satyamResult = JSonUtils.ConvertJSonToObject <SatyamResult>(entry.ResultString);
SatyamTask task = JSonUtils.ConvertJSonToObject <SatyamTask>(satyamResult.TaskParametersString);
SatyamJob job = task.jobEntry;
string worker = satyamResult.amazonInfo.WorkerID;
string hitID = satyamResult.amazonInfo.HITID;
int taskID = entry.SatyamTaskTableEntryID;
if (!workerSubmissions.ContainsKey(worker))
{
workerSubmissions.Add(worker, 0);
}
workerSubmissions[worker]++;
if (!HitSubmissions.ContainsKey(hitID))
{
HitSubmissions.Add(hitID, 0);
}
HitSubmissions[hitID]++;
if (!TaskSubmissions.ContainsKey(taskID))
{
TaskSubmissions.Add(taskID, 0);
}
TaskSubmissions[taskID]++;
}
Console.WriteLine(workerSubmissions.Count);
Console.WriteLine(HitSubmissions.Count);
Console.WriteLine(TaskSubmissions.Count);
}
protected void SubmitButton_Click(object sender, EventArgs e)
{
DateTime SubmitTime = DateTime.Now;
DateTime PageLoadTime = Convert.ToDateTime(Hidden_PageLoadTime.Value);
SatyamTaskTableEntry taskEntry = JSonUtils.ConvertJSonToObject <SatyamTaskTableEntry>(Hidden_TaskEntryString.Value);
SatyamResult result = new SatyamResult();
result.TaskParametersString = taskEntry.TaskParametersString;
result.TaskStartTime = PageLoadTime;
result.TaskEndTime = SubmitTime;
result.TaskTableEntryID = taskEntry.ID;
AmazonTaskResultInfo amazonInfo = new AmazonTaskResultInfo();
amazonInfo.AssignmentID = "";
amazonInfo.WorkerID = "";
amazonInfo.HITID = "";
result.amazonInfo = amazonInfo;
result.TaskResult = Hidden_Result.Value;
string resultString = JSonUtils.ConvertObjectToJSon <SatyamResult>(result);
SatyamResultsTableAccess resultdb = new SatyamResultsTableAccess();
resultdb.AddEntry(taskEntry.JobTemplateType, taskEntry.UserID, taskEntry.JobGUID, resultString, taskEntry.ID, PageLoadTime, SubmitTime);
resultdb.close();
//SatyamTaskTableAccess taskDB = new SatyamTaskTableAccess();
//taskDB.IncrementDoneScore(taskEntry.ID);
bool NotDone = getNewRandomJob();
if (NotDone == false)
{
Response.Redirect("AllJobsDone.aspx");
}
}
public static void SaveByGUIDSatyam(string guid)
{
//get all the results of a GUID
SatyamResultsTableAccess resultsDB = new SatyamResultsTableAccess();
List <SatyamResultsTableEntry> results = resultsDB.getEntriesByGUID(guid);
if (results.Count == 0)
{
resultsDB.close();
return;
}
//create the data to be saved
List <SatyamResultSaveDataSatyam> savingDataList = new List <SatyamResultSaveDataSatyam>();
StringBuilder s = new StringBuilder();
for (int i = 0; i < results.Count; i++)
{
SatyamResultSaveDataSatyam data = new SatyamResultSaveDataSatyam(results[i]);
String jsonString = JSonUtils.ConvertObjectToJSon(data);
s.Append(jsonString);
if (i == results.Count - 1)
{
s.Append("\n");
}
}
string dataToBeSaved = s.ToString();
SatyamJobStorageAccountAccess storage = new SatyamJobStorageAccountAccess();
string FileName = "Results-" + results[0].JobGUID + ".txt";
string satyamDirectoryName = SatyamTaskGenerator.JobTemplateToSatyamContainerNameMap[results[0].JobTemplateType];
storage.SaveATextFile(satyamDirectoryName, results[0].JobGUID, FileName, dataToBeSaved);
resultsDB.close();
}
public static SatyamResultPerJobDataAnalysis AnalyzeFromResultsTable(List <string> jobGUIDs)
{
Console.WriteLine("Analyzing Results Statistics of {0}", jobGUIDs[0]);
SatyamResultsTableAccess resultsDB = new SatyamResultsTableAccess();
List <SatyamResultsTableEntry> entries = new List <SatyamResultsTableEntry>();
foreach (string guid in jobGUIDs)
{
entries.AddRange(resultsDB.getEntriesByGUIDOrderByID(guid));
}
resultsDB.close();
if (entries.Count == 0)
{
return(null);
}
SatyamResultPerJobDataAnalysis ana = new SatyamResultPerJobDataAnalysis();
ana.JobGUID = entries[0].JobGUID;
ana.totalResults = entries.Count;
for (int i = 0; i < entries.Count; i++)
{
SatyamResultsTableEntry entry = entries[i];
SatyamResult satyamResult = JSonUtils.ConvertJSonToObject <SatyamResult>(entry.ResultString);
SatyamTask task = JSonUtils.ConvertJSonToObject <SatyamTask>(satyamResult.TaskParametersString);
SatyamJob job = task.jobEntry;
if (ana.job == null)
{
ana.job = job;
}
ana.jobSubmitTime = job.JobSubmitTime;
double moneyPerResult = 0;
if (job.TasksPerJob > 0)
{
moneyPerResult = (double)satyamResult.amazonInfo.PricePerHIT / (double)job.TasksPerJob;
}
ana.PricePerResult.Add(entry.ID, moneyPerResult);
int taskEntryID = entry.SatyamTaskTableEntryID;
if (!ana.noResultsPerTask.ContainsKey(taskEntryID))
{
ana.noResultsPerTask.Add(taskEntryID, 0);
ana.noAcceptedResultsPerTask.Add(taskEntryID, 0);
ana.noPaidResultsPerTask.Add(taskEntryID, 0);
ana.moneyPaidPerTask.Add(taskEntryID, 0);
ana.finalTaskEndTime.Add(taskEntryID, entry.SubmitTime);
ana.ResultsPerWorkerPerTask.Add(taskEntryID, new Dictionary <string, List <string> >());
ana.noDifferentDuplicateResultsPerWorkerPerTask.Add(taskEntryID, new Dictionary <string, int>());
ana.taskSatyamUri.Add(taskEntryID, task.SatyamURI);
}
ana.noResultsPerTask[taskEntryID]++;
ana.SequenceNumberOfResultPerTask.Add(ana.noResultsPerTask[taskEntryID]);
if (entry.Status == ResultStatus.accepted || entry.Status == ResultStatus.accepted_Paid || entry.Status == ResultStatus.accepted_NotPaid)
{
ana.noAcceptedResultsPerTask[taskEntryID]++;
ana.totalAcceptedResults++;
}
if (entry.Status == ResultStatus.accepted_Paid || entry.Status == ResultStatus.rejected_Paid)
{
ana.noPaidResultsPerTask[taskEntryID]++;
ana.totalpaidResults++;
ana.moneyPaidPerTask[taskEntryID] += moneyPerResult;
ana.totalMoneyPaid += moneyPerResult;
}
if (ana.finalTaskEndTime[taskEntryID] < entry.SubmitTime)
{
ana.finalTaskEndTime[taskEntryID] = entry.SubmitTime;
}
if (ana.lastTaskEndTime < entry.SubmitTime)
{
ana.lastTaskEndTime = entry.SubmitTime;
}
TimeSpan dt = satyamResult.TaskEndTime - satyamResult.TaskStartTime;
ana.timeTakenPerResult.Add(dt.TotalSeconds);
if (!ana.AcceptedTimePerSequenceNumber.ContainsKey(ana.noResultsPerTask[taskEntryID]))
{
ana.AcceptedTimePerSequenceNumber.Add(ana.noResultsPerTask[taskEntryID], new List <double>());
ana.TimePerSequenceNumber.Add(ana.noResultsPerTask[taskEntryID], new List <double>());
}
//ana.acceptedTimeTakenPerResultOrganizedByResultSequencePerTask[ana.noResultsPerTask[taskEntryID]].Add(dt.TotalSeconds);
ana.TimePerSequenceNumber[ana.noResultsPerTask[taskEntryID]].Add(dt.TotalSeconds);
if (!ana.AcceptedTimePerTask.ContainsKey(taskEntryID))
{
ana.AcceptedTimePerTask.Add(taskEntryID, new List <double>());
ana.TimePerTask.Add(taskEntryID, new List <double>());
}
//ana.AcceptedTimePerTask[taskEntryID].Add(dt.TotalSeconds);
ana.TimePerTask[taskEntryID].Add(dt.TotalSeconds);
if (ana.lastTaskEndTime < satyamResult.TaskEndTime)
{
ana.lastTaskEndTime = satyamResult.TaskEndTime;
}
if (!ana.finalHITEndTime.ContainsKey(satyamResult.amazonInfo.HITID))
{
ana.finalHITEndTime.Add(satyamResult.amazonInfo.HITID, (entry.SubmitTime - job.JobSubmitTime).TotalSeconds);
}
if (ana.finalHITEndTime[satyamResult.amazonInfo.HITID] < (entry.SubmitTime - job.JobSubmitTime).TotalSeconds)
{
ana.finalHITEndTime[satyamResult.amazonInfo.HITID] = (entry.SubmitTime - job.JobSubmitTime).TotalSeconds;
}
if (entry.Status == ResultStatus.accepted || entry.Status == ResultStatus.accepted_Paid || entry.Status == ResultStatus.accepted_NotPaid)
{
ana.acceptedTimeTakenPerResult.Add(dt.TotalSeconds);
ana.AcceptedTimePerTask[taskEntryID].Add(dt.TotalSeconds);
ana.AcceptedTimePerSequenceNumber[ana.noResultsPerTask[taskEntryID]].Add(dt.TotalSeconds);
}
ana.resultArrivalTimes.Add((entry.SubmitTime - job.JobSubmitTime).TotalSeconds);
if (!ana.noJobsPerWorker.ContainsKey(satyamResult.amazonInfo.WorkerID))
{
ana.noJobsPerWorker.Add(satyamResult.amazonInfo.WorkerID, 0);
ana.noAcceptedJobsPerWorker.Add(satyamResult.amazonInfo.WorkerID, 0);
ana.noPaidJobsPerWorker.Add(satyamResult.amazonInfo.WorkerID, 0);
}
ana.noJobsPerWorker[satyamResult.amazonInfo.WorkerID]++;
ana.SequenceNumberOfResultPerWorker.Add(ana.noJobsPerWorker[satyamResult.amazonInfo.WorkerID]);
if (entry.Status == ResultStatus.accepted || entry.Status == ResultStatus.accepted_Paid || entry.Status == ResultStatus.accepted_NotPaid)
{
ana.noAcceptedJobsPerWorker[satyamResult.amazonInfo.WorkerID]++;
}
if (entry.Status == ResultStatus.accepted_Paid || entry.Status == ResultStatus.rejected_Paid)
{
ana.noPaidJobsPerWorker[satyamResult.amazonInfo.WorkerID]++;
}
if (!ana.ResultsPerWorkerPerTask[taskEntryID].ContainsKey(satyamResult.amazonInfo.WorkerID))
{
ana.ResultsPerWorkerPerTask[taskEntryID].Add(satyamResult.amazonInfo.WorkerID, new List <string>());
ana.noDifferentDuplicateResultsPerWorkerPerTask[taskEntryID].Add(satyamResult.amazonInfo.WorkerID, 0);
}
else
{
// this worker has done this task before
ana.noDuplicateResultsFromSameWorker++;
}
if (!ana.ResultsPerWorkerPerTask[taskEntryID][satyamResult.amazonInfo.WorkerID].Contains(satyamResult.TaskResult))
{
ana.noDifferentDuplicateResultsPerWorkerPerTask[taskEntryID][satyamResult.amazonInfo.WorkerID]++;
if (ana.noDifferentDuplicateResultsPerWorkerPerTask[taskEntryID][satyamResult.amazonInfo.WorkerID] > 1)
{
ana.noDecisionChangeAmongDuplicateResultsOfSameWorkerSameTask++;
}
}
ana.ResultsPerWorkerPerTask[taskEntryID][satyamResult.amazonInfo.WorkerID].Add(satyamResult.TaskResult);
///////////////// imagenet only
//List<SingleObjectLabelingResult> CurrentResultsOfThisTask = new List<SingleObjectLabelingResult>();
//foreach(string worker in ana.ResultsPerWorkerPerTask[taskEntryID].Keys)
//{
// foreach(string workerResult in ana.ResultsPerWorkerPerTask[taskEntryID][worker])
// {
// CurrentResultsOfThisTask.Add(JSonUtils.ConvertJSonToObject<SingleObjectLabelingResult>(workerResult));
// }
//}
//if (SingleObjectLabelingAggregator.getAggregatedResult(CurrentResultsOfThisTask) != null)
//{
// // aggregation happened
// if (!ana.timeTakenTillAggregationPerTask.ContainsKey(taskEntryID))
// {
// // for the first time aggregated
// ana.timeTakenTillAggregationPerTask.Add(taskEntryID, (entry.SubmitTime - job.JobSubmitTime).TotalSeconds);
// }
//}
///////////////////////
}
ana.resultArrivalTimes.Sort();
ana.noTasks = ana.noResultsPerTask.Count;
ana.timeTakenPerResult.Sort();
ana.medianTimeTakenPerResultInSeconds = ana.timeTakenPerResult[ana.timeTakenPerResult.Count / 2];
if (ana.acceptedTimeTakenPerResult.Count != 0)
{
ana.acceptedTimeTakenPerResult.Sort();
ana.medianAcceptedTimeTakenPerResultInSeconds = ana.acceptedTimeTakenPerResult[ana.acceptedTimeTakenPerResult.Count / 2];
}
ana.totalTimeTakenForJob = ana.lastTaskEndTime - ana.jobSubmitTime;
///////////////////////////////////// Per Task Analysis //////////////////////////////////////
ana.resultsPerTaskHistogram = new SortedDictionary <int, int>();
ana.resultsAcceptedPerTaskHistogram = new SortedDictionary <int, int>();
ana.resultsPaidPerTaskHistogram = new SortedDictionary <int, int>();
ana.moneyPaidPerTaskHistogram = new SortedDictionary <int, int>(); //cents
ana.ResultsPerWorkerPerTaskHistogram = new SortedDictionary <int, int>();
int noAggregatedTasks = 0;
List <int> taskIDs = ana.noResultsPerTask.Keys.ToList();
foreach (int taskID in taskIDs)
{
if (!ana.resultsPerTaskHistogram.ContainsKey(ana.noResultsPerTask[taskID]))
{
ana.resultsPerTaskHistogram.Add(ana.noResultsPerTask[taskID], 0);
}
ana.resultsPerTaskHistogram[ana.noResultsPerTask[taskID]]++;
if (!ana.resultsAcceptedPerTaskHistogram.ContainsKey(ana.noAcceptedResultsPerTask[taskID]))
{
ana.resultsAcceptedPerTaskHistogram.Add(ana.noAcceptedResultsPerTask[taskID], 0);
}
ana.resultsAcceptedPerTaskHistogram[ana.noAcceptedResultsPerTask[taskID]]++;
if (!ana.resultsPaidPerTaskHistogram.ContainsKey(ana.noPaidResultsPerTask[taskID]))
{
ana.resultsPaidPerTaskHistogram.Add(ana.noPaidResultsPerTask[taskID], 0);
}
ana.resultsPaidPerTaskHistogram[ana.noPaidResultsPerTask[taskID]]++;
int moneyPaid = (int)Math.Floor(ana.moneyPaidPerTask[taskID] * 100);
if (!ana.moneyPaidPerTaskHistogram.ContainsKey(moneyPaid))
{
ana.moneyPaidPerTaskHistogram.Add(moneyPaid, 0);
}
ana.moneyPaidPerTaskHistogram[moneyPaid]++;
if (ana.ResultsPerWorkerPerTask[taskID].Count != ana.noResultsPerTask[taskID])
{
//has multiple results from same turker
ana.noTasksWithDuplicateResultsFromSameWorker++;
}
if (ana.noPaidResultsPerTask[taskID] != 0)
{
noAggregatedTasks++;
}
}
ana.avgNoPaidResultsPerTask = (double)ana.totalpaidResults / (double)noAggregatedTasks;
//////////// duplicate analysis
//Console.WriteLine("DuplicateResultsHistogram");
//foreach (int no in ana.ResultsPerWorkerPerTaskHistogram.Keys)
//{
// Console.WriteLine("{0}, {1}", no, ana.ResultsPerWorkerPerTaskHistogram[no]);
//}
//Console.WriteLine("{0} images (in total {1}({2})) has duplicate results from same turker, \n" +
// "\t{3} images have >=1 workers making more than majority number of results.\n" +
// "\t{11} images aggregation is changed ({12} of which incorrectly) by duplicate results from same turker.\n" +
// "\t{4} images(In total {5} times) with duplicate results include mixed(>= 2) decisions from same worker.\n" +
// //"\t\t{6} images has correct decision among duplicate decisions.\n" +
// //"\t\t{7} images({8} times) a worker has switched from incorrect to correct choice\n" +
// //"\t\t\t{9} images({10} times) of which maintained the correct choice till the last time of their job on that image.",
// ana.noTasksWithDuplicateResultsFromSameWorker, ana.noDuplicateResultsFromSameWorker, 0,
// ana.noTaskWithDuplicateResultsAsMajority,
// ana.noTasksWithMixedResultsFromSameWorker,ana.noDecisionChangeAmongDuplicateResultsOfSameWorkerSameTask,
// ana.noTasksWithCorrectDuplicateResultsOfSameWorkerSameTask
// //ana.noTasksWithWorkerSwitchedToCorrect, ana.noWorkerSwitchedToCorrect,
// //ana.noTasksWithWorkerSwitchedToCorrectAndMaintained, ana.noWorkerSwitchedToCorrectAndMaintained,
// //ana.noTaskWhoseDuplicateResultsChangedAggregation, ana.noTaskWhoseDuplicateResultsChangedAggregationIncorrectly
//);
//////////////////////////////
string outputDir = DirectoryConstants.defaultTempDirectory + jobGUIDs[0] + "\\ResultStatistics\\";
if (!Directory.Exists(outputDir))
{
Directory.CreateDirectory(outputDir);
}
//Console.WriteLine("timePerResult");
TextWriter tw = new StreamWriter(outputDir + "timePerResult.txt");
foreach (double time in ana.timeTakenPerResult)
{
tw.WriteLine(time);
}
tw.Close();
tw = new StreamWriter(outputDir + "AcceptedTimePerSequenceNumber.txt");
foreach (int seq in ana.AcceptedTimePerSequenceNumber.Keys)
{
foreach (double time in ana.AcceptedTimePerSequenceNumber[seq])
{
tw.WriteLine(time);
}
tw.WriteLine("");
}
tw.Close();
tw = new StreamWriter(outputDir + "AcceptedTimePerTask.txt");
foreach (int taskid in ana.AcceptedTimePerTask.Keys)
{
foreach (double time in ana.AcceptedTimePerTask[taskid])
{
tw.WriteLine(time);
}
tw.WriteLine("");
}
tw.Close();
tw = new StreamWriter(outputDir + "TimePerSequenceNumber.txt");
foreach (int seq in ana.TimePerSequenceNumber.Keys)
{
foreach (double time in ana.TimePerSequenceNumber[seq])
{
tw.WriteLine(time);
}
tw.WriteLine("");
}
tw.Close();
tw = new StreamWriter(outputDir + "TimePerTask.txt");
foreach (int taskid in ana.TimePerTask.Keys)
{
foreach (double time in ana.TimePerTask[taskid])
{
tw.WriteLine(time);
}
tw.WriteLine("");
}
tw.Close();
//Console.WriteLine("resultArrivalTime");
TextWriter tw0 = new StreamWriter(outputDir + "resultArrivalTime.txt");
foreach (double time in ana.resultArrivalTimes)
{
tw0.WriteLine(time);
}
tw0.Close();
//Console.WriteLine("HITFinishTime");
TextWriter tw2 = new StreamWriter(outputDir + "HITFinishTime.txt");
foreach (double time in ana.finalHITEndTime.Values)
{
tw2.WriteLine(time);
}
tw2.Close();
//Console.WriteLine("SequenceNumberOfResultsPerTask");
TextWriter tw3 = new StreamWriter(outputDir + "SequenceNumberOfResultsPerTask.txt");
foreach (int seq in ana.SequenceNumberOfResultPerTask)
{
tw3.WriteLine(seq);
}
tw3.Close();
//Console.WriteLine("SequenceNumberOfResultsPerWorker");
TextWriter tw4 = new StreamWriter(outputDir + "SequenceNumberOfResultsPerWorker.txt");
foreach (int seq in ana.SequenceNumberOfResultPerWorker)
{
tw4.WriteLine(seq);
}
tw4.Close();
//Console.WriteLine("MoneyPaidPerResult");
TextWriter tw5 = new StreamWriter(outputDir + "MoneyPaidPerResult.txt");
foreach (int seq in ana.PricePerResult.Keys)
{
tw5.WriteLine(ana.PricePerResult[seq]);
}
tw5.Close();
TextWriter tw6 = new StreamWriter(outputDir + "acceptedTimePerResult.txt");
foreach (double time in ana.acceptedTimeTakenPerResult)
{
tw6.WriteLine(time);
}
tw6.Close();
///////////////////// aggregation time histogram, out of scope
//Console.WriteLine("resultArrivalTime",
// TaskConstants.SINGLE_OBJECT_LABLING_MTURK_MIN_RESULTS_TO_AGGREGATE, TaskConstants.SINGLE_OBJECT_LABLING_MTURK_MAX_RESULTS_TO_AGGREGATE);
//string filepath = String.Format(@"C:\research\MSRDVA\SatyamResearch\aggregationTimePerTask_MinNoResults_{0}_MaxNoResults_{1}.txt",
// TaskConstants.SINGLE_OBJECT_LABLING_MTURK_MIN_RESULTS_TO_AGGREGATE, TaskConstants.SINGLE_OBJECT_LABLING_MTURK_MAX_RESULTS_TO_AGGREGATE);
//TextWriter tw1 = new StreamWriter(filepath);
//foreach (double time in ana.timeTakenTillAggregationPerTask.Values)
//{
// tw1.WriteLine(time);
//}
//tw1.Close();
Console.WriteLine("Median Time For Accepted Task: {0}", ana.medianAcceptedTimeTakenPerResultInSeconds);
Console.WriteLine("Median Time For All Task: {0}", ana.medianTimeTakenPerResultInSeconds);
Console.WriteLine("Total Tasks: {0}", ana.noTasks);
Console.WriteLine("Total Accepted Results: {0}", ana.totalAcceptedResults);
Console.WriteLine("Total Results: {0}", ana.totalResults);
Console.WriteLine("Avg Paid Results Per Task: {0}", ana.avgNoPaidResultsPerTask);
return(ana);
}
protected void SubmitButton_Click(object sender, EventArgs e)
{
DateTime SubmitTime = DateTime.Now;
DateTime PageLoadTime = Convert.ToDateTime(Hidden_PageLoadTime.Value);
int count;
bool isValidCount = int.TryParse(CountTextBox.Text, out count);
if (!isValidCount)
{
ErrorLabel.Text = "Error : Ivalid Count.";
ErrorLabel.ForeColor = System.Drawing.Color.Red;
ErrorLabel.Font.Bold = true;
}
else if (count < 0)
{
ErrorLabel.Text = "Error : Count cannot be less than zero.";
ErrorLabel.ForeColor = System.Drawing.Color.Red;
ErrorLabel.Font.Bold = true;
}
else
{
ErrorLabel.Text = "";
CountTextBox.Text = "";
SatyamTaskTableEntry taskEntry = JSonUtils.ConvertJSonToObject <SatyamTaskTableEntry>(Hidden_TaskEntryString.Value);
SatyamResult result = new SatyamResult();
result.TaskParametersString = taskEntry.TaskParametersString;
result.TaskStartTime = PageLoadTime;
result.TaskEndTime = SubmitTime;
result.TaskTableEntryID = taskEntry.ID;
AmazonTaskResultInfo amazonInfo = new AmazonTaskResultInfo();
amazonInfo.AssignmentID = Hidden_AmazonAssignmentID.Value;
amazonInfo.WorkerID = Hidden_AmazonWorkerID.Value;
amazonInfo.HITID = Hidden_HITID.Value;
amazonInfo.PricePerHIT = Convert.ToDouble(Hidden_Price.Value);
result.amazonInfo = amazonInfo;
ObjectCountingResult sresult = new ObjectCountingResult();
sresult.Count = count;
string sresultString = JSonUtils.ConvertObjectToJSon <ObjectCountingResult>(sresult);
result.TaskResult = sresultString;
string resultString = JSonUtils.ConvertObjectToJSon <SatyamResult>(result);
SatyamResultsTableAccess resultdb = new SatyamResultsTableAccess();
resultdb.AddEntry(taskEntry.JobTemplateType, taskEntry.UserID, taskEntry.JobGUID, resultString, taskEntry.ID, PageLoadTime, SubmitTime);
resultdb.close();
SatyamTaskTableManagement.UpdateResultNumber(taskEntry.ID);
//SatyamTaskTableAccess taskDB = new SatyamTaskTableAccess();
//taskDB.IncrementDoneScore(taskEntry.ID);
int noDone = Convert.ToInt32(Hidden_NoImagesDone.Value);
noDone++;
Hidden_NoImagesDone.Value = noDone.ToString();
bool NotDone = false;
int noTasksPerJob = Convert.ToInt32(Hidden_TasksPerJob.Value);
if (noDone < noTasksPerJob)
{
NotDone = getNewRandomJob();
}
if (NotDone == false)
{
if (!Testing)
{
SatyamAmazonHITTableAccess HITdb = new SatyamAmazonHITTableAccess();
string HITID = result.amazonInfo.HITID;
HITdb.UpdateStatusByHITID(HITID, HitStatus.submitted);
HITdb.close();
AmazonMTurkNotification.submitAmazonTurkHit(result.amazonInfo.AssignmentID, result.amazonInfo.WorkerID, false);
}
Response.Redirect("MTurkTaskDonePage.aspx");
}
}
}
public static void AggregateWithParameterAndValidateSatyamVideoClassificationResultByGUID(string guid,
int MinResults = TaskConstants.SINGLE_OBJECT_LABLING_MTURK_MIN_RESULTS_TO_AGGREGATE,
int MaxResults = TaskConstants.SINGLE_OBJECT_LABLING_MTURK_MAX_RESULTS_TO_AGGREGATE,
double MajorityThreshold = TaskConstants.SINGLE_OBJECT_LABLING_MTURK_MAJORITY_THRESHOLD,
bool prepareDataForTraining = false, string outputDirectory = null,
string confusingVideoListFilePath = null)
{
string configString = "Min_" + MinResults + "_Max_" + MaxResults + "_Thresh_" + MajorityThreshold;
Console.WriteLine("Aggregating for param set " + configString);
int noTerminatedTasks = 0;
SatyamResultsTableAccess resultsDB = new SatyamResultsTableAccess();
List <SatyamResultsTableEntry> entries = resultsDB.getEntriesByGUIDOrderByID(guid);
resultsDB.close();
SortedDictionary <DateTime, List <SatyamResultsTableEntry> > entriesBySubmitTime = SatyamResultValidationToolKit.SortResultsBySubmitTime(entries);
int noTotalConverged = 0;
int noCorrect = 0;
Dictionary <int, List <SatyamResultsTableEntry> > ResultsPerTask = new Dictionary <int, List <SatyamResultsTableEntry> >();
List <int> aggregatedTasks = new List <int>();
SortedDictionary <string, Dictionary <string, int> > confusionMatrix_res_groundtruth = new SortedDictionary <string, Dictionary <string, int> >();
SortedDictionary <string, Dictionary <string, int> > confusionMatrix_groundtruth_res = new SortedDictionary <string, Dictionary <string, int> >();
List <SatyamAggregatedResultsTableEntry> aggEntries = new List <SatyamAggregatedResultsTableEntry>();
Dictionary <int, int> noResultsNeededForAggregation = SatyamResultsAnalysis.getNoResultsNeededForAggregationFromLog(configString, guid);
Dictionary <int, int> noResultsNeededForAggregation_new = new Dictionary <int, int>();
foreach (DateTime t in entriesBySubmitTime.Keys)
{
//Console.WriteLine("Processing Results of time: {0}", t);
foreach (SatyamResultsTableEntry entry in entriesBySubmitTime[t])
{
SatyamResult satyamResult = JSonUtils.ConvertJSonToObject <SatyamResult>(entry.ResultString);
SatyamTask task = JSonUtils.ConvertJSonToObject <SatyamTask>(satyamResult.TaskParametersString);
SatyamJob job = task.jobEntry;
string fileName = URIUtilities.filenameFromURI(task.SatyamURI);
string VideoCategoryGroundTruth = getVideoCategoryFromFileName(fileName);
int taskEntryID = entry.SatyamTaskTableEntryID;
if (aggregatedTasks.Contains(taskEntryID))
{
continue;
}
if (!ResultsPerTask.ContainsKey(taskEntryID))
{
ResultsPerTask.Add(taskEntryID, new List <SatyamResultsTableEntry>());
}
ResultsPerTask[taskEntryID].Add(entry);
// check log if enough results are collected
if (noResultsNeededForAggregation != null && noResultsNeededForAggregation.ContainsKey(taskEntryID) &&
ResultsPerTask[taskEntryID].Count < noResultsNeededForAggregation[taskEntryID])
{
continue;
}
//SingleObjectLabelingAggregatedResult aggResult = SingleObjectLabelingAggregator.getAggregatedResult(ResultsPerTask[taskEntryID], MinResults, MaxResult, MajorityThreshold);
string aggResultString = SingleObjectLabelingAggregator.GetAggregatedResultString(ResultsPerTask[taskEntryID], MinResults, MaxResults, MajorityThreshold);
if (aggResultString == null)
{
continue;
}
SatyamAggregatedResultsTableEntry aggEntry = new SatyamAggregatedResultsTableEntry();
aggEntry.JobGUID = ResultsPerTask[taskEntryID][0].JobGUID;
aggEntry.JobTemplateType = ResultsPerTask[taskEntryID][0].JobTemplateType;
aggEntry.SatyamTaskTableEntryID = taskEntryID;
aggEntry.UserID = ResultsPerTask[taskEntryID][0].UserID;
aggEntry.ResultString = aggResultString;
/// aggregation happen
// record logs
if (noResultsNeededForAggregation == null || !noResultsNeededForAggregation.ContainsKey(taskEntryID))
{
noResultsNeededForAggregation_new.Add(taskEntryID, ResultsPerTask[taskEntryID].Count);
}
aggEntries.Add(aggEntry);
noTotalConverged++;
if (ResultsPerTask[taskEntryID].Count >= MaxResults)
{
noTerminatedTasks++;
}
aggregatedTasks.Add(taskEntryID);
}
}
SatyamResultsAnalysis.RecordAggregationLog(noResultsNeededForAggregation_new, configString, guid);
string outputmatFile = DirectoryConstants.defaultTempDirectory + guid + "\\" + configString + "_mat.txt";
EvaluateAndPrintConfusionMatrixOfAggregatedResultEntries(aggEntries, outputmatFile, out noCorrect, prepareDataForTraining, outputDirectory);
Console.WriteLine("noTotalConverged {0}", noTotalConverged);
Console.WriteLine("noTerminatedTasks {0}", noTerminatedTasks);
Console.WriteLine("Result: {0}/{1}, precision: {2}", noCorrect, noTotalConverged, (double)noCorrect / noTotalConverged);
/// local file
string outputString = String.Format("{0} {1} {2} {3} {4} {5}\n", configString, noCorrect, noTotalConverged, (double)noCorrect / noTotalConverged, noTerminatedTasks, ResultsPerTask.Count - noTotalConverged);
string outputfile = DirectoryConstants.defaultTempDirectory + guid + "\\resultSummary.txt";
File.AppendAllText(outputfile, outputString);
//for (double prob = 0; prob < 1;prob +=0.2)
//{
// SatyamResultsAnalysis.AnalyzeApprovalRate(aggEntries, entries, guid, configString, anotherChanceProbablity: prob);
//}
//for (double ratio = 0; ratio < 1; ratio += 0.2)
//{
// SatyamResultsAnalysis.AnalyzeApprovalRate(aggEntries, entriesBySubmitTime, noResultsNeededForAggregation, noResultsNeededForAggregation_new, guid, configString, approvalRatioThreshold: ratio);
//}
SatyamResultsAnalysis.AggregationAnalysis(aggEntries, entriesBySubmitTime, noResultsNeededForAggregation, noResultsNeededForAggregation_new, guid, configString);
}
// If the results were not good enough, or aggregation method changed, retargeted, users might want to reopen the job
// All we need to do: (IN STRICT ORDER)
// clear all results back to inconclusive,
// remove all aggregated results
// restore the task table as it was for the guid,
// change the job status back to launched, and
// WARNING:
// for safety, this has to be run atomically synchronously, without any parallel process. for now.
public static void reopenJobForMoreResults(string guid)
{
SatyamJobSubmissionsTableAccess jobDB = new SatyamJobSubmissionsTableAccess();
SatyamJobSubmissionsTableAccessEntry jobEntry = jobDB.getEntryByJobGIUD(guid);
jobDB.close();
if (jobEntry.JobStatus != JobStatus.completed)
{
Console.WriteLine("Job not completed yet!");
return;
}
SatyamAggregatedResultsTableAccess aggResultDB = new SatyamAggregatedResultsTableAccess();
bool delSuccess = aggResultDB.DeleteEntriesByGUID(guid);
aggResultDB.close();
if (!delSuccess)
{
Console.WriteLine("Delete Agg Result DB Failed");
return;
}
SatyamResultsTableAccess resultDB = new SatyamResultsTableAccess();
if (!resultDB.UpdateStatusByGUID(guid, ResultStatus.inconclusive))
{
Console.WriteLine("Update Result DB Failed");
//resultDB.close();
//return;
}
List <SatyamResultsTableEntry> results = resultDB.getEntriesByGUID(guid);
resultDB.close();
Dictionary <int, SatyamTask> taskParamsByTaskID = new Dictionary <int, SatyamTask>();
foreach (SatyamResultsTableEntry result in results)
{
if (taskParamsByTaskID.ContainsKey(result.SatyamTaskTableEntryID))
{
continue;
}
SatyamResult satyamRes = JSonUtils.ConvertJSonToObject <SatyamResult>(result.ResultString);
SatyamTask task = JSonUtils.ConvertJSonToObject <SatyamTask>(satyamRes.TaskParametersString);
taskParamsByTaskID.Add(result.SatyamTaskTableEntryID, task);
}
SatyamTaskTableAccess taskDB = new SatyamTaskTableAccess();
foreach (int taskID in taskParamsByTaskID.Keys)
{
SatyamTask task = taskParamsByTaskID[taskID];
SatyamJob job = task.jobEntry;
bool suc = taskDB.AddEntryWithSpecificID(taskID, job.JobTemplateType, job.UserID, job.JobGUIDString, JSonUtils.ConvertObjectToJSon(task),
job.JobSubmitTime, job.amazonHITInformation.Price);
if (!suc)
{
Console.WriteLine("Update Task Table Failed");
taskDB.close();
return;
}
}
taskDB.close();
jobDB = new SatyamJobSubmissionsTableAccess();
bool success = jobDB.UpdateEntryStatus(guid, JobStatus.launched);
jobDB.close();
if (!success)
{
Console.WriteLine("Update Job Entry Failed");
return;
}
}
protected void SubmitButton_Click(object sender, EventArgs e)
{
DateTime SubmitTime = DateTime.Now;
DateTime PageLoadTime = Convert.ToDateTime(Hidden_PageLoadTime.Value);
int count;
bool isValidCount = int.TryParse(CountTextBox.Text, out count);
if (!isValidCount)
{
ErrorLabel.Text = "Error : Ivalid Count.";
ErrorLabel.ForeColor = System.Drawing.Color.Red;
ErrorLabel.Font.Bold = true;
}
else if (count < 0)
{
ErrorLabel.Text = "Error : Count cannot be less than zero.";
ErrorLabel.ForeColor = System.Drawing.Color.Red;
ErrorLabel.Font.Bold = true;
}
else
{
ErrorLabel.Text = "";
SatyamTaskTableEntry taskEntry = JSonUtils.ConvertJSonToObject <SatyamTaskTableEntry>(Hidden_TaskEntryString.Value);
SatyamResult result = new SatyamResult();
result.TaskParametersString = taskEntry.TaskParametersString;
result.TaskStartTime = PageLoadTime;
result.TaskEndTime = SubmitTime;
result.TaskTableEntryID = taskEntry.ID;
AmazonTaskResultInfo amazonInfo = new AmazonTaskResultInfo();
amazonInfo.AssignmentID = "";
amazonInfo.WorkerID = "";
amazonInfo.HITID = "";
result.amazonInfo = amazonInfo;
ObjectCountingResult sresult = new ObjectCountingResult();
sresult.Count = count;
string sresultString = JSonUtils.ConvertObjectToJSon <ObjectCountingResult>(sresult);
result.TaskResult = sresultString;
string resultString = JSonUtils.ConvertObjectToJSon <SatyamResult>(result);
SatyamResultsTableAccess resultdb = new SatyamResultsTableAccess();
resultdb.AddEntry(taskEntry.JobTemplateType, taskEntry.UserID, taskEntry.JobGUID, resultString, taskEntry.ID, PageLoadTime, SubmitTime);
resultdb.close();
//SatyamTaskTableAccess taskDB = new SatyamTaskTableAccess();
//taskDB.IncrementDoneScore(taskEntry.ID);
bool NotDone = getNewRandomJob();
if (NotDone == false)
{
Response.Redirect("AllJobsDone.aspx");
}
}
}
public static void Run([TimerTrigger("0 */5 * * * *")] TimerInfo myTimer, TraceWriter log)
{
DateTime start = DateTime.Now;
bool logging = false;
if (logging)
{
log.Info($"Judge Result Dispatch executed at: {DateTime.Now}");
}
//get all inconclusive results and group them by taskIDs
SatyamResultsTableAccess resultsDB = new SatyamResultsTableAccess();
List <SatyamResultsTableEntry> results = resultsDB.getAllEntriesByStatus(ResultStatus.inconclusive);
resultsDB.close();
if (results.Count == 0)
{
return;
}
Dictionary <int, List <SatyamResultsTableEntry> > resultsByTaskID = new Dictionary <int, List <SatyamResultsTableEntry> >();
List <string> pendingGUID = new List <string>();
foreach (SatyamResultsTableEntry result in results)
{
if (!resultsByTaskID.ContainsKey(result.SatyamTaskTableEntryID))
{
resultsByTaskID.Add(result.SatyamTaskTableEntryID, new List <SatyamResultsTableEntry>());
}
resultsByTaskID[result.SatyamTaskTableEntryID].Add(result);
if (!pendingGUID.Contains(result.JobGUID))
{
pendingGUID.Add(result.JobGUID);
}
}
//now check against the aggregated result to see if the result is aceptable
SatyamDispatchStorageAccountAccess satyamQueue = new SatyamDispatchStorageAccountAccess();
Dictionary <string, Dictionary <int, SatyamAggregatedResultsTableEntry> > aggEntriesPerTaskPerGUID = SatyamAggregatedResultManagement.getAggregatedEntriesPerTaskByGuidList(pendingGUID);
foreach (KeyValuePair <int, List <SatyamResultsTableEntry> > entry in resultsByTaskID)
{
if (entry.Value.Count == 0)
{
continue;
}
int taskEntryID = entry.Key;
string taskGUID = entry.Value[0].JobGUID;
if (aggEntriesPerTaskPerGUID.ContainsKey(taskGUID) && aggEntriesPerTaskPerGUID[taskGUID].ContainsKey(taskEntryID))
{
// this task has been aggregated
foreach (SatyamResultsTableEntry result in resultsByTaskID[taskEntryID]) //now got through each task to see if they satify pass criterion
{
if (logging)
{
log.Info($"Dispatching Judgement for {taskEntryID}");
}
string queueName = "judge-result";
string m = taskGUID + "_" + taskEntryID + "_" + result.ID;
satyamQueue.Enqueue(queueName, m);
if ((DateTime.Now - start).TotalSeconds > 280)
{
break;
}
}
}
if ((DateTime.Now - start).TotalSeconds > 280)
{
break;
}
}
if (logging)
{
log.Info($"Judge Result Dispatch finished at: {DateTime.Now}");
}
}
//protected void SubmitButton_Click(object sender, EventArgs e)
private void storeResult()
{
DateTime SubmitTime = DateTime.Now;
DateTime PageLoadTime = Convert.ToDateTime(Hidden_PageLoadTime.Value);
SatyamTaskTableEntry taskEntry = JSonUtils.ConvertJSonToObject <SatyamTaskTableEntry>(Hidden_TaskEntryString.Value);
string tracksString = TracksOutput_Hidden.Value;
string urlList = Hidden_ImageURLList.Value;
string[] fields = urlList.Split(',');
DateTime start = DateTime.MinValue;
List <DateTime> frameTimes = new List <DateTime>();
//double frameTimeSpanInMiliseconds = (Convert.ToDouble(Hidden_ChunkDuration.Value) / (double)fields.Length) * 1000;
double frameTimeSpanInMiliseconds = (double)(1000) / Convert.ToDouble(fps_Hidden.Value);
for (int i = 0; i < fields.Length; i++)
{
DateTime t;
t = start.AddMilliseconds(frameTimeSpanInMiliseconds * i);
frameTimes.Add(t);
}
string s = Raw_VATIC_DVA_Crowdsourced_Track_Collection.Raw_VATIC_DVA_Crowdsourced_Track_Collection_ToTrackStrings(tracksString, frameTimes);
SatyamResult result = new SatyamResult();
result.TaskParametersString = taskEntry.TaskParametersString;
result.TaskStartTime = PageLoadTime;
result.TaskEndTime = SubmitTime;
result.TaskTableEntryID = taskEntry.ID;
AmazonTaskResultInfo amazonInfo = new AmazonTaskResultInfo();
amazonInfo.AssignmentID = Hidden_AmazonAssignmentID.Value;
amazonInfo.WorkerID = Hidden_AmazonWorkerID.Value;
amazonInfo.HITID = Hidden_HITID.Value;
amazonInfo.PricePerHIT = Convert.ToDouble(Hidden_Price.Value);
result.amazonInfo = amazonInfo;
result.TaskResult = s;
string resultString = JSonUtils.ConvertObjectToJSon <SatyamResult>(result);
SatyamResultsTableAccess resultdb = new SatyamResultsTableAccess();
resultdb.AddEntry(taskEntry.JobTemplateType, taskEntry.UserID, taskEntry.JobGUID, resultString, taskEntry.ID, PageLoadTime, SubmitTime);
resultdb.close();
SatyamTaskTableManagement.UpdateResultNumber(taskEntry.ID);
if (!Testing)
{
SatyamAmazonHITTableAccess HITdb = new SatyamAmazonHITTableAccess();
string HITID = result.amazonInfo.HITID;
HITdb.UpdateStatusByHITID(HITID, HitStatus.submitted);
HITdb.close();
AmazonMTurkNotification.submitAmazonTurkHit(result.amazonInfo.AssignmentID, result.amazonInfo.WorkerID, false);
}
Response.Redirect("AllJobsDone.aspx");
}
public static void Run([QueueTrigger("judge-result")] string myQueueItem, TraceWriter log)
{
bool logging = false;
string[] fields = myQueueItem.Split('_');
string guid = fields[0];
int taskID = Convert.ToInt32(fields[1]);
string resultID = fields[2];
if (logging)
{
log.Info($"Judge Result: {myQueueItem}");
}
SatyamResultsTableAccess resultsDB = new SatyamResultsTableAccess();
List <SatyamResultsTableEntry> results = resultsDB.getEntriesID(resultID);
resultsDB.close();
if (results.Count == 0)
{
return;
}
SatyamAggregatedResultsTableAccess aggDB = new SatyamAggregatedResultsTableAccess();
SatyamAggregatedResultsTableEntry aggResult = aggDB.getEntryByTaskID(taskID);
aggDB.close();
if (aggResult == null)
{
return;
}
SatyamResultsTableEntry result = results[0];
SatyamResult res = JSonUtils.ConvertJSonToObject <SatyamResult>(result.ResultString);
string taskTemplateType = result.JobTemplateType;
string workerID = res.amazonInfo.WorkerID;
DateTime doneTime = res.TaskEndTime;
if (AcceptanceCriterionChecker.IsAcceptable(aggResult, result))
{
resultsDB = new SatyamResultsTableAccess();
resultsDB.UpdateStatusByID(result.ID, ResultStatus.accepted);
resultsDB.close();
WorkerStatisticsManagement.UpdateWorkerStatistics(workerID, taskTemplateType, true, doneTime);
if (logging)
{
log.Info($"Accepted");
}
}
else
{
resultsDB = new SatyamResultsTableAccess();
resultsDB.UpdateStatusByID(result.ID, ResultStatus.rejected);
resultsDB.close();
WorkerStatisticsManagement.UpdateWorkerStatistics(workerID, taskTemplateType, false, doneTime);
if (logging)
{
log.Info($"Rejected");
}
}
}
public static void AggregateWithParameter(string guid,
int MinResults = TaskConstants.TRACKLET_LABELING_MTURK_MIN_RESULTS_TO_AGGREGATE,
int MaxResults = TaskConstants.TRACKLET_LABELING_MTURK_MAX_RESULTS_TO_AGGREGATE,
double boxToleranceThreshold = TaskConstants.TRACKLET_LABELING_BOX_DEVIATION_THRESHOLD,
double ObjectCoverageApprovalThresholdPerVideo = TaskConstants.TRACKLET_LABELING_APPROVALRATIO_PER_VIDEO,
double BoxCoverageApprovalThresholdPerTrack = TaskConstants.TRACKLET_LABELING_APPROVALRATIO_PER_TRACK,
int consensusNumber = TaskConstants.TRACKLET_LABELING_MIN_RESULTS_FOR_CONSENSUS,
double minTubeletIoUSimilarityThreshold = TaskConstants.TRACKLET_LABELING_MIN_TUBELET_SIMILARITY_THRESHOLD,
double attributeMajority = TaskConstants.TRACKLET_LABELING_MTURK_ATTRIBUTE_MAJORITY_THRESHOLD,
bool allFalseAttributeInvalid = false
)
{
string configString = "Min_" + MinResults + "_Max_" + MaxResults + "_IoU_" + minTubeletIoUSimilarityThreshold + "_Ratio_" + ObjectCoverageApprovalThresholdPerVideo;
Console.WriteLine("Aggregating for " + guid + " with param set " + configString);
SatyamResultsTableAccess resultsDB = new SatyamResultsTableAccess();
List <SatyamResultsTableEntry> entries = resultsDB.getEntriesByGUID(guid);
resultsDB.close();
SortedDictionary <DateTime, List <SatyamResultsTableEntry> > entriesBySubmitTime = SatyamResultValidationToolKit.SortResultsBySubmitTime_OneResultPerTurkerPerTask(entries);
Dictionary <int, List <MultiObjectTrackingResult> > ResultsPerTask = new Dictionary <int, List <MultiObjectTrackingResult> >();
List <int> aggregatedTasks = new List <int>();
int noTotalConverged = 0;
//int noCorrect = 0;
int noTerminatedTasks = 0;
List <SatyamAggregatedResultsTableEntry> aggEntries = new List <SatyamAggregatedResultsTableEntry>();
Dictionary <int, int> noResultsNeededForAggregation = SatyamResultsAnalysis.getNoResultsNeededForAggregationFromLog(configString, guid);
Dictionary <int, int> noResultsNeededForAggregation_new = new Dictionary <int, int>();
// play back by time
Dictionary <int, List <string> > WorkersPerTask = new Dictionary <int, List <string> >();
foreach (DateTime t in entriesBySubmitTime.Keys)
{
//Console.WriteLine("Processing Results of time: {0}", t);
List <SatyamResultsTableEntry> ResultEntries = entriesBySubmitTime[t];
foreach (SatyamResultsTableEntry entry in ResultEntries)
{
SatyamResult satyamResult = JSonUtils.ConvertJSonToObject <SatyamResult>(entry.ResultString);
SatyamTask task = JSonUtils.ConvertJSonToObject <SatyamTask>(satyamResult.TaskParametersString);
MultiObjectTrackingSubmittedJob job = JSonUtils.ConvertJSonToObject <MultiObjectTrackingSubmittedJob>(task.jobEntry.JobParameters);
string fileName = URIUtilities.filenameFromURINoExtension(task.SatyamURI);
int taskEntryID = entry.SatyamTaskTableEntryID;
if (aggregatedTasks.Contains(taskEntryID))
{
continue;
}
if (!ResultsPerTask.ContainsKey(taskEntryID))
{
ResultsPerTask.Add(taskEntryID, new List <MultiObjectTrackingResult>());
WorkersPerTask.Add(taskEntryID, new List <string>());
}
// remove duplicate workers result
string workerID = satyamResult.amazonInfo.WorkerID;
if (WorkersPerTask[taskEntryID].Contains(workerID))
{
continue;
}
//enclose only non-duplicate results, one per each worker.
WorkersPerTask[taskEntryID].Add(workerID);
string videoName = URIUtilities.localDirectoryNameFromURI(task.SatyamURI);
string[] fields = videoName.Split('_');
int startingFrame = Convert.ToInt32(fields[fields.Length - 1]);
int maxChunkEndFrame = startingFrame + job.ChunkDuration * job.FrameRate;
int noFrameOverlap = (int)(job.ChunkOverlap * job.FrameRate);
if (startingFrame != 0)
{
startingFrame -= noFrameOverlap;
}
string blobDir = URIUtilities.localDirectoryFullPathFromURI(task.SatyamURI);
VATIC_DVA_CrowdsourcedResult taskr = TrackletLabelingAggregator.createVATIC_DVA_CrowdsourcedResultUsingSatyamBlobImageCount(satyamResult.TaskResult, blobDir, entry.SatyamTaskTableEntryID.ToString(), job.FrameRate);
//MultiObjectTrackingResult result_tmp = vatic_tmp.getCompressedTracksInTimeSegment();
//VATIC_DVA_CrowdsourcedResult taskr = new VATIC_DVA_CrowdsourcedResult(satyamResult.TaskResult, videoName, entry.ID.ToString(), ImageURLs.Count, job.FrameRate);
MultiObjectTrackingResult res = taskr.getCompressedTracksInTimeSegment();
if (allFalseAttributeInvalid)
{
if (TrackletLabelingAggregator.AllAttributeAllFalse(res))
{
continue;
}
}
ResultsPerTask[taskEntryID].Add(res);
// check log if enough results are collected
if (noResultsNeededForAggregation != null && noResultsNeededForAggregation.ContainsKey(taskEntryID) &&
ResultsPerTask[taskEntryID].Count < noResultsNeededForAggregation[taskEntryID])
{
continue;
}
// hack for masters
int tempMin = MinResults;
if (TaskConstants.masterGUIDs.Contains(entry.JobGUID))
{
tempMin = 1;
}
TrackletLabelingAggregatedResult aggResult = TrackletLabelingAggregator.getAggregatedResult(ResultsPerTask[taskEntryID],
guid, task.SatyamURI,
job.FrameRate, job.BoundaryLines,
MinResults, MaxResults, boxToleranceThreshold,
ObjectCoverageApprovalThresholdPerVideo,
BoxCoverageApprovalThresholdPerTrack,
consensusNumber, minTubeletIoUSimilarityThreshold,
attributeMajority);
if (aggResult == null)
{
continue;
}
//////////////// aggregation happen
// record logs
if (noResultsNeededForAggregation == null || !noResultsNeededForAggregation.ContainsKey(taskEntryID))
{
noResultsNeededForAggregation_new.Add(taskEntryID, ResultsPerTask[taskEntryID].Count);
}
aggregatedTasks.Add(taskEntryID);
noTotalConverged++;
if (ResultsPerTask[taskEntryID].Count >= MaxResults)
{
noTerminatedTasks++;
}
SatyamAggregatedResult SatyamAggResult = new SatyamAggregatedResult();
SatyamAggResult.SatyamTaskTableEntryID = taskEntryID;
SatyamAggResult.AggregatedResultString = JSonUtils.ConvertObjectToJSon <TrackletLabelingAggregatedResult>(aggResult);
SatyamAggResult.TaskParameters = JSonUtils.ConvertJSonToObject <SatyamResult>(entry.ResultString).TaskParametersString;
SatyamAggregatedResultsTableEntry aggEntry = new SatyamAggregatedResultsTableEntry();
aggEntry.SatyamTaskTableEntryID = taskEntryID;
aggEntry.JobGUID = entry.JobGUID;
aggEntry.ResultString = JSonUtils.ConvertObjectToJSon <SatyamAggregatedResult>(SatyamAggResult);
aggEntries.Add(aggEntry);
}
}
Console.WriteLine("Total_Aggregated_Tasks: {0}", noTotalConverged);
Console.WriteLine("Total_Terminated_Tasks: {0}", noTerminatedTasks);
SatyamResultsAnalysis.RecordAggregationLog(noResultsNeededForAggregation_new, configString, guid);
TrackletLabelingAnalyzer.GroupEntriesByVideoNameAndStitchAndSaveAggregatedResultVideosLocally(aggEntries, DirectoryConstants.defaultTempDirectory + guid);
}
public static bool SafeDeleteHit(SatyamAmazonHITTableAccessEntry hitEntryToBeRemoved)
{
// first expire
AmazonMTurkHIT hit = new AmazonMTurkHIT();
hit.setAccount(hitEntryToBeRemoved.AmazonAccessKeyID, hitEntryToBeRemoved.AmazonSecretAccessKeyID, false);
hit.expireHIT(hitEntryToBeRemoved.HITID);
SatyamAmazonHITTableAccess hitDB = new SatyamAmazonHITTableAccess();
hitDB.UpdateStatus(hitEntryToBeRemoved.ID, HitStatus.expired);
hitDB.close();
// then try dispose
if (!hit.DeleteHIT(hitEntryToBeRemoved.HITID))
{
// can't dispose due to invalid status: reviewable but payment not cleared, or disposed.
// because payment not cleared
// find all results of this hitID and mark for rejection
if (!resultCache.ContainsKey(hitEntryToBeRemoved.JobGUID))
{
List <SatyamResultsTableEntry> ret = new List <SatyamResultsTableEntry>();
SatyamResultsTableAccess resDB = new SatyamResultsTableAccess();
List <SatyamResultsTableEntry> tempres = resDB.getEntriesByGUID(hitEntryToBeRemoved.JobGUID);
resDB.close();
resultCache.Add(hitEntryToBeRemoved.JobGUID, tempres);
}
List <SatyamResultsTableEntry> results = resultCache[hitEntryToBeRemoved.JobGUID];
List <SatyamResultsTableEntry> res = new List <SatyamResultsTableEntry>();
foreach (SatyamResultsTableEntry r in results)
{
SatyamResult rr = JSonUtils.ConvertJSonToObject <SatyamResult>(r.ResultString);
if (rr.amazonInfo.HITID == hitEntryToBeRemoved.HITID)
{
res.Add(r);
}
}
if (res.Count == 0)
{
// the hit can't be disposed because there are pending assignments, yet we don't have any on records,
// so delete it. AWS will automatically handle it after 120 days of inactivity.
hitDB = new SatyamAmazonHITTableAccess();
hitDB.DeleteEntry(hitEntryToBeRemoved.ID);
hitDB.close();
Console.WriteLine("Delete Success\n");
return(true);
}
//bool HITDone = true;
foreach (SatyamResultsTableEntry r in res)
{
//SatyamResult rr = JSonUtils.ConvertJSonToObject<SatyamResult>(r.ResultString);
//string assignmentID = rr.amazonInfo.AssignmentID;
//hit.RejectAssignment(assignmentID, "Sorry! Your work was not within acceptable parameters!");
if (r.Status == ResultStatus.inconclusive || r.Status == ResultStatus.outdated)
{
// reject all for now... TODO: check if there is an aggregated results.
//Console.WriteLine("Rejecting result ID {0}", r.ID);
//SatyamResultsTableAccess resultsDB = new SatyamResultsTableAccess();
//resultsDB.UpdateStatusByID(r.ID, ResultStatus.rejected);
//resultsDB.close();
//HITDone = false;
}
else
{
Console.WriteLine("Result ID {0} is of status {1}", r.ID, r.Status);
if (r.Status == ResultStatus.accepted || r.Status == ResultStatus.rejected)
{
// need to be handled by payment function
//HITDone = false;
}
// just in case last payment was not made successfully,
// mark it to be paid again....
SatyamResultsTableAccess resultsDB = new SatyamResultsTableAccess();
if (ResultStatus.acceptedStatusList.Contains(r.Status))
{
Console.WriteLine("Marking as Accepted so Next Cycle will Pay");
resultsDB.UpdateStatusByID(r.ID, ResultStatus.accepted);
}
if (ResultStatus.rejectedStatusList.Contains(r.Status))
{
Console.WriteLine("Marking as Rejected so Next Cycle will Reject");
resultsDB.UpdateStatusByID(r.ID, ResultStatus.rejected);
}
resultsDB.close();
}
}
// Commented out cuz the case where the inconsistency status happen is rare,
// so don't delete the hit too aggressively
//if (HITDone)
//{
// hitDB = new SatyamAmazonHITTableAccess();
// hitDB.DeleteEntry(hitEntryToBeRemoved.ID);
// hitDB.close();
// Console.WriteLine("Delete Success\n");
// return true;
//}
Console.WriteLine("Delete Failure\n");
return(false);
}
else
{
hitDB = new SatyamAmazonHITTableAccess();
hitDB.DeleteEntry(hitEntryToBeRemoved.ID);
hitDB.close();
Console.WriteLine("Delete Success\n");
return(true);
}
}
protected void SubmitButton_Click(object sender, EventArgs e)
{
DateTime SubmitTime = DateTime.Now;
DateTime PageLoadTime = Convert.ToDateTime(Hidden_PageLoadTime.Value);
if (CategorySelection_RadioButtonList.SelectedIndex != -1)
{
SatyamTaskTableEntry taskEntry = JSonUtils.ConvertJSonToObject <SatyamTaskTableEntry>(Hidden_TaskEntryString.Value);
SatyamResult result = new SatyamResult();
result.TaskParametersString = taskEntry.TaskParametersString;
result.TaskStartTime = PageLoadTime;
result.TaskEndTime = SubmitTime;
result.TaskTableEntryID = taskEntry.ID;
AmazonTaskResultInfo amazonInfo = new AmazonTaskResultInfo();
amazonInfo.AssignmentID = Hidden_AmazonAssignmentID.Value;
amazonInfo.WorkerID = Hidden_AmazonWorkerID.Value;
amazonInfo.HITID = Hidden_HITID.Value;
amazonInfo.PricePerHIT = Convert.ToDouble(Hidden_Price.Value);
result.amazonInfo = amazonInfo;
SingleObjectLabelingResult sresult = new SingleObjectLabelingResult();
sresult.Category = CategorySelection_RadioButtonList.Items[CategorySelection_RadioButtonList.SelectedIndex].Text;
string sresultString = JSonUtils.ConvertObjectToJSon <SingleObjectLabelingResult>(sresult);
result.TaskResult = sresultString;
string resultString = JSonUtils.ConvertObjectToJSon <SatyamResult>(result);
SatyamResultsTableAccess resultdb = new SatyamResultsTableAccess();
resultdb.AddEntry(taskEntry.JobTemplateType, taskEntry.UserID, taskEntry.JobGUID, resultString, taskEntry.ID, PageLoadTime, SubmitTime);
resultdb.close();
SatyamTaskTableManagement.UpdateResultNumber(taskEntry.ID);
//SatyamTaskTableAccess taskDB = new SatyamTaskTableAccess();
//taskDB.IncrementDoneScore(taskEntry.ID);
int noDone = Convert.ToInt32(Hidden_NoImagesDone.Value);
noDone++;
Hidden_NoImagesDone.Value = noDone.ToString();
int noTasksPerJob = Convert.ToInt32(Hidden_TasksPerJob.Value);
bool NotDone = false;
if (noDone < noTasksPerJob)
{
NotDone = getNewRandomJob();
}
if (NotDone == false)
{
if (!Testing)
{
SatyamAmazonHITTableAccess HITdb = new SatyamAmazonHITTableAccess();
string HITID = result.amazonInfo.HITID;
HITdb.UpdateStatusByHITID(HITID, HitStatus.submitted);
HITdb.close();
AmazonMTurkNotification.submitAmazonTurkHit(result.amazonInfo.AssignmentID, result.amazonInfo.WorkerID, false);
}
Response.Redirect("MTurkTaskDonePage.aspx");
}
}
}
