public static void TestSatyamResultAnalysis()
{
List <List <string> > jobs = new List <List <string> >()
{
//// Video Classification
//new List<string>(){ "ca2a97e3-3c32-48cc-b7d9-fdd945b50f23" }, //10 class human gesture in JHMDB
//// Image Classification
//new List<string>(){ "9c6283dd-f218-4588-a0d9-8c0afe841503" }, // all img 10class
//new List<string>(){ "03977c5f-0e87-49ab-9c49-f20bfb47726a" }, // 2img 10 class
//// Image Counting
//new List<string>(){
// "94979929-4315-4c81-9782-55c8955222a8",// CARPK NTU 20161029
// "58a85bc8-4fb9-4672-8719-38cf62352fbc", // KITTI all
//},
//// Image Detection
new List <string>()
{
//"3609af9c-e734-41dc-997c-94c060a1f63d",
//"5f14ecaa-1c4c-41d6-ad05-6ee0929e494e",
//"8fbc6ca1-e6c0-4720-93a3-1d49aa873a65",
//"d6f1c3ee-cd59-4197-b40a-0f45512596eb", // KITTI Detection all
//"a7e3fe3a-5097-4149-99bc-bae06da18aba", // rivision tasks satyam
//"aefd12fc-42ed-4ec5-8c9a-6c41703a843f", // revision task kitti
//"2c239102-e38f-4f0e-b639-6fae05250475", // without revision task kitti
//"2b1a86ee-7d39-49db-90bc-fb8d2ff63019", // revision kitti 10 internal test with recordings
//"a328f600-3b1f-4845-87a0-402648ad51b1", // revision Seattle 10, TF Serving backend
//"5c403d7f-6c48-4638-b545-472b9f190606", // no revision Seattle 10
//"40eee69b-c260-4760-940f-92f30cdfbb7f", // RevisionTF Seattle9000 on TestStewart 10
//"a55c9c21-16c0-432a-930e-de428d8213a5", // RevisionTF Fastrcnn_KITTI on TestStewart 10
//"9e3096f6-e898-45eb-8b15-e2854485a59c", // RevisionTF Fastrcnn_KITTI on TestStewart 30, but aggregation is not functioning for most of the time
"3e835157-d058-46ee-a13b-9c1566cba505", //RevisionTF Seattle9000 on TestStewart 30
},
////// Video Tracking
//new List<string>(){
// "1e43a983-548d-4a2e-8161-5537eb985902",
// "77c4f6af-f669-42fb-925a-4ff670cf9ae2"}, // KITTI Tracking all
////// Caprk Counting Adaptive Pricing Test
//new List<string>(){
// //"f7c498d8-c06e-4b46-9169-71048c0e4eac",// carpk 10
// "e5fdff10-f018-4779-9d18-70c25fa3259f",}, // carpk all
//new List<string>(){
// "a388df30-8742-4b6e-911d-32093502c2e8"}, // KITTI all
//////// Image Segmentation
//new List<string>()
//{
// "86cbfe43-175a-4a5a-ba89-073f3004e7df", // pascal VOC
//},
////// Cam Pose Estimation
//new List<string>()
//{
// "dff5415b-b278-43db-8e32-4f654ca4b3a1", //3000 results
//}
};
foreach (List <string> guids in jobs)
{
SatyamResultsAnalysis.AnalyzeFromResultsTable(guids);
}
}
public static void AdaptTaskParameters()
{
double TargetPricePerHour = -1;
double Budget = -1;
//double TargetPricePerHour = 13.56;
//double MinPricePerTask = 0.01;
//double MaxPricePerTask = 0.5;
double MinPricePerTaskDifferenceToAdjust = 0.01;//USD
int MinNoResultsToAdjust = 10;
Dictionary <string, List <SatyamTaskTableEntry> > MTurkTasksByGUID = OrganizeMTurkTasksByGUID();
if (MTurkTasksByGUID == null)
{
return;
}
// get statistics
foreach (string guid in MTurkTasksByGUID.Keys)
{
// testing for counting only
List <string> ACGuids = new List <string>()
{
"e5fdff10-f018-4779-9d18-70c25fa3259f", //carpk 164
"a388df30-8742-4b6e-911d-32093502c2e8", //kitti 100
};
if (!ACGuids.Contains(guid))
{
continue;
}
SatyamTaskTableEntry task0 = MTurkTasksByGUID[guid][0];
double CurrentPricePerTask = task0.PricePerTask;
SatyamTask t = JSonUtils.ConvertJSonToObject <SatyamTask>(task0.TaskParametersString);
TargetPricePerHour = t.jobEntry.TargetPricePerTask;
//TargetPricePerHour = 13.56; //temporary
Budget = t.jobEntry.TotalBudget;
if (TargetPricePerHour <= 0)
{
continue; //invalid
}
SatyamResultPerJobDataAnalysis ana = SatyamResultsAnalysis.AnalyzeFromResultsTable(new List <string>()
{
guid
});
if (ana == null)
{
continue;
}
int noAcceptedResults = ana.totalAcceptedResults;
if (noAcceptedResults < MinNoResultsToAdjust)
{
continue;
}
double medianTimePerTaskInSeconds = ana.medianAcceptedTimeTakenPerResultInSeconds;
double TargetPricePerTask = TargetPricePerHour / 3600 * medianTimePerTaskInSeconds;
//TargetPricePerTask = Math.Max(TargetPricePerTask, MinPricePerTask);
//TargetPricePerTask = Math.Min(TargetPricePerTask, MaxPricePerTask);
TargetPricePerTask = Math.Round(TargetPricePerTask, 2);// round to cents.
// is the price within the budget?
double MinBudgetNeeded = MTurkTasksByGUID[guid].Count * TargetPricePerTask * 3;// at least 3 results per task.
if (Budget < MinBudgetNeeded)
{
Console.WriteLine("Budget too small to adapt price");
continue;
}
if (Math.Abs(CurrentPricePerTask - TargetPricePerTask) > MinPricePerTaskDifferenceToAdjust)
{
// need to adjust
Console.WriteLine("Adapting the price to {0}", TargetPricePerTask);
ExpireHitByGUID(guid);
AdjustTasksByGUID(guid, TargetPricePerTask);
}
}
}
