public static void TestSatyamResultAnalysis() { List <List <string> > jobs = new List <List <string> >() { //// Video Classification //new List<string>(){ "ca2a97e3-3c32-48cc-b7d9-fdd945b50f23" }, //10 class human gesture in JHMDB //// Image Classification //new List<string>(){ "9c6283dd-f218-4588-a0d9-8c0afe841503" }, // all img 10class //new List<string>(){ "03977c5f-0e87-49ab-9c49-f20bfb47726a" }, // 2img 10 class //// Image Counting //new List<string>(){ // "94979929-4315-4c81-9782-55c8955222a8",// CARPK NTU 20161029 // "58a85bc8-4fb9-4672-8719-38cf62352fbc", // KITTI all //}, //// Image Detection new List <string>() { //"3609af9c-e734-41dc-997c-94c060a1f63d", //"5f14ecaa-1c4c-41d6-ad05-6ee0929e494e", //"8fbc6ca1-e6c0-4720-93a3-1d49aa873a65", //"d6f1c3ee-cd59-4197-b40a-0f45512596eb", // KITTI Detection all //"a7e3fe3a-5097-4149-99bc-bae06da18aba", // rivision tasks satyam //"aefd12fc-42ed-4ec5-8c9a-6c41703a843f", // revision task kitti //"2c239102-e38f-4f0e-b639-6fae05250475", // without revision task kitti //"2b1a86ee-7d39-49db-90bc-fb8d2ff63019", // revision kitti 10 internal test with recordings //"a328f600-3b1f-4845-87a0-402648ad51b1", // revision Seattle 10, TF Serving backend //"5c403d7f-6c48-4638-b545-472b9f190606", // no revision Seattle 10 //"40eee69b-c260-4760-940f-92f30cdfbb7f", // RevisionTF Seattle9000 on TestStewart 10 //"a55c9c21-16c0-432a-930e-de428d8213a5", // RevisionTF Fastrcnn_KITTI on TestStewart 10 //"9e3096f6-e898-45eb-8b15-e2854485a59c", // RevisionTF Fastrcnn_KITTI on TestStewart 30, but aggregation is not functioning for most of the time "3e835157-d058-46ee-a13b-9c1566cba505", //RevisionTF Seattle9000 on TestStewart 30 }, ////// Video Tracking //new List<string>(){ // "1e43a983-548d-4a2e-8161-5537eb985902", // "77c4f6af-f669-42fb-925a-4ff670cf9ae2"}, // KITTI Tracking all ////// Caprk Counting Adaptive Pricing Test //new List<string>(){ // //"f7c498d8-c06e-4b46-9169-71048c0e4eac",// carpk 10 // "e5fdff10-f018-4779-9d18-70c25fa3259f",}, // carpk all //new List<string>(){ // "a388df30-8742-4b6e-911d-32093502c2e8"}, // KITTI all //////// Image Segmentation //new List<string>() //{ // "86cbfe43-175a-4a5a-ba89-073f3004e7df", // pascal VOC //}, ////// Cam Pose Estimation //new List<string>() //{ // "dff5415b-b278-43db-8e32-4f654ca4b3a1", //3000 results //} }; foreach (List <string> guids in jobs) { SatyamResultsAnalysis.AnalyzeFromResultsTable(guids); } }
public static void AdaptTaskParameters() { double TargetPricePerHour = -1; double Budget = -1; //double TargetPricePerHour = 13.56; //double MinPricePerTask = 0.01; //double MaxPricePerTask = 0.5; double MinPricePerTaskDifferenceToAdjust = 0.01;//USD int MinNoResultsToAdjust = 10; Dictionary <string, List <SatyamTaskTableEntry> > MTurkTasksByGUID = OrganizeMTurkTasksByGUID(); if (MTurkTasksByGUID == null) { return; } // get statistics foreach (string guid in MTurkTasksByGUID.Keys) { // testing for counting only List <string> ACGuids = new List <string>() { "e5fdff10-f018-4779-9d18-70c25fa3259f", //carpk 164 "a388df30-8742-4b6e-911d-32093502c2e8", //kitti 100 }; if (!ACGuids.Contains(guid)) { continue; } SatyamTaskTableEntry task0 = MTurkTasksByGUID[guid][0]; double CurrentPricePerTask = task0.PricePerTask; SatyamTask t = JSonUtils.ConvertJSonToObject <SatyamTask>(task0.TaskParametersString); TargetPricePerHour = t.jobEntry.TargetPricePerTask; //TargetPricePerHour = 13.56; //temporary Budget = t.jobEntry.TotalBudget; if (TargetPricePerHour <= 0) { continue; //invalid } SatyamResultPerJobDataAnalysis ana = SatyamResultsAnalysis.AnalyzeFromResultsTable(new List <string>() { guid }); if (ana == null) { continue; } int noAcceptedResults = ana.totalAcceptedResults; if (noAcceptedResults < MinNoResultsToAdjust) { continue; } double medianTimePerTaskInSeconds = ana.medianAcceptedTimeTakenPerResultInSeconds; double TargetPricePerTask = TargetPricePerHour / 3600 * medianTimePerTaskInSeconds; //TargetPricePerTask = Math.Max(TargetPricePerTask, MinPricePerTask); //TargetPricePerTask = Math.Min(TargetPricePerTask, MaxPricePerTask); TargetPricePerTask = Math.Round(TargetPricePerTask, 2);// round to cents. // is the price within the budget? double MinBudgetNeeded = MTurkTasksByGUID[guid].Count * TargetPricePerTask * 3;// at least 3 results per task. if (Budget < MinBudgetNeeded) { Console.WriteLine("Budget too small to adapt price"); continue; } if (Math.Abs(CurrentPricePerTask - TargetPricePerTask) > MinPricePerTaskDifferenceToAdjust) { // need to adjust Console.WriteLine("Adapting the price to {0}", TargetPricePerTask); ExpireHitByGUID(guid); AdjustTasksByGUID(guid, TargetPricePerTask); } } }