//public static void AnalyzeApprovalRateByGUID(List<string> guids)
        //{
        //    Console.WriteLine("Analyzing Results Statistics of {0}", guids[0]);
        //    SatyamResultsTableAccess resultsDB = new SatyamResultsTableAccess();
        //    List<SatyamResultsTableEntry> entries = new List<SatyamResultsTableEntry>();
        //    foreach (string guid in guids)
        //    {
        //        entries.AddRange(resultsDB.getEntriesByGUIDOrderByID(guid));
        //    }

        //    resultsDB.close();
        //}



        public static void AggregationAnalysis(List <SatyamAggregatedResultsTableEntry> aggEntries,
                                               //List<SatyamResultsTableEntry> ResultEntries,
                                               SortedDictionary <DateTime, List <SatyamResultsTableEntry> > entriesBySubmitTime,
                                               Dictionary <int, int> noResultsNeededForAggregation,
                                               Dictionary <int, int> noResultsNeededForAggregation_new,
                                               string guid, string configString,
                                               int minChancesGiven            = 3,
                                               double anotherChanceProbablity = 0,
                                               double approvalRatioThreshold  = 0.5,
                                               bool usePreApprovalResult      = false)
        {
            if (noResultsNeededForAggregation == null)
            {
                noResultsNeededForAggregation = new Dictionary <int, int>();
            }

            if (noResultsNeededForAggregation_new == null)
            {
                noResultsNeededForAggregation_new = new Dictionary <int, int>();
            }

            string approvalString = configString +
                                    "_Thresh_" + approvalRatioThreshold +
                                    "_Explore_" + anotherChanceProbablity;

            Dictionary <int, SatyamAggregatedResultsTableEntry> aggEntriesByTaskID    = new Dictionary <int, SatyamAggregatedResultsTableEntry>();
            Dictionary <int, List <SatyamResultsTableEntry> >   ResultEntriesByTaskID = new Dictionary <int, List <SatyamResultsTableEntry> >();

            //Dictionary<DateTime, double> TotalApprovalRatioByTime = new Dictionary<DateTime, double>();
            List <string> TotalApprovalRatioByTime           = new List <string>();
            List <string> TotalApprovalRatioByResult         = new List <string>();
            List <string> FilteredTotalApprovalRatioByTime   = new List <string>();
            List <string> FilteredTotalApprovalRatioByResult = new List <string>();

            List <SatyamResultsTableEntry> approvedEntries             = new List <SatyamResultsTableEntry>();
            List <SatyamResultsTableEntry> rejectedEntries             = new List <SatyamResultsTableEntry>();
            List <SatyamResultsTableEntry> InconclusiveEntries         = new List <SatyamResultsTableEntry>();
            List <SatyamResultsTableEntry> approvedEntriesActuallyPaid = new List <SatyamResultsTableEntry>();


            Dictionary <string, int>    noTasksApprovedPerWorker = new Dictionary <string, int>();
            Dictionary <string, int>    noTasksDonePerWorker     = new Dictionary <string, int>();
            Dictionary <string, double> ApprovalRatioPerWorker   = new Dictionary <string, double>();
            Dictionary <string, bool>   FirstTimersWorkersRecord = new Dictionary <string, bool>();


            List <SatyamResultsTableEntry> FilteredApprovedEntries = new List <SatyamResultsTableEntry>();
            List <SatyamResultsTableEntry> filteredRejectedEntries = new List <SatyamResultsTableEntry>();


            Dictionary <int, double> AggregationTimePerTask = new Dictionary <int, double>();

            foreach (SatyamAggregatedResultsTableEntry aggEntry in aggEntries)
            {
                if (!aggEntriesByTaskID.ContainsKey(aggEntry.SatyamTaskTableEntryID))
                {
                    aggEntriesByTaskID.Add(aggEntry.SatyamTaskTableEntryID, aggEntry);
                }
                else
                {
                    // duplicate agg entry for a task
                }
            }

            //Dictionary<int, int> noResultsNeededForAggregation = getNoResultsNeededForAggregationFromLog(approvalString, guid);
            foreach (int k in noResultsNeededForAggregation_new.Keys)
            {
                noResultsNeededForAggregation.Add(k, noResultsNeededForAggregation_new[k]);
            }
            int count = 0;

            foreach (DateTime t in entriesBySubmitTime.Keys)
            {
                //Console.WriteLine("Processing Results of time: {0}", t);
                Console.WriteLine("Judging result: {0}/{1}", count, entriesBySubmitTime.Count);
                count++;

                List <SatyamResultsTableEntry> ResultEntries = entriesBySubmitTime[t];
                foreach (SatyamResultsTableEntry res in ResultEntries)
                {
                    SatyamResult satyamResult = JSonUtils.ConvertJSonToObject <SatyamResult>(res.ResultString);



                    int taskID = res.SatyamTaskTableEntryID;

                    if (!ResultEntriesByTaskID.ContainsKey(taskID))
                    {
                        ResultEntriesByTaskID.Add(taskID, new List <SatyamResultsTableEntry>());
                    }
                    ResultEntriesByTaskID[taskID].Add(res);

                    if (!aggEntriesByTaskID.ContainsKey(taskID))
                    {
                        //foreach (SatyamResultsTableEntry res in ResultEntriesByTaskID[taskID])
                        //{
                        //InconclusiveEntries.Add(res);
                        //}
                        InconclusiveEntries.Add(res);
                        continue;
                    }
                    SatyamAggregatedResultsTableEntry agg = aggEntriesByTaskID[taskID];
                    //foreach (SatyamResultsTableEntry res in ResultEntriesByTaskID[taskID])
                    //{
                    //SatyamResult satyamResult = JSonUtils.ConvertJSonToObject<SatyamResult>(res.ResultString);
                    string workerId = satyamResult.amazonInfo.WorkerID;

                    // the worker pre-filtering step
                    bool filtered = false;
                    if (noTasksDonePerWorker.ContainsKey(workerId) && noTasksDonePerWorker[workerId] >= minChancesGiven)
                    {
                        double approvalRatio = (double)noTasksApprovedPerWorker[workerId] / (double)noTasksDonePerWorker[workerId];
                        ApprovalRatioPerWorker[workerId] = approvalRatio;
                        if (approvalRatio < approvalRatioThreshold)
                        {
                            // roll a dice to decide whether to skip this worker
                            int rnd = new Random().Next(100);
                            if (rnd > 100 * anotherChanceProbablity)
                            {
                                filtered = true;
                            }
                            else
                            {
                                // another chance granted
                            }
                        }
                    }

                    if (!noTasksDonePerWorker.ContainsKey(workerId))
                    {
                        noTasksDonePerWorker.Add(workerId, 0);
                        noTasksApprovedPerWorker.Add(workerId, 0);
                        ApprovalRatioPerWorker.Add(workerId, -1);
                    }
                    if (!filtered)
                    {
                        noTasksDonePerWorker[workerId]++;
                    }

                    bool acceptable = false;
                    if (usePreApprovalResult)
                    {
                        if (ResultStatus.acceptedStatusList.Contains(res.Status))
                        {
                            acceptable = true;
                        }
                    }
                    else
                    {
                        acceptable = AcceptanceCriterionChecker.IsAcceptable(agg, res);
                    }

                    if (acceptable)
                    {
                        approvedEntries.Add(res);
                        if (!FirstTimersWorkersRecord.ContainsKey(workerId))
                        {
                            FirstTimersWorkersRecord.Add(workerId, true);
                        }

                        if (!filtered)
                        {
                            noTasksApprovedPerWorker[workerId]++;
                            FilteredApprovedEntries.Add(res);
                        }


                        if (noResultsNeededForAggregation != null && noResultsNeededForAggregation.ContainsKey(taskID) &&
                            ResultEntriesByTaskID[taskID].Count <= noResultsNeededForAggregation[taskID])
                        {
                            approvedEntriesActuallyPaid.Add(res);
                        }
                    }
                    else
                    {
                        rejectedEntries.Add(res);
                        if (!FirstTimersWorkersRecord.ContainsKey(workerId))
                        {
                            FirstTimersWorkersRecord.Add(workerId, false);
                        }
                        if (!filtered)
                        {
                            filteredRejectedEntries.Add(res);
                        }
                    }
                    //}

                    /// timetaken till aggregation
                    if (noResultsNeededForAggregation != null && noResultsNeededForAggregation.ContainsKey(taskID) &&
                        ResultEntriesByTaskID[taskID].Count == noResultsNeededForAggregation[taskID])
                    {
                        // aggregation happen
                        SatyamTask task = JSonUtils.ConvertJSonToObject <SatyamTask>(satyamResult.TaskParametersString);
                        SatyamJob  job  = task.jobEntry;
                        if (!AggregationTimePerTask.ContainsKey(taskID))
                        {
                            AggregationTimePerTask.Add(taskID, (res.SubmitTime - job.JobSubmitTime).TotalSeconds);
                        }
                    }



                    string ApprovalString = approvedEntries.Count + " " + (approvedEntries.Count + rejectedEntries.Count) + " "
                                            + (double)approvedEntries.Count / (double)(approvedEntries.Count + rejectedEntries.Count);
                    TotalApprovalRatioByResult.Add(ApprovalString);

                    string filteredApprovalString = FilteredApprovedEntries.Count + " " + (FilteredApprovedEntries.Count + filteredRejectedEntries.Count) + " "
                                                    + (double)FilteredApprovedEntries.Count / (double)(FilteredApprovedEntries.Count + filteredRejectedEntries.Count);
                    FilteredTotalApprovalRatioByResult.Add(filteredApprovalString);
                }

                string approvalStringByTime = t.ToString() + " " + approvedEntries.Count + " " + (approvedEntries.Count + rejectedEntries.Count) + " "
                                              + (double)approvedEntries.Count / (double)(approvedEntries.Count + rejectedEntries.Count);
                TotalApprovalRatioByTime.Add(approvalStringByTime);

                string filteredApprovalStringByTime = t.ToString() + " " + FilteredApprovedEntries.Count + " " + (FilteredApprovedEntries.Count + filteredRejectedEntries.Count) + " "
                                                      + (double)FilteredApprovedEntries.Count / (double)(FilteredApprovedEntries.Count + filteredRejectedEntries.Count);
                FilteredTotalApprovalRatioByTime.Add(filteredApprovalStringByTime);
            }
            // count first timer succ rate
            int FirstTimeSucc = 0;

            foreach (KeyValuePair <string, bool> wid in FirstTimersWorkersRecord)
            {
                if (wid.Value)
                {
                    FirstTimeSucc++;
                }
            }

            double FirstTimeSuccRate = (double)FirstTimeSucc / FirstTimersWorkersRecord.Count;

            Console.WriteLine("Approval #, ratio: {0},{1}", approvedEntries.Count, (double)approvedEntries.Count / (double)(approvedEntries.Count + rejectedEntries.Count));
            Console.WriteLine("Filtered Approval #, ratio: {0},{1}", FilteredApprovedEntries.Count, (double)FilteredApprovedEntries.Count / (double)(FilteredApprovedEntries.Count + filteredRejectedEntries.Count));

            int approvePrevRejected = 0;

            List <string> ApprovedAssignmentIDs = new List <string>();

            foreach (SatyamResultsTableEntry approved in approvedEntries)
            {
                SatyamResult satyamResult = JSonUtils.ConvertJSonToObject <SatyamResult>(approved.ResultString);
                ApprovedAssignmentIDs.Add(satyamResult.amazonInfo.AssignmentID);
                if (approved.Status == ResultStatus.accepted_NotPaid || approved.Status == ResultStatus.rejected_NotPaid || approved.Status == ResultStatus.rejected)
                {
                    approvePrevRejected++;
                }
            }

            Console.WriteLine("Approving Previously Rejected: {0}, {1}", approvePrevRejected, (double)approvePrevRejected / (double)approvedEntries.Count);


            string outputFileDir = DirectoryConstants.defaultTempDirectory + guid + "\\ApprovalAnalysis\\";

            if (guid != null && approvalString != null)
            {
                if (!Directory.Exists(outputFileDir))
                {
                    Directory.CreateDirectory(outputFileDir);
                }
            }

            string summaryFile = outputFileDir + "resultSummary.txt";
            string summary     = String.Format("{0} {1} {2} {3} {4} {5} {6} {7}\n",
                                               approvalString,
                                               approvedEntries.Count, (double)approvedEntries.Count / (double)(approvedEntries.Count + rejectedEntries.Count),
                                               approvedEntriesActuallyPaid.Count,
                                               FilteredApprovedEntries.Count, (double)FilteredApprovedEntries.Count / (double)(FilteredApprovedEntries.Count + filteredRejectedEntries.Count),
                                               approvePrevRejected, (double)approvePrevRejected / (double)approvedEntries.Count);

            File.AppendAllText(summaryFile, summary);

            string outputFile = outputFileDir + approvalString + ".txt";

            File.WriteAllLines(outputFile, ApprovedAssignmentIDs.ToArray());

            //string outputFilePaid = outputFileDir + approvalString + "_ActuallyPaid.txt";
            //File.WriteAllText(outputFilePaid, approvedEntriesActuallyPaid.Count.ToString());

            List <string> approvalRates = new List <string>();

            foreach (string worker in ApprovalRatioPerWorker.Keys)
            {
                approvalRates.Add(ApprovalRatioPerWorker[worker].ToString());
            }
            string approvalDistributionFile = outputFileDir + approvalString + "_dist.txt";

            File.WriteAllLines(approvalDistributionFile, approvalRates.ToArray());

            string approvalByTimeFile   = outputFileDir + approvalString + "_approvalRateByTime.txt";
            string approvalByResultFile = outputFileDir + approvalString + "_approvalRateByResults.txt";

            File.WriteAllLines(approvalByTimeFile, TotalApprovalRatioByTime.ToArray());
            File.WriteAllLines(approvalByResultFile, TotalApprovalRatioByResult.ToArray());

            approvalByTimeFile   = outputFileDir + approvalString + "_approvalRateByTime_filtered.txt";
            approvalByResultFile = outputFileDir + approvalString + "_approvalRateByResults_filtered.txt";
            File.WriteAllLines(approvalByTimeFile, FilteredTotalApprovalRatioByTime.ToArray());
            File.WriteAllLines(approvalByResultFile, FilteredTotalApprovalRatioByResult.ToArray());

            List <string> aggregationTime = new List <string>();

            foreach (int task in AggregationTimePerTask.Keys)
            {
                aggregationTime.Add(task.ToString() + " " + AggregationTimePerTask[task].ToString());
            }
            string aggregationTimeFile = outputFileDir + approvalString + "_aggregationTime.txt";

            File.WriteAllLines(aggregationTimeFile, aggregationTime.ToArray());
        }
示例#2
0
        public static void Run([QueueTrigger("judge-result")] string myQueueItem, TraceWriter log)
        {
            bool logging = false;

            string[] fields   = myQueueItem.Split('_');
            string   guid     = fields[0];
            int      taskID   = Convert.ToInt32(fields[1]);
            string   resultID = fields[2];

            if (logging)
            {
                log.Info($"Judge Result: {myQueueItem}");
            }

            SatyamResultsTableAccess       resultsDB = new SatyamResultsTableAccess();
            List <SatyamResultsTableEntry> results   = resultsDB.getEntriesID(resultID);

            resultsDB.close();
            if (results.Count == 0)
            {
                return;
            }

            SatyamAggregatedResultsTableAccess aggDB     = new SatyamAggregatedResultsTableAccess();
            SatyamAggregatedResultsTableEntry  aggResult = aggDB.getEntryByTaskID(taskID);

            aggDB.close();
            if (aggResult == null)
            {
                return;
            }

            SatyamResultsTableEntry result = results[0];
            SatyamResult            res    = JSonUtils.ConvertJSonToObject <SatyamResult>(result.ResultString);
            string   taskTemplateType      = result.JobTemplateType;
            string   workerID = res.amazonInfo.WorkerID;
            DateTime doneTime = res.TaskEndTime;

            if (AcceptanceCriterionChecker.IsAcceptable(aggResult, result))
            {
                resultsDB = new SatyamResultsTableAccess();
                resultsDB.UpdateStatusByID(result.ID, ResultStatus.accepted);
                resultsDB.close();
                WorkerStatisticsManagement.UpdateWorkerStatistics(workerID, taskTemplateType, true, doneTime);
                if (logging)
                {
                    log.Info($"Accepted");
                }
            }
            else
            {
                resultsDB = new SatyamResultsTableAccess();
                resultsDB.UpdateStatusByID(result.ID, ResultStatus.rejected);
                resultsDB.close();
                WorkerStatisticsManagement.UpdateWorkerStatistics(workerID, taskTemplateType, false, doneTime);
                if (logging)
                {
                    log.Info($"Rejected");
                }
            }
        }
示例#3
0
        //go through all the inconclusive results and check with aggregated table, if aggreagted, compare and see if it is acceptable and then
        //accept/reject
        public static void AcceptRejectResults()
        {
            //get all inconclusive results and group them by taskIDs
            SatyamResultsTableAccess resultsDB = new SatyamResultsTableAccess();

            List <SatyamResultsTableEntry> results = resultsDB.getAllEntriesByStatus(ResultStatus.inconclusive);

            if (results.Count == 0)
            {
                resultsDB.close();
                return;
            }

            Dictionary <int, List <SatyamResultsTableEntry> > resultsByTaskID = new Dictionary <int, List <SatyamResultsTableEntry> >();
            List <string> pendingGUID = new List <string>();

            foreach (SatyamResultsTableEntry result in results)
            {
                if (!resultsByTaskID.ContainsKey(result.SatyamTaskTableEntryID))
                {
                    resultsByTaskID.Add(result.SatyamTaskTableEntryID, new List <SatyamResultsTableEntry>());
                }
                resultsByTaskID[result.SatyamTaskTableEntryID].Add(result);

                if (!pendingGUID.Contains(result.JobGUID))
                {
                    pendingGUID.Add(result.JobGUID);
                }
            }

            //now check against the aggregated result to see if the result is aceptable

            Dictionary <string, Dictionary <int, SatyamAggregatedResultsTableEntry> > aggEntriesPerTaskPerGUID = SatyamAggregatedResultManagement.getAggregatedEntriesPerTaskByGuidList(pendingGUID);

            foreach (KeyValuePair <int, List <SatyamResultsTableEntry> > entry in resultsByTaskID)
            {
                if (entry.Value.Count == 0)
                {
                    continue;
                }
                int    taskEntryID = entry.Key;
                string taskGUID    = entry.Value[0].JobGUID;
                if (aggEntriesPerTaskPerGUID.ContainsKey(taskGUID) && aggEntriesPerTaskPerGUID[taskGUID].ContainsKey(taskEntryID))
                {
                    // this task has been aggregated
                    foreach (SatyamResultsTableEntry result in resultsByTaskID[taskEntryID]) //now got through each task to see if they satify pass criterion
                    {
                        SatyamResult res = JSonUtils.ConvertJSonToObject <SatyamResult>(result.ResultString);
                        string       taskTemplateType = result.JobTemplateType;
                        string       workerID         = res.amazonInfo.WorkerID;
                        DateTime     doneTime         = res.TaskEndTime;

                        if (AcceptanceCriterionChecker.IsAcceptable(aggEntriesPerTaskPerGUID[taskGUID][taskEntryID], result))
                        {
                            resultsDB.UpdateStatusByID(result.ID, ResultStatus.accepted);
                            WorkerStatisticsManagement.UpdateWorkerStatistics(workerID, taskTemplateType, true, doneTime);
                        }
                        else
                        {
                            resultsDB.UpdateStatusByID(result.ID, ResultStatus.rejected);
                            WorkerStatisticsManagement.UpdateWorkerStatistics(workerID, taskTemplateType, false, doneTime);
                        }
                    }
                }
            }
            resultsDB.close();
        }