private bool getNewRandomJob() { double price = 0; bool success = Double.TryParse(Hidden_Price.Value, out price); if (!success) { price = 0; } SatyamTaskTableAccess taskTableDB = new SatyamTaskTableAccess(); SatyamTaskTableEntry entry = null; //Thread.Sleep(5000); if (SubmitButton.Enabled == true) { //entry = taskTableDB.getMinimumTriedEntryByTemplateAndPrice(TaskConstants.Detection_Image_MTurk, price); //entry = taskTableDB.getMinimumTriedNewEntryForWorkerIDByTemplateAndPrice(Hidden_AmazonWorkerID.Value, // TaskConstants.Detection_Image_MTurk, price, doneScore); entry = taskTableDB.getMinimumTriedNewEntryForWorkerIDByTemplateAndPrice(Hidden_AmazonWorkerID.Value, TaskConstants.Detection_Image_MTurk, price); } else { entry = taskTableDB.getMinimumTriedEntryByTemplate(TaskConstants.Detection_Image_MTurk); } if (entry != null) { //doneScore = entry.DoneScore; taskTableDB.IncrementDoneScore(entry.ID); SatyamTask task = JSonUtils.ConvertJSonToObject <SatyamTask>(entry.TaskParametersString); string uri = task.SatyamURI; DisplayImage.ImageUrl = uri; SatyamJob jobDefinitionEntry = task.jobEntry; MultiObjectLocalizationAndLabelingSubmittedJob job = JSonUtils.ConvertJSonToObject <MultiObjectLocalizationAndLabelingSubmittedJob>(jobDefinitionEntry.JobParameters); List <string> categories = job.Categories; CategorySelection_RadioButtonList.Items.Clear(); for (int i = 0; i < categories.Count; i++) { ListItem l = new ListItem(categories[i]); CategorySelection_RadioButtonList.Items.Add(l); } if (job.Description != "") { DescriptionPanel.Visible = true; DescriptionTextPanel.Controls.Add(new LiteralControl(job.Description)); } Hidden_BoundaryLines.Value = JSonUtils.ConvertObjectToJSon(job.BoundaryLines); Hidden_TaskEntryString.Value = JSonUtils.ConvertObjectToJSon <SatyamTaskTableEntry>(entry); Hidden_PageLoadTime.Value = DateTime.Now.ToString(); NoLabeled.Text = Hidden_NoImagesDone.Value; Hidden_TasksPerJob.Value = jobDefinitionEntry.TasksPerJob.ToString(); taskTableDB.close(); return(true); } else { taskTableDB.close(); return(false); } }
public static bool IsAcceptable( SatyamAggregatedResultsTableEntry aggResultEntry, SatyamResultsTableEntry resultEntry, double APPROVAL_RATIO_OF_POLYGONS_THRESHOLD = TaskConstants.IMAGE_SEGMENTATION_MTURK_OBJECT_COVERAGE_THRESHOLD_FOR_PAYMENT, //the person must have made at least 80% of the polygon double POLYGON_IOU_THRESHOLD = TaskConstants.IMAGE_SEGMENTATION_MTURK_POLYGON_IOU_THRESHOLD_FOR_PAYMENT ) { //most boxes should be within limits //most categories should be right SatyamAggregatedResult satyamAggResult = JSonUtils.ConvertJSonToObject <SatyamAggregatedResult>(aggResultEntry.ResultString); ImageSegmentationAggregatedResult aggresult = JSonUtils.ConvertJSonToObject <ImageSegmentationAggregatedResult>(satyamAggResult.AggregatedResultString); SatyamResult satyamResult = JSonUtils.ConvertJSonToObject <SatyamResult>(resultEntry.ResultString); ImageSegmentationResult result = JSonUtils.ConvertJSonToObject <ImageSegmentationResult>(satyamResult.TaskResult); if (result == null) { return(false); } //first check if the number of boxes are within limit // the objects are dummy polygons just for counting int boxLimit = (int)Math.Ceiling((double)aggresult.boxesAndCategories.objects.Count * APPROVAL_RATIO_OF_POLYGONS_THRESHOLD); if (result.objects.Count < boxLimit) { return(false); } //now find how many of the results match aggregated results int noAccepted = 0; byte[] resPNG = ImageSegmentationResult.PolygonResult2Bitmap(result); int width = -1; int height = -1; byte[] aggPNG = ImageUtilities.readPNGRawDataFromURL(aggresult.metaData.PNG_URL, out width, out height); if (resPNG.Length != aggPNG.Length) { Console.WriteLine("res and agg res are different size"); return(false); } SortedDictionary <int, SortedDictionary <int, int> > overlapping = new SortedDictionary <int, SortedDictionary <int, int> >(); SortedDictionary <int, int> resObjectArea = new SortedDictionary <int, int>(); SortedDictionary <int, int> aggObjectArea = new SortedDictionary <int, int>(); for (int i = 0; i < width; i++) { for (int j = 0; j < height; j++) { int idx = j * width + i; int resObjID = resPNG[idx]; int aggObjID = aggPNG[idx]; if (!overlapping.ContainsKey(resObjID)) { overlapping.Add(resObjID, new SortedDictionary <int, int>()); } if (!overlapping[resObjID].ContainsKey(aggObjID)) { overlapping[resObjID].Add(aggObjID, 0); } overlapping[resObjID][aggObjID]++; if (!resObjectArea.ContainsKey(resObjID)) { resObjectArea.Add(resObjID, 0); } resObjectArea[resObjID]++; if (!aggObjectArea.ContainsKey(aggObjID)) { aggObjectArea.Add(aggObjID, 0); } aggObjectArea[aggObjID]++; } } foreach (int id in resObjectArea.Keys) { if (id == 0) { continue; } int maxOverlap = -1; int maxOverlapAggObjID = -1; SortedDictionary <int, int> overlapArea = overlapping[id]; foreach (int aggid in overlapArea.Keys) { if (aggid == 0) { continue; } if (overlapArea[aggid] > maxOverlap) { maxOverlap = overlapArea[aggid]; maxOverlapAggObjID = aggid; } } if (maxOverlapAggObjID == -1) { continue; } double iou = (double)maxOverlap / (double)(resObjectArea[id] + aggObjectArea[maxOverlapAggObjID] - maxOverlap); if (iou >= POLYGON_IOU_THRESHOLD) { noAccepted++; ////now check category //if (result.objects[match.elementList[0]].Category == aggresult.boxesAndCategories.objects[match.elementList[1]].Category) //{ // //both category and bounding box tests have passed //} } } if (noAccepted >= boxLimit) { return(true); } return(false); }
public static bool ProcessAndUploadToAzureBlob(SatyamJobSubmissionsTableAccessEntry jobEntry) { // if the input is a folder of folders of frames, then copy directly SatyamJobStorageAccountAccess satyamStorage = new SatyamJobStorageAccountAccess(); string satyamContainerName = SatyamTaskGenerator.JobTemplateToSatyamContainerNameMap[jobEntry.JobTemplateType]; string GUID = jobEntry.JobGUID; string satyamDirectoryName = GUID; SatyamJob job = JSonUtils.ConvertJSonToObject <SatyamJob>(jobEntry.JobParametersString); MultiObjectTrackingSubmittedJob jobParams = JSonUtils.ConvertJSonToObject <MultiObjectTrackingSubmittedJob>(job.JobParameters); BlobContainerManager bcm = new BlobContainerManager(); string status = bcm.Connect(job.azureInformation.AzureBlobStorageConnectionString); List <string> FileTypes = SatyamTaskGenerator.ValidFileTypesByTemplate[job.JobTemplateType]; if (status != "SUCCESS") { return(false); } string guidFolder = DirectoryConstants.defaultTempDirectory + "\\" + GUID; Directory.CreateDirectory(guidFolder); int chunkLength = jobParams.ChunkDuration; // sec int outputFPS = jobParams.FrameRate; double chunkOverlap = jobParams.ChunkOverlap; // sec var client = new WebClient(); if (jobParams.DataSrcFormat == DataFormat.Video) { // sample to frames int noFramePerChunk = (int)(chunkLength * outputFPS); int noFrameOverlap = (int)(chunkOverlap * outputFPS); List <string> videoUrls = bcm.getURLList(job.azureInformation.AzureBlobStorageContainerName, job.azureInformation.AzureBlobStorageContainerDirectoryName); foreach (string video in videoUrls) { string videoName = URIUtilities.filenameFromURINoExtension(video); string videonameExtension = URIUtilities.filenameFromURI(video); string outputDirectory = guidFolder + "\\" + videoName; Directory.CreateDirectory(outputDirectory); client.DownloadFile(video, outputDirectory + "\\" + videonameExtension); FFMpegWrappers.ExtractFrames(DirectoryConstants.ffmpeg, outputDirectory + "\\" + videonameExtension, outputDirectory, videoName, DateTime.Now, outputFPS); Console.WriteLine("deleting downloaded file..."); File.Delete(outputDirectory + "\\" + videonameExtension); GroupFramesIntoChunksAndUploadChunks(videoName, outputDirectory, noFramePerChunk, noFrameOverlap, satyamContainerName, satyamDirectoryName); Directory.Delete(outputDirectory, true); } } if (jobParams.DataSrcFormat == DataFormat.VideoFrame) { int noFramePerChunk = (int)(chunkLength * outputFPS);//just use one fps for now, assume input should've already downsampled int noFrameOverlap = (int)(chunkOverlap * outputFPS); // chunk according to parameters List <string> frameUrls = bcm.getURLList(job.azureInformation.AzureBlobStorageContainerName, job.azureInformation.AzureBlobStorageContainerDirectoryName); Dictionary <string, List <string> > framesPerVideo = new Dictionary <string, List <string> >(); foreach (string url in frameUrls) { // assumed hierarchy: blob/directory.../videoname/frameid.jpg //string frameName = URIUtilities.filenameFromURINoExtension(url); string[] urlparts = url.Split('/'); string videoName = urlparts[urlparts.Length - 2]; if (!framesPerVideo.ContainsKey(videoName)) { framesPerVideo.Add(videoName, new List <string>()); } framesPerVideo[videoName].Add(url); } foreach (string video in framesPerVideo.Keys) { string outputDirectory = guidFolder + "\\" + video; Directory.CreateDirectory(outputDirectory); foreach (string frameURL in framesPerVideo[video]) { string frameName = URIUtilities.filenameFromURI(frameURL); client.DownloadFile(frameURL, outputDirectory + "\\" + frameName); } GroupFramesIntoChunksAndUploadChunks(video, outputDirectory, noFramePerChunk, noFrameOverlap, satyamContainerName, satyamDirectoryName); Directory.Delete(outputDirectory, true); } } return(true); }
public static void Run([TimerTrigger("0 */5 * * * *")] TimerInfo myTimer, TraceWriter log) { DateTime start = DateTime.Now; bool logging = false; if (logging) { log.Info($"Payment Dispatch executed at: {DateTime.Now}"); } SatyamDispatchStorageAccountAccess satyamQueue = new SatyamDispatchStorageAccountAccess(); //get IDs of all accepted and rejected results related to MTurk Dictionary <string, string> AssignmentIDToHITIDMap = new Dictionary <string, string>(); List <SatyamResultsTableEntry> acceptedEntries = new List <SatyamResultsTableEntry>(); foreach (string taskTemplate in TaskConstants.MTurkTaskTemplates) { SatyamResultsTableAccess resultsDB = new SatyamResultsTableAccess(); List <SatyamResultsTableEntry> entries = resultsDB.getAllEntriesByJobtemplateTypeAndStatus(taskTemplate, ResultStatus.accepted); resultsDB.close(); acceptedEntries.AddRange(entries); } List <SatyamResultsTableEntry> rejectedEntries = new List <SatyamResultsTableEntry>(); foreach (string taskTemplate in TaskConstants.MTurkTaskTemplates) { SatyamResultsTableAccess resultsDB = new SatyamResultsTableAccess(); List <SatyamResultsTableEntry> entries = resultsDB.getAllEntriesByJobtemplateTypeAndStatus(taskTemplate, ResultStatus.rejected); resultsDB.close(); rejectedEntries.AddRange(entries); } if (acceptedEntries.Count == 0 && rejectedEntries.Count == 0) //nothing to do { return; } // a single assignment may have multiple accepted results Dictionary <string, List <SatyamResultsTableEntry> > acceptedResultsByAssignmentID = new Dictionary <string, List <SatyamResultsTableEntry> >(); foreach (SatyamResultsTableEntry entry in acceptedEntries) { SatyamResult sr = JSonUtils.ConvertJSonToObject <SatyamResult>(entry.ResultString); string AssignmentID = sr.amazonInfo.AssignmentID; if (!AssignmentIDToHITIDMap.ContainsKey(AssignmentID)) { string HITID = sr.amazonInfo.HITID; AssignmentIDToHITIDMap.Add(AssignmentID, HITID); } if (!acceptedResultsByAssignmentID.ContainsKey(AssignmentID)) { acceptedResultsByAssignmentID.Add(AssignmentID, new List <SatyamResultsTableEntry>()); } acceptedResultsByAssignmentID[AssignmentID].Add(entry); } List <string> assignmentIDList = acceptedResultsByAssignmentID.Keys.ToList(); // a single assignment may have multiple rejected results Dictionary <string, List <SatyamResultsTableEntry> > rejectedResultsByAssignmentID = new Dictionary <string, List <SatyamResultsTableEntry> >(); foreach (SatyamResultsTableEntry entry in rejectedEntries) { SatyamResult sr = JSonUtils.ConvertJSonToObject <SatyamResult>(entry.ResultString); string AssignmentID = sr.amazonInfo.AssignmentID; if (!AssignmentIDToHITIDMap.ContainsKey(AssignmentID)) { string HITID = sr.amazonInfo.HITID; AssignmentIDToHITIDMap.Add(AssignmentID, HITID); } if (!rejectedResultsByAssignmentID.ContainsKey(AssignmentID)) { rejectedResultsByAssignmentID.Add(AssignmentID, new List <SatyamResultsTableEntry>()); if (!assignmentIDList.Contains(AssignmentID)) { assignmentIDList.Add(AssignmentID); } } rejectedResultsByAssignmentID[AssignmentID].Add(entry); } List <string> rejectedHITs = new List <string>(); List <string> acceptedHITs = new List <string>(); //for each assignment ID we check if it should be paid or not and then pay or reject them. //pay all accepted entries foreach (string assignmentID in assignmentIDList) { int noAccepted = 0; SatyamResult r = null; SatyamResultsTableEntry entry = null; if (acceptedResultsByAssignmentID.ContainsKey(assignmentID)) { noAccepted = acceptedResultsByAssignmentID[assignmentID].Count; entry = acceptedResultsByAssignmentID[assignmentID][0]; r = JSonUtils.ConvertJSonToObject <SatyamResult>(entry.ResultString); } int noRejected = 0; if (rejectedResultsByAssignmentID.ContainsKey(assignmentID)) { noRejected = rejectedResultsByAssignmentID[assignmentID].Count; if (r == null) { entry = rejectedResultsByAssignmentID[assignmentID][0]; r = JSonUtils.ConvertJSonToObject <SatyamResult>(entry.ResultString); } } double ratio = (double)noAccepted / ((double)noAccepted + (double)noRejected); SatyamTask task = JSonUtils.ConvertJSonToObject <SatyamTask>(r.TaskParametersString); string AmazonAccessKeyID = task.jobEntry.amazonHITInformation.AmazonAccessKeyID; string AmazonSecretAccessKeyID = task.jobEntry.amazonHITInformation.AmazonSecretAccessKeyID; string m = AmazonAccessKeyID + "_" + AmazonSecretAccessKeyID + "_" + assignmentID; SatyamResultsTableAccess resultsDB = new SatyamResultsTableAccess(); if (ratio >= AmazonMTurkPayments.assignement_acceptance_threshold) //this is acceptable { if (logging) { log.Info($"Dispatching Payment for {assignmentID}"); } string queueName = "payment"; satyamQueue.Enqueue(queueName, m); if (acceptedResultsByAssignmentID.ContainsKey(assignmentID)) { foreach (SatyamResultsTableEntry result in acceptedResultsByAssignmentID[assignmentID]) { resultsDB.UpdateStatusByID(result.ID, ResultStatus.accepted_Paid); } } if (rejectedResultsByAssignmentID.ContainsKey(assignmentID)) { foreach (SatyamResultsTableEntry result in rejectedResultsByAssignmentID[assignmentID]) { resultsDB.UpdateStatusByID(result.ID, ResultStatus.rejected_Paid); } } if (AssignmentIDToHITIDMap.ContainsKey(assignmentID)) { if (!acceptedHITs.Contains(AssignmentIDToHITIDMap[assignmentID])) { acceptedHITs.Add(AssignmentIDToHITIDMap[assignmentID]); SatyamAmazonHITTableAccess hitDB = new SatyamAmazonHITTableAccess(); hitDB.UpdateStatusByHITID(AssignmentIDToHITIDMap[assignmentID], HitStatus.accepted); hitDB.close(); } } } else { if (logging) { log.Info($"Dispatching NoPayment for {assignmentID}"); } string queueName = "nopayment"; satyamQueue.Enqueue(queueName, m); if (acceptedResultsByAssignmentID.ContainsKey(assignmentID)) { foreach (SatyamResultsTableEntry result in acceptedResultsByAssignmentID[assignmentID]) { resultsDB.UpdateStatusByID(result.ID, ResultStatus.accepted_NotPaid); } } if (rejectedResultsByAssignmentID.ContainsKey(assignmentID)) { foreach (SatyamResultsTableEntry result in rejectedResultsByAssignmentID[assignmentID]) { resultsDB.UpdateStatusByID(result.ID, ResultStatus.rejected_NotPaid); } } if (AssignmentIDToHITIDMap.ContainsKey(assignmentID)) { if (!rejectedHITs.Contains(AssignmentIDToHITIDMap[assignmentID])) { rejectedHITs.Add(AssignmentIDToHITIDMap[assignmentID]); SatyamAmazonHITTableAccess hitDB = new SatyamAmazonHITTableAccess(); hitDB.UpdateStatusByHITID(AssignmentIDToHITIDMap[assignmentID], HitStatus.rejected); hitDB.close(); } } } if ((DateTime.Now - start).TotalSeconds > 280) { break; } } }
public static void SaveResultImagesLocally(List <SatyamResultsTableEntry> entries, string directoryName) { if (!Directory.Exists(directoryName)) { Directory.CreateDirectory(directoryName); } directoryName = directoryName + "\\Raw"; if (!Directory.Exists(directoryName)) { Directory.CreateDirectory(directoryName); } for (int i = 0; i < entries.Count; i++) { SatyamResultsTableEntry entry = entries[i]; SatyamResult satyamResult = JSonUtils.ConvertJSonToObject <SatyamResult>(entry.ResultString); SatyamTask task = JSonUtils.ConvertJSonToObject <SatyamTask>(satyamResult.TaskParametersString); SatyamJob job = task.jobEntry; MultiObjectLocalizationAndLabelingResult res = JSonUtils.ConvertJSonToObject <MultiObjectLocalizationAndLabelingResult>(satyamResult.TaskResult); string ofilename = URIUtilities.filenameFromURI(task.SatyamURI); string[] fields = ofilename.Split('.'); string fileName = fields[0]; Image originalImage = ImageUtilities.getImageFromURI(task.SatyamURI); MultiObjectLocalizationAndLabelingSubmittedJob jobDefinition = JSonUtils.ConvertJSonToObject <MultiObjectLocalizationAndLabelingSubmittedJob>(job.JobParameters); Image imageWithBoundary = DrawingBoxesAndLinesOnImages.addLinesToImage(originalImage, jobDefinition.BoundaryLines, Color.Red, true); List <Rectangle> rectangles = new List <Rectangle>(); List <Color> colors = new List <Color>(); List <string> ids = new List <string>(); List <bool> dashed = new List <bool>(); for (int j = 0; j < res.objects.Count; j++) { MultiObjectLocalizationAndLabelingResultSingleEntry box = res.objects[j]; int x = box.boundingBox.tlx; int y = box.boundingBox.tly; int width = box.boundingBox.brx - box.boundingBox.tlx; int height = box.boundingBox.bry - box.boundingBox.tly; Rectangle r = new Rectangle(x, y, width, height); rectangles.Add(r); string category = box.Category; int colorIndex = jobDefinition.Categories.IndexOf(category); colors.Add(DrawingBoxesAndLinesOnImages.Colors[colorIndex]); string id = j + "-" + category; ids.Add(id); dashed.Add(false); } Image imageWithBoxesAndBoundary = DrawingBoxesAndLinesOnImages.addRectanglesToImage(imageWithBoundary, rectangles, colors, ids, dashed); fileName = fileName + "-Result"; if (satyamResult.amazonInfo.AssignmentID != "") { //fileName = fileName + "-" + satyamResult.amazonInfo.AssignmentID + "_" + entry.ID; fileName = fileName + "-" + entry.ID + "_" + satyamResult.PrevResultID; } ImageUtilities.saveImage(imageWithBoxesAndBoundary, directoryName, fileName); } }
//go through all the inconclusive results and check with aggregated table, if aggreagted, compare and see if it is acceptable and then //accept/reject public static void AcceptRejectResults() { //get all inconclusive results and group them by taskIDs SatyamResultsTableAccess resultsDB = new SatyamResultsTableAccess(); List <SatyamResultsTableEntry> results = resultsDB.getAllEntriesByStatus(ResultStatus.inconclusive); if (results.Count == 0) { resultsDB.close(); return; } Dictionary <int, List <SatyamResultsTableEntry> > resultsByTaskID = new Dictionary <int, List <SatyamResultsTableEntry> >(); List <string> pendingGUID = new List <string>(); foreach (SatyamResultsTableEntry result in results) { if (!resultsByTaskID.ContainsKey(result.SatyamTaskTableEntryID)) { resultsByTaskID.Add(result.SatyamTaskTableEntryID, new List <SatyamResultsTableEntry>()); } resultsByTaskID[result.SatyamTaskTableEntryID].Add(result); if (!pendingGUID.Contains(result.JobGUID)) { pendingGUID.Add(result.JobGUID); } } //now check against the aggregated result to see if the result is aceptable Dictionary <string, Dictionary <int, SatyamAggregatedResultsTableEntry> > aggEntriesPerTaskPerGUID = SatyamAggregatedResultManagement.getAggregatedEntriesPerTaskByGuidList(pendingGUID); foreach (KeyValuePair <int, List <SatyamResultsTableEntry> > entry in resultsByTaskID) { if (entry.Value.Count == 0) { continue; } int taskEntryID = entry.Key; string taskGUID = entry.Value[0].JobGUID; if (aggEntriesPerTaskPerGUID.ContainsKey(taskGUID) && aggEntriesPerTaskPerGUID[taskGUID].ContainsKey(taskEntryID)) { // this task has been aggregated foreach (SatyamResultsTableEntry result in resultsByTaskID[taskEntryID]) //now got through each task to see if they satify pass criterion { SatyamResult res = JSonUtils.ConvertJSonToObject <SatyamResult>(result.ResultString); string taskTemplateType = result.JobTemplateType; string workerID = res.amazonInfo.WorkerID; DateTime doneTime = res.TaskEndTime; if (AcceptanceCriterionChecker.IsAcceptable(aggEntriesPerTaskPerGUID[taskGUID][taskEntryID], result)) { resultsDB.UpdateStatusByID(result.ID, ResultStatus.accepted); WorkerStatisticsManagement.UpdateWorkerStatistics(workerID, taskTemplateType, true, doneTime); } else { resultsDB.UpdateStatusByID(result.ID, ResultStatus.rejected); WorkerStatisticsManagement.UpdateWorkerStatistics(workerID, taskTemplateType, false, doneTime); } } } } resultsDB.close(); }
private bool getNewRandomJob() { SatyamTaskTableAccess taskTableDB = new SatyamTaskTableAccess(); SatyamTaskTableEntry entry = taskTableDB.getMinimumTriedEntryByTemplate(TaskConstants.TrackletLabeling); if (entry != null) { taskTableDB.IncrementDoneScore(entry.ID); } taskTableDB.close(); if (entry != null) { SatyamTask task = JSonUtils.ConvertJSonToObject <SatyamTask>(entry.TaskParametersString); SatyamJobStorageAccountAccess satyamStorage = new SatyamJobStorageAccountAccess(); string videoDir = URIUtilities.localDirectoryFullPathFromURI(task.SatyamURI); List <string> ImageURLs = satyamStorage.getURLListOfSpecificExtensionUnderSubDirectoryByURI(videoDir, new List <string>() { "jpg" }); string annotationFilePath = task.SatyamURI; //string urls = ""; //for (int i=0;i<ImageURLs.Count;i++) //{ // urls += ImageURLs[i]; // if (i == ImageURLs.Count - 1) break; // urls += ','; //} Hidden_ImageURLList.Value = ObjectsToStrings.ListString(ImageURLs, ','); SatyamJob jobDefinitionEntry = task.jobEntry; MultiObjectTrackingSubmittedJob job = JSonUtils.ConvertJSonToObject <MultiObjectTrackingSubmittedJob>(jobDefinitionEntry.JobParameters); Dictionary <string, List <string> > subcategories = job.Categories; List <string> categories = subcategories.Keys.ToList(); //CategorySelection_RakdioButtonList.Items.Clear(); for (int i = 0; i < categories.Count; i++) { ListItem l = new ListItem(categories[i]); //CategorySelection_RadioButtonList.Items.Add(l); } if (job.Description != "") { //DescriptionPanel.Visible = true; //DescriptionTextPanel.Controls.Add(new LiteralControl(job.Description)); } //Hidden_BoundaryLines.Value = JSonUtils.ConvertObjectToJSon(job.BoundaryLines); Hidden_TaskEntryString.Value = JSonUtils.ConvertObjectToJSon <SatyamTaskTableEntry>(entry); Hidden_PageLoadTime.Value = DateTime.Now.ToString(); // pass parameters from old template Slug_Hidden.Value = "null"; Start_Hidden.Value = "0"; Stop_Hidden.Value = (ImageURLs.Count - 1).ToString(); Skip_Hidden.Value = "0"; PerObject_Hidden.Value = "0.1"; Completion_Hidden.Value = "0.5"; BlowRadius_Hidden.Value = "0"; JobId_Hidden.Value = "1"; LabelString_Hidden.Value = ObjectsToStrings.ListString(categories.ToList(), ','); Attributes_Hidden.Value = ObjectsToStrings.DictionaryStringListString(subcategories, ',', ':', '_'); Training_Hidden.Value = "0"; fps_Hidden.Value = job.FrameRate.ToString(); Hidden_ChunkDuration.Value = job.ChunkDuration.ToString(); var web = new WebClient(); System.Drawing.Image x = System.Drawing.Image.FromStream(web.OpenRead(ImageURLs[0])); ImageWidth_Hidden.Value = x.Width.ToString(); ImageHeight_Hidden.Value = x.Height.ToString(); // image boundary for now //string[] region = new string[] { "0-0-1242-0-1242-375-0-375-0-0" }; string[] region = new string[] { "0-0-" + x.Width + "-0-" + x.Width + "-" + x.Height + "-0-" + x.Height + "-0-0" }; RegionString_Hidden.Value = ObjectsToStrings.ListString(region, ','); // temp test List <VATIC_Tracklet> prevTracesTemp = new List <VATIC_Tracklet>(); WebClient client = new WebClient(); Stream stream = client.OpenRead(annotationFilePath); StreamReader reader = new StreamReader(stream); List <string> trace = new List <string>(); while (reader.Peek() >= 0) { string content = reader.ReadLine(); trace.Add(content); } Dictionary <string, VATIC_Tracklet> tracklets = VATIC_Tracklet.ReadTrackletsFromVIRAT(trace); foreach (string id in tracklets.Keys) { //string output = JSonUtils.ConvertObjectToJSon(tracklets[id]); prevTracesTemp.Add(tracklets[id]); } string output = JSonUtils.ConvertObjectToJSon(prevTracesTemp); PreviousTrackString_Hidden.Value = output; return(true); } else { return(false); } }
/// <summary> /// function generalizable.. TODO... to all tasks /// </summary> /// <param name="guids"></param> /// <param name="IoUTreshold"></param> /// <param name="saveImage"></param> /// <param name="outputDirectory"></param> /// <param name="MinResults"></param> /// <param name="MaxResults"></param> /// <param name="CategoryMajorityThreshold"></param> /// <param name="PolygonBoundaryMajorityThreshold"></param> /// <param name="ObjectsCoverageThreshold"></param> /// <param name="overwrite"></param> /// <param name="approvalAnalysis"></param> public static void AggregateWithParameterAndValidatePascalVOCImageSegmentationByGUID(List <string> guids, double IoUTreshold, bool saveImage = false, string outputDirectory = null, int MinResults = TaskConstants.IMAGE_SEGMENTATION_MTURK_MIN_RESULTS_TO_AGGREGATE, int MaxResults = TaskConstants.IMAGE_SEGMENTATION_MTURK_MAX_RESULTS_TO_AGGREGATE, double CategoryMajorityThreshold = TaskConstants.IMAGE_SEGMENTATION_MTURK_MAJORITY_CATEGORY_THRESHOLD, double PolygonBoundaryMajorityThreshold = TaskConstants.IMAGE_SEGMENTATION_MTURK_MAJORITY_POLYGON_BOUNDARY_THRESHOLD, double ObjectsCoverageThreshold = TaskConstants.IMAGE_SEGMENTATION_MTURK_OBJECT_COVERAGE_THRESHOLD_FOR_AGGREGATION_TERMINATION, double minSimilarityThreshold = TaskConstants.IMAGE_SEGMENTATION_MTURK_POLYGON_IOU_THRESHOLD, int minResultsForConsensus = TaskConstants.IMAGE_SEGMENTATION_MTURK_MIN_RESULTS_FOR_CONSENSUS, bool overwrite = false, bool approvalAnalysis = false) { string configString = "Min_" + MinResults + "_Max_" + MaxResults + "_Majority_" + PolygonBoundaryMajorityThreshold + "_Ratio_" + ObjectsCoverageThreshold; Console.WriteLine("Aggregating for param set " + configString); if (!overwrite && File.Exists(DirectoryConstants.defaultTempDirectory + "\\ImageSegmentationResult\\" + configString + ".txt")) { return; } SatyamResultsTableAccess resultsDB = new SatyamResultsTableAccess(); Dictionary <int, List <string> > WorkersPerTask = new Dictionary <int, List <string> >(); List <SatyamResultsTableEntry> entries = new List <SatyamResultsTableEntry>(); foreach (string guid in guids) { entries.AddRange(resultsDB.getEntriesByGUIDOrderByID(guid)); } resultsDB.close(); SortedDictionary <DateTime, List <SatyamResultsTableEntry> > entriesBySubmitTime = SatyamResultValidationToolKit.SortResultsBySubmitTime_OneResultPerTurkerPerTask(entries); Dictionary <int, List <ImageSegmentationResult> > ResultsPerTask = new Dictionary <int, List <ImageSegmentationResult> >(); List <int> aggregatedTasks = new List <int>(); int noTotalConverged = 0; //int noCorrect = 0; int noTerminatedTasks = 0; List <SatyamAggregatedResultsTableEntry> aggEntries = new List <SatyamAggregatedResultsTableEntry>(); Dictionary <int, int> noResultsNeededForAggregation = SatyamResultsAnalysis.getNoResultsNeededForAggregationFromLog(configString, guids[0]); Dictionary <int, int> noResultsNeededForAggregation_new = new Dictionary <int, int>(); // play back by time foreach (DateTime t in entriesBySubmitTime.Keys) { //Console.WriteLine("Processing Results of time: {0}", t); List <SatyamResultsTableEntry> ResultEntries = entriesBySubmitTime[t]; foreach (SatyamResultsTableEntry entry in ResultEntries) { SatyamResult satyamResult = JSonUtils.ConvertJSonToObject <SatyamResult>(entry.ResultString); SatyamTask task = JSonUtils.ConvertJSonToObject <SatyamTask>(satyamResult.TaskParametersString); SatyamJob job = task.jobEntry; string fileName = URIUtilities.filenameFromURINoExtension(task.SatyamURI); int taskEntryID = entry.SatyamTaskTableEntryID; if (aggregatedTasks.Contains(taskEntryID)) { continue; } if (!ResultsPerTask.ContainsKey(taskEntryID)) { ResultsPerTask.Add(taskEntryID, new List <ImageSegmentationResult>()); } ResultsPerTask[taskEntryID].Add(JSonUtils.ConvertJSonToObject <ImageSegmentationResult>(satyamResult.TaskResult)); // check log if enough results are collected if (noResultsNeededForAggregation != null && noResultsNeededForAggregation.ContainsKey(taskEntryID) && ResultsPerTask[taskEntryID].Count < noResultsNeededForAggregation[taskEntryID]) { continue; } ImageSegmentationAggregatedResult aggResult = ImageSegmentationAggregator.getAggregatedResult( ResultsPerTask[taskEntryID], task.SatyamURI, entry.JobGUID, MinResults, MaxResults, CategoryMajorityThreshold, PolygonBoundaryMajorityThreshold, ObjectsCoverageThreshold, minSimilarityThreshold, minResultsForConsensus); if (aggResult == null) { continue; } // aggregation happen // record logs if (noResultsNeededForAggregation == null || !noResultsNeededForAggregation.ContainsKey(taskEntryID)) { noResultsNeededForAggregation_new.Add(taskEntryID, ResultsPerTask[taskEntryID].Count); } aggregatedTasks.Add(taskEntryID); noTotalConverged++; if (ResultsPerTask[taskEntryID].Count >= MaxResults) { noTerminatedTasks++; } SatyamAggregatedResult SatyamAggResult = new SatyamAggregatedResult(); SatyamAggResult.SatyamTaskTableEntryID = taskEntryID; SatyamAggResult.AggregatedResultString = JSonUtils.ConvertObjectToJSon <ImageSegmentationAggregatedResult>(aggResult); SatyamAggResult.TaskParameters = JSonUtils.ConvertJSonToObject <SatyamResult>(entry.ResultString).TaskParametersString; SatyamAggregatedResultsTableEntry aggEntry = new SatyamAggregatedResultsTableEntry(); aggEntry.SatyamTaskTableEntryID = taskEntryID; aggEntry.JobGUID = entry.JobGUID; aggEntry.ResultString = JSonUtils.ConvertObjectToJSon <SatyamAggregatedResult>(SatyamAggResult); aggEntries.Add(aggEntry); List <SatyamAggregatedResultsTableEntry> tmpEntries = new List <SatyamAggregatedResultsTableEntry>(); tmpEntries.Add(aggEntry); //ValidateSatyamKITTIDetectionAggregationResult(tmpEntries, saveImage, MinHeight, MaxOcclusion, Max_Truncation); } } Console.WriteLine("Total_Aggregated_Tasks: {0}", noTotalConverged); Console.WriteLine("Total_Terminated_Tasks: {0}", noTerminatedTasks); SatyamResultsAnalysis.RecordAggregationLog(noResultsNeededForAggregation_new, configString, guids[0]); string r = ValidatePascalVOCImageSegmentationResult_InstanceLevel(aggEntries, IoUTreshold); r = noTotalConverged + " " + noTerminatedTasks + " " + r; File.WriteAllText(DirectoryConstants.defaultTempDirectory + "\\ImageSegmentationResult\\" + configString + ".txt", r); if (approvalAnalysis) { string approvalString = configString + "_PayCover_" + TaskConstants.IMAGE_SEGMENTATION_MTURK_OBJECT_COVERAGE_THRESHOLD_FOR_PAYMENT + "_PayIoU_" + TaskConstants.IMAGE_SEGMENTATION_MTURK_POLYGON_IOU_THRESHOLD_FOR_PAYMENT; //for (double ratio = 0; ratio < 1; ratio += 0.2) //{ // SatyamResultsAnalysis.AnalyzeApprovalRate(aggEntries, entriesBySubmitTime, noResultsNeededForAggregation, noResultsNeededForAggregation_new, guids[0], configString, approvalRatioThreshold: ratio); //} SatyamResultsAnalysis.AggregationAnalysis(aggEntries, entriesBySubmitTime, noResultsNeededForAggregation, noResultsNeededForAggregation_new, guids[0], configString); } }
public static void ValidateSatyamImageNetClassificationAggregationResultByGUID(string guid, string confusingImageListFilePath = null, bool prepareDataForTraining = false, string outputDirectory = null) { Dictionary <string, ConfusingReason> imageBlackListReason = new Dictionary <string, ConfusingReason>(); if (confusingImageListFilePath != null) { imageBlackListReason = getConfusingImageList(confusingImageListFilePath); } //get all aggregated results SatyamAggregatedResultsTableAccess resultsDB = new SatyamAggregatedResultsTableAccess(); List <SatyamAggregatedResultsTableEntry> results = resultsDB.getEntriesByGUID(guid); resultsDB.close(); int noTotal = 0; int noCorrect = 0; SortedDictionary <string, Dictionary <string, int> > confusionMatrix_res_groundtruth = new SortedDictionary <string, Dictionary <string, int> >(); SortedDictionary <string, Dictionary <string, int> > confusionMatrix_groundtruth_res = new SortedDictionary <string, Dictionary <string, int> >(); StringBuilder s = new StringBuilder(); s.Append("<!DOCTYPE html>\n"); s.Append("<html>\n"); s.Append("<body>\n"); String title = String.Format("<h1>Job GUID {0} Incorrect Result Summary</h1>\n", guid); s.Append(title); WebClient wc = new WebClient(); for (int i = 0; i < results.Count; i++) { SatyamSaveAggregatedDataSatyam data = new SatyamSaveAggregatedDataSatyam(results[i]); //String uri = data.SatyamURI; //string[] uri_parts= uri.Split('/'); //string[] name_parts = uri_parts[uri_parts.Length - 1].Split('_'); string fileName = URIUtilities.filenameFromURI(data.SatyamURI); string imageCategoryName = fileName.Split('_')[0]; String resultString = data.AggregatedResultString; SingleObjectLabelingAggregatedResult result = JSonUtils.ConvertJSonToObject <SingleObjectLabelingAggregatedResult>(resultString); // skip all ambulances and black listed if (IsBlackListed(data.SatyamURI, imageBlackListReason)) { continue; } if (!confusionMatrix_res_groundtruth.ContainsKey(result.Category)) { confusionMatrix_res_groundtruth.Add(result.Category, new Dictionary <string, int>() { { GroundTruth[imageCategoryName], 0 } }); } else { if (!confusionMatrix_res_groundtruth[result.Category].ContainsKey(GroundTruth[imageCategoryName])) { confusionMatrix_res_groundtruth[result.Category].Add(GroundTruth[imageCategoryName], 0); } } if (!confusionMatrix_groundtruth_res.ContainsKey(GroundTruth[imageCategoryName])) { confusionMatrix_groundtruth_res.Add(GroundTruth[imageCategoryName], new Dictionary <string, int>() { { result.Category, 0 } }); } else { if (!confusionMatrix_groundtruth_res[GroundTruth[imageCategoryName]].ContainsKey(result.Category)) { confusionMatrix_groundtruth_res[GroundTruth[imageCategoryName]].Add(result.Category, 0); } } if (result.Category.Equals(GroundTruth[imageCategoryName], StringComparison.InvariantCultureIgnoreCase)) { noCorrect++; } else { //Console.WriteLine("{0}, Groundtruth: {1}, Aggregated: {2}, Votes: {3}", // fileName, GroundTruth[imageCategoryName], result.Category, // JSonUtils.ConvertObjectToJSon(result.metaData)); String record = String.Format("<p>{0}, Groundtruth: {1}, Aggregated: {2}, Votes: {3}</p>\n", fileName, GroundTruth[imageCategoryName], result.Category, JSonUtils.ConvertObjectToJSon(result.metaData)); String img = String.Format("<img src=\"{0}\" >\n", data.SatyamURI); s.Append(record); s.Append(img); } // prepare training dataset if (prepareDataForTraining) { if (GroundTruth[imageCategoryName] != "ambulance" && result.Category != "ambulance") { Image im = Image.FromStream(wc.OpenRead(data.SatyamURI)); if (!Directory.Exists(outputDirectory + result.Category)) { Directory.CreateDirectory(outputDirectory + result.Category); } im.Save(outputDirectory + "\\" + result.Category + "\\" + fileName); } } noTotal++; confusionMatrix_res_groundtruth[result.Category][GroundTruth[imageCategoryName]]++; confusionMatrix_groundtruth_res[GroundTruth[imageCategoryName]][result.Category]++; } Console.WriteLine("Result: {0}/{1}, precision: {2}", noCorrect, noTotal, (double)noCorrect / noTotal); // write the confusion matrix s.Append("<p>"); String row = "\t\t"; foreach (string resultCategory in confusionMatrix_res_groundtruth.Keys) { row += resultCategory + "\t"; } row += "<br>\n"; s.Append(row); Console.WriteLine(row); foreach (string groundTruthCategory in confusionMatrix_groundtruth_res.Keys) { row = groundTruthCategory + "\t"; foreach (string resultCategory in confusionMatrix_res_groundtruth.Keys) { if (confusionMatrix_groundtruth_res[groundTruthCategory].ContainsKey(resultCategory)) { row += confusionMatrix_groundtruth_res[groundTruthCategory][resultCategory].ToString(); } else { row += "0"; } row += "\t"; } row += "<br>\n"; s.Append(row); Console.WriteLine(row); } s.Append("</p>\n"); s.Append("</body>\n"); s.Append("</html>\n"); string dataToBeSaved = s.ToString(); SatyamJobStorageAccountAccess storage = new SatyamJobStorageAccountAccess(); string FileName = "AggregatedIncorrectResults-" + guid + ".html"; storage.SaveATextFile("singleobjectlabeling", guid, FileName, dataToBeSaved); }
public static void ValidatePascalVOCImageSegmentationResult_ClassLevel( List <SatyamAggregatedResultsTableEntry> resultsPerImage, List <double> IoUTresholds) { SortedDictionary <int, SatyamAggregatedResultsTableEntry> aggResults = new SortedDictionary <int, SatyamAggregatedResultsTableEntry>(); for (int i = 0; i < resultsPerImage.Count; i++) { int taskID = resultsPerImage[i].SatyamTaskTableEntryID; aggResults.Add(taskID, resultsPerImage[i]); } SortedDictionary <double, int> tpPerIoU = new SortedDictionary <double, int>(); SortedDictionary <double, int> fpPerIoU = new SortedDictionary <double, int>(); SortedDictionary <double, int> fnPerIoU = new SortedDictionary <double, int>(); foreach (double iou in IoUTresholds) { tpPerIoU.Add(iou, 0); fpPerIoU.Add(iou, 0); fnPerIoU.Add(iou, 0); } foreach (int id in aggResults.Keys) { SatyamSaveAggregatedDataSatyam data = new SatyamSaveAggregatedDataSatyam(aggResults[id]); string fileName = URIUtilities.filenameFromURINoExtension(data.SatyamURI); PascalVOCSegmentationGroundTruth gt = new PascalVOCSegmentationGroundTruth(fileName); Console.WriteLine("Results for {0}", fileName); String resultString = data.AggregatedResultString; ImageSegmentationAggregatedResult result = JSonUtils.ConvertJSonToObject <ImageSegmentationAggregatedResult>(resultString); string PNG_URL = result.metaData.PNG_URL; //List<List<double>> IoUs = gt.getIoUs(result.boxesAndCategories); SortedDictionary <int, int> noGroundTruthPixels = new SortedDictionary <int, int>(); SortedDictionary <int, int> noDetectionPixels = new SortedDictionary <int, int>(); //List<List<double>> IoUs = gt.getIoUs(PNG_URL, out noDetectionPixels, out noGroundTruthPixels); List <List <double> > IoUs = gt.getGTOverlaps(PNG_URL, out noDetectionPixels, out noGroundTruthPixels); List <int> DetectionAreas = noDetectionPixels.Values.ToList(); List <int> GroundtruthAreas = noGroundTruthPixels.Values.ToList(); if (IoUs == null) { continue; } //double smallAreaToIgnore = 1000;//px //double smallAreaToIgnore = 625;//px double smallAreaToIgnore = 1600;//px List <int> matchedDetections = new List <int>(); for (int i = 0; i < IoUs.Count; i++) { if (GroundtruthAreas[i] < smallAreaToIgnore) { continue; } List <double> ious = IoUs[i]; double maxIoU = 0; int idx = -1; for (int j = 0; j < ious.Count; j++) { if (ious[j] > maxIoU) { maxIoU = ious[j]; idx = j; } } if (maxIoU == 0) { Console.WriteLine("No Match"); foreach (double iou in IoUTresholds) { fnPerIoU[iou]++; } continue; } if (DetectionAreas[idx] < smallAreaToIgnore) { continue; } Console.WriteLine("Match: {0}", maxIoU); matchedDetections.Add(idx); foreach (double iou in IoUTresholds) { if (maxIoU >= iou) { tpPerIoU[iou]++; } else { fpPerIoU[iou]++; fnPerIoU[iou]++; } } } for (int j = 0; j < IoUs[0].Count; j++) { if (matchedDetections.Contains(j)) { continue; } if (DetectionAreas[j] < smallAreaToIgnore) { continue; } double maxIoU = 0; int idx = -1; for (int i = 0; i < IoUs.Count; i++) { if (IoUs[i][j] > maxIoU) { maxIoU = IoUs[i][j]; idx = j; } } bool alreadyCounted = false; foreach (double iou in IoUTresholds) { if (maxIoU >= iou) { alreadyCounted = true; break; } } if (alreadyCounted) { continue; } Console.WriteLine("No Groundtruth"); foreach (double iou in IoUTresholds) { fpPerIoU[iou]++; } } } double AvgPrec = 0; foreach (double iou in IoUTresholds) { int tp = tpPerIoU[iou] + missingGroundTruth.Count; int fp = fpPerIoU[iou] - missingGroundTruth.Count; int fn = fnPerIoU[iou] - GroundTruthFilter.Count; double ap = (double)tp / (double)(tp + fp + fn); double prec = (double)tp / (double)(tp + fp); double recl = (double)tp / (double)(tp + fn); string ret = String.Format("TP: {0}, FP: {1}, FN {2}, AP, {3}, Precision, {4}, Recall, {5}", tp, fp, fn, ap, prec, recl); Console.WriteLine(ret); AvgPrec += prec; } AvgPrec /= IoUTresholds.Count; Console.WriteLine("AvgPrec :{0}", AvgPrec); }
public static void StaticOfflineAggregationWithParameterAndValidation(List <string> guids, double IoUTreshold, int MinResults = TaskConstants.IMAGE_SEGMENTATION_MTURK_MIN_RESULTS_TO_AGGREGATE, int MaxResults = TaskConstants.IMAGE_SEGMENTATION_MTURK_MAX_RESULTS_TO_AGGREGATE, double CategoryMajorityThreshold = TaskConstants.IMAGE_SEGMENTATION_MTURK_MAJORITY_CATEGORY_THRESHOLD, double PolygonBoundaryMajorityThreshold = TaskConstants.IMAGE_SEGMENTATION_MTURK_MAJORITY_POLYGON_BOUNDARY_THRESHOLD, double ObjectsCoverageThreshold = TaskConstants.IMAGE_SEGMENTATION_MTURK_OBJECT_COVERAGE_THRESHOLD_FOR_AGGREGATION_TERMINATION, double minSimilarityThreshold = TaskConstants.IMAGE_SEGMENTATION_MTURK_POLYGON_IOU_THRESHOLD, int minResultsForConsensus = TaskConstants.IMAGE_SEGMENTATION_MTURK_MIN_RESULTS_FOR_CONSENSUS) { SatyamResultsTableAccess resultsDB = new SatyamResultsTableAccess(); Dictionary <int, List <string> > WorkersPerTask = new Dictionary <int, List <string> >(); List <SatyamResultsTableEntry> entries = new List <SatyamResultsTableEntry>(); foreach (string guid in guids) { entries.AddRange(resultsDB.getEntriesByGUIDOrderByID(guid)); } resultsDB.close(); SortedDictionary <int, List <SatyamResultsTableEntry> > EntriesPerTask = new SortedDictionary <int, List <SatyamResultsTableEntry> >(); SortedDictionary <int, List <ImageSegmentationResult> > ResultsPerTask = new SortedDictionary <int, List <ImageSegmentationResult> >(); List <int> aggregatedTasks = new List <int>(); int noTotalConverged = 0; //int noCorrect = 0; int noTerminatedTasks = 0; List <SatyamAggregatedResultsTableEntry> aggEntries = new List <SatyamAggregatedResultsTableEntry>(); // Organize by taskID foreach (SatyamResultsTableEntry entry in entries) { SatyamResult satyamResult = JSonUtils.ConvertJSonToObject <SatyamResult>(entry.ResultString); int taskEntryID = entry.SatyamTaskTableEntryID; if (!EntriesPerTask.ContainsKey(taskEntryID)) { EntriesPerTask.Add(taskEntryID, new List <SatyamResultsTableEntry>()); ResultsPerTask.Add(taskEntryID, new List <ImageSegmentationResult>()); } EntriesPerTask[taskEntryID].Add(entry); ResultsPerTask[taskEntryID].Add(JSonUtils.ConvertJSonToObject <ImageSegmentationResult>(satyamResult.TaskResult)); } foreach (int taskEntryID in EntriesPerTask.Keys) { SatyamResult satyamResult = JSonUtils.ConvertJSonToObject <SatyamResult>(EntriesPerTask[taskEntryID][0].ResultString); SatyamTask task = JSonUtils.ConvertJSonToObject <SatyamTask>(satyamResult.TaskParametersString); SatyamJob job = task.jobEntry; string fileName = URIUtilities.filenameFromURINoExtension(task.SatyamURI); List <int> taskidfilter = new List <int>() { 40430, 40432, 40433, 40434, 40440, 40447, 40451, 40460, }; //if (fileName != "2007_000549") continue; if (!taskidfilter.Contains(satyamResult.TaskTableEntryID)) { continue; } Console.WriteLine("Aggregating task {0}: {1} results", taskEntryID, EntriesPerTask[taskEntryID].Count); ImageSegmentationAggregatedResult aggResult = ImageSegmentationAggregator.getAggregatedResult( ResultsPerTask[taskEntryID], task.SatyamURI, job.JobGUIDString, MinResults, MaxResults, CategoryMajorityThreshold, PolygonBoundaryMajorityThreshold, ObjectsCoverageThreshold, minSimilarityThreshold, minResultsForConsensus); if (aggResult == null) { continue; } // aggregation happen aggregatedTasks.Add(taskEntryID); noTotalConverged++; if (ResultsPerTask[taskEntryID].Count >= MaxResults) { noTerminatedTasks++; } SatyamAggregatedResult SatyamAggResult = new SatyamAggregatedResult(); SatyamAggResult.SatyamTaskTableEntryID = taskEntryID; SatyamAggResult.AggregatedResultString = JSonUtils.ConvertObjectToJSon <ImageSegmentationAggregatedResult>(aggResult); SatyamAggResult.TaskParameters = satyamResult.TaskParametersString; SatyamAggregatedResultsTableEntry aggEntry = new SatyamAggregatedResultsTableEntry(); aggEntry.SatyamTaskTableEntryID = taskEntryID; aggEntry.JobGUID = job.JobGUIDString; aggEntry.ResultString = JSonUtils.ConvertObjectToJSon <SatyamAggregatedResult>(SatyamAggResult); aggEntries.Add(aggEntry); List <SatyamAggregatedResultsTableEntry> tmpEntries = new List <SatyamAggregatedResultsTableEntry>(); tmpEntries.Add(aggEntry); ValidatePascalVOCImageSegmentationResult_InstanceLevel(tmpEntries, IoUTreshold); } Console.WriteLine("Total_Aggregated_Tasks: {0}", noTotalConverged); Console.WriteLine("Total_Terminated_Tasks: {0}", noTerminatedTasks); string r = ValidatePascalVOCImageSegmentationResult_InstanceLevel(aggEntries, IoUTreshold); }
public static string ValidatePascalVOCImageSegmentationResult_InstanceLevel( List <SatyamAggregatedResultsTableEntry> resultsPerImage, double IoUTreshold) { SortedDictionary <int, SatyamAggregatedResultsTableEntry> aggResults = new SortedDictionary <int, SatyamAggregatedResultsTableEntry>(); for (int i = 0; i < resultsPerImage.Count; i++) { int taskID = resultsPerImage[i].SatyamTaskTableEntryID; aggResults.Add(taskID, resultsPerImage[i]); } int tp = 0; int fp = 0; int fn = 0; foreach (int id in aggResults.Keys) { SatyamSaveAggregatedDataSatyam data = new SatyamSaveAggregatedDataSatyam(aggResults[id]); string fileName = URIUtilities.filenameFromURINoExtension(data.SatyamURI); PascalVOCSegmentationGroundTruth gt = new PascalVOCSegmentationGroundTruth(fileName); Console.WriteLine("Results for {0}", fileName); String resultString = data.AggregatedResultString; ImageSegmentationAggregatedResult result = JSonUtils.ConvertJSonToObject <ImageSegmentationAggregatedResult>(resultString); string PNG_URL = result.metaData.PNG_URL; //List<List<double>> IoUs = gt.getIoUs(result.boxesAndCategories); SortedDictionary <int, int> noGroundTruthPixels = new SortedDictionary <int, int>(); SortedDictionary <int, int> noDetectionPixels = new SortedDictionary <int, int>(); List <List <double> > IoUs = gt.getIoUs(PNG_URL, out noDetectionPixels, out noGroundTruthPixels); List <int> DetectionAreas = noDetectionPixels.Values.ToList(); List <int> GroundtruthAreas = noGroundTruthPixels.Values.ToList(); if (IoUs == null) { continue; } MultipartiteWeightTensor matches = new MultipartiteWeightTensor(2); matches.setNumPartitionElements(0, IoUs.Count); matches.setNumPartitionElements(1, IoUs[0].Count); double[,] array = new double[IoUs.Count, IoUs[0].Count]; for (int j = 0; j < IoUs.Count; j++) { for (int k = 0; k < IoUs[0].Count; k++) { array[j, k] = IoUs[j][k]; } } matches.setWeightMatrix(0, 1, array); MultipartiteWeightedMatching.GreedyMean matching = new MultipartiteWeightedMatching.GreedyMean(); List <MultipartiteWeightedMatch> polygonAssociation = matching.getMatching(matches); //double smallAreaToIgnore = 1000;//px //double smallAreaToIgnore = 625;//px double smallAreaToIgnore = 1600;//px foreach (MultipartiteWeightedMatch match in polygonAssociation) { if (match.elementList.ContainsKey(0)) // this contains gt { if (GroundtruthAreas[match.elementList[0]] < smallAreaToIgnore) { continue; } if (match.elementList.ContainsKey(1)) // an aggregated result box has been associated { if (DetectionAreas[match.elementList[1]] < smallAreaToIgnore) { continue; } double IoU = IoUs[match.elementList[0]][match.elementList[1]]; Console.WriteLine("Match: {0}", IoU); if (IoU >= IoUTreshold) { tp++; } else { fp++; fn++; } } else { Console.WriteLine("No Match"); fn++; } } else { if (DetectionAreas[match.elementList[1]] < smallAreaToIgnore) { continue; } Console.WriteLine("No Groundtruth"); fp++; } } } double ap = (double)tp / (double)(tp + fp + fn); double prec = (double)tp / (double)(tp + fp); double recl = (double)tp / (double)(tp + fn); string ret = String.Format("TP: {0}, FP: {1}, FN {2}, AP, {3}, Precision, {4}, Recall, {5}", tp, fp, fn, ap, prec, recl); Console.WriteLine(ret); return(ret); }
protected void SubmitButton_Click(object sender, EventArgs e) { DateTime SubmitTime = DateTime.Now; DateTime PageLoadTime = Convert.ToDateTime(Hidden_PageLoadTime.Value); int count; bool isValidCount = int.TryParse(CountTextBox.Text, out count); if (!isValidCount) { ErrorLabel.Text = "Error : Ivalid Count."; ErrorLabel.ForeColor = System.Drawing.Color.Red; ErrorLabel.Font.Bold = true; } else if (count < 0) { ErrorLabel.Text = "Error : Count cannot be less than zero."; ErrorLabel.ForeColor = System.Drawing.Color.Red; ErrorLabel.Font.Bold = true; } else { ErrorLabel.Text = ""; SatyamTaskTableEntry taskEntry = JSonUtils.ConvertJSonToObject <SatyamTaskTableEntry>(Hidden_TaskEntryString.Value); SatyamResult result = new SatyamResult(); result.TaskParametersString = taskEntry.TaskParametersString; result.TaskStartTime = PageLoadTime; result.TaskEndTime = SubmitTime; result.TaskTableEntryID = taskEntry.ID; AmazonTaskResultInfo amazonInfo = new AmazonTaskResultInfo(); amazonInfo.AssignmentID = ""; amazonInfo.WorkerID = ""; amazonInfo.HITID = ""; result.amazonInfo = amazonInfo; ObjectCountingResult sresult = new ObjectCountingResult(); sresult.Count = count; string sresultString = JSonUtils.ConvertObjectToJSon <ObjectCountingResult>(sresult); result.TaskResult = sresultString; string resultString = JSonUtils.ConvertObjectToJSon <SatyamResult>(result); SatyamResultsTableAccess resultdb = new SatyamResultsTableAccess(); resultdb.AddEntry(taskEntry.JobTemplateType, taskEntry.UserID, taskEntry.JobGUID, resultString, taskEntry.ID, PageLoadTime, SubmitTime); resultdb.close(); //SatyamTaskTableAccess taskDB = new SatyamTaskTableAccess(); //taskDB.IncrementDoneScore(taskEntry.ID); bool NotDone = getNewRandomJob(); if (NotDone == false) { Response.Redirect("AllJobsDone.aspx"); } } }
private void storeResult() { DateTime SubmitTime = DateTime.Now; DateTime PageLoadTime = Convert.ToDateTime(Hidden_PageLoadTime.Value); SatyamTaskTableEntry taskEntry = JSonUtils.ConvertJSonToObject <SatyamTaskTableEntry>(Hidden_TaskEntryString.Value); string tracksString = TracksOutput_Hidden.Value; Console.WriteLine(tracksString); string urlList = Hidden_ImageURLList.Value; string[] fields = urlList.Split(','); DateTime start = DateTime.MinValue; List <DateTime> frameTimes = new List <DateTime>(); //double frameTimeSpanInMiliseconds = (Convert.ToDouble(Hidden_ChunkDuration.Value) / (double)fields.Length) * 1000; double frameTimeSpanInMiliseconds = (double)(1000) / Convert.ToDouble(fps_Hidden.Value); for (int i = 0; i < fields.Length; i++) { DateTime t; t = start.AddMilliseconds(frameTimeSpanInMiliseconds * i); frameTimes.Add(t); } string s = Raw_VATIC_DVA_Crowdsourced_Track_Collection.Raw_VATIC_DVA_Crowdsourced_Track_Collection_ToTrackStrings(tracksString, frameTimes); SatyamResult result = new SatyamResult(); result.TaskParametersString = taskEntry.TaskParametersString; result.TaskStartTime = PageLoadTime; result.TaskEndTime = SubmitTime; result.TaskTableEntryID = taskEntry.ID; AmazonTaskResultInfo amazonInfo = new AmazonTaskResultInfo(); amazonInfo.AssignmentID = ""; amazonInfo.WorkerID = ""; amazonInfo.HITID = ""; result.amazonInfo = amazonInfo; result.TaskResult = s; string resultString = JSonUtils.ConvertObjectToJSon <SatyamResult>(result); SatyamResultsTableAccess resultdb = new SatyamResultsTableAccess(); resultdb.AddEntry(taskEntry.JobTemplateType, taskEntry.UserID, taskEntry.JobGUID, resultString, taskEntry.ID, PageLoadTime, SubmitTime); resultdb.close(); //SatyamTaskTableAccess taskDB = new SatyamTaskTableAccess(); //taskDB.IncrementDoneScore(taskEntry.ID); bool NotDone = getNewRandomJob(); if (NotDone == false) { Response.Redirect("AllJobsDone.aspx"); } }
protected void SubmitButton_Click(object sender, EventArgs e) { DateTime SubmitTime = DateTime.Now; DateTime PageLoadTime = Convert.ToDateTime(Hidden_PageLoadTime.Value); SatyamTaskTableEntry taskEntry = JSonUtils.ConvertJSonToObject <SatyamTaskTableEntry>(Hidden_TaskEntryString.Value); SatyamResult result = new SatyamResult(); result.TaskParametersString = taskEntry.TaskParametersString; result.TaskStartTime = PageLoadTime; result.TaskEndTime = SubmitTime; result.TaskTableEntryID = taskEntry.ID; AmazonTaskResultInfo amazonInfo = new AmazonTaskResultInfo(); amazonInfo.AssignmentID = Hidden_AmazonAssignmentID.Value; amazonInfo.WorkerID = Hidden_AmazonWorkerID.Value; amazonInfo.HITID = Hidden_HITID.Value; amazonInfo.PricePerHIT = Convert.ToDouble(Hidden_Price.Value); result.amazonInfo = amazonInfo; result.TaskResult = Hidden_Result.Value; string resultString = JSonUtils.ConvertObjectToJSon <SatyamResult>(result); SatyamResultsTableAccess resultdb = new SatyamResultsTableAccess(); resultdb.AddEntry(taskEntry.JobTemplateType, taskEntry.UserID, taskEntry.JobGUID, resultString, taskEntry.ID, PageLoadTime, SubmitTime); resultdb.close(); SatyamTaskTableManagement.UpdateResultNumber(taskEntry.ID); //SatyamTaskTableAccess taskDB = new SatyamTaskTableAccess(); //taskDB.IncrementDoneScore(taskEntry.ID); int noDone = Convert.ToInt32(Hidden_NoImagesDone.Value); noDone++; Hidden_NoImagesDone.Value = noDone.ToString(); int noTasksPerJob = Convert.ToInt32(Hidden_TasksPerJob.Value); bool NotDone = false; if (noDone < noTasksPerJob) { NotDone = getNewRandomJob(); } if (NotDone == false) { if (!Testing) { SatyamAmazonHITTableAccess HITdb = new SatyamAmazonHITTableAccess(); string HITID = result.amazonInfo.HITID; HITdb.UpdateStatusByHITID(HITID, HitStatus.submitted); HITdb.close(); AmazonMTurkNotification.submitAmazonTurkHit(result.amazonInfo.AssignmentID, result.amazonInfo.WorkerID, false); } Response.Redirect("MTurkTaskDonePage.aspx"); } }
public static void AggregateWithParameterAndValidateSatyamImageNetClassificationResultByGUID(string guid, int MinResults = TaskConstants.SINGLE_OBJECT_LABLING_MTURK_MIN_RESULTS_TO_AGGREGATE, int MaxResults = TaskConstants.SINGLE_OBJECT_LABLING_MTURK_MAX_RESULTS_TO_AGGREGATE, double MajorityThreshold = TaskConstants.SINGLE_OBJECT_LABLING_MTURK_MAJORITY_THRESHOLD, string confusingImageListFilePath = null) { string configString = "Min_" + MinResults + "_Max_" + MaxResults + "_Thresh_" + MajorityThreshold; Console.WriteLine("Aggregating for param set " + configString); int noTerminatedTasks = 0; SatyamResultsTableAccess resultsDB = new SatyamResultsTableAccess(); List <SatyamResultsTableEntry> entries = resultsDB.getEntriesByGUIDOrderByID(guid); resultsDB.close(); //SortedDictionary<DateTime, List<SatyamResultsTableEntry>> entriesBySubmitTime = // SatyamResultValidation.SortResultsBySubmitTime_OneResultPerTurkerPerTask(entries); SortedDictionary <DateTime, List <SatyamResultsTableEntry> > entriesBySubmitTime = SatyamResultValidationToolKit.SortResultsBySubmitTime(entries); Dictionary <string, ConfusingReason> imageBlackListReason = new Dictionary <string, ConfusingReason>(); if (confusingImageListFilePath != null) { imageBlackListReason = getConfusingImageList(confusingImageListFilePath); } //Dictionary<int, List<SingleObjectLabelingResult>> ResultsPerTask = new Dictionary<int, List<SingleObjectLabelingResult>>(); Dictionary <int, List <SatyamResultsTableEntry> > ResultsPerTask = new Dictionary <int, List <SatyamResultsTableEntry> >(); List <int> aggregatedTasks = new List <int>(); int noTotalConverged = 0; int noCorrect = 0; SortedDictionary <string, Dictionary <string, int> > confusionMatrix_res_groundtruth = new SortedDictionary <string, Dictionary <string, int> >(); SortedDictionary <string, Dictionary <string, int> > confusionMatrix_groundtruth_res = new SortedDictionary <string, Dictionary <string, int> >(); StringBuilder s = new StringBuilder(); s.Append("<!DOCTYPE html>\n"); s.Append("<html>\n"); s.Append("<body>\n"); String title = String.Format("<h1>Job GUID {0} Incorrect Result Summary</h1>\n", guid); s.Append(title); Dictionary <int, int> noResultsNeededForAggregation = SatyamResultsAnalysis.getNoResultsNeededForAggregationFromLog(configString, guid); Dictionary <int, int> noResultsNeededForAggregation_new = new Dictionary <int, int>(); List <SatyamAggregatedResultsTableEntry> aggEntries = new List <SatyamAggregatedResultsTableEntry>(); foreach (DateTime t in entriesBySubmitTime.Keys) { foreach (SatyamResultsTableEntry entry in entriesBySubmitTime[t]) { //SatyamResultsTableEntry entry = entries[i]; SatyamResult satyamResult = JSonUtils.ConvertJSonToObject <SatyamResult>(entry.ResultString); SatyamTask task = JSonUtils.ConvertJSonToObject <SatyamTask>(satyamResult.TaskParametersString); SatyamJob job = task.jobEntry; string fileName = URIUtilities.filenameFromURI(task.SatyamURI); string imageCategoryName = fileName.Split('_')[0]; if (IsBlackListed(task.SatyamURI, imageBlackListReason)) { continue; } int taskEntryID = entry.SatyamTaskTableEntryID; if (aggregatedTasks.Contains(taskEntryID)) { continue; } if (!ResultsPerTask.ContainsKey(taskEntryID)) { ResultsPerTask.Add(taskEntryID, new List <SatyamResultsTableEntry>()); } //ResultEntriesPerTask[taskEntryID].Add(JSonUtils.ConvertJSonToObject<SingleObjectLabelingResult>(satyamResult.TaskResult)); ResultsPerTask[taskEntryID].Add(entry); // check log if enough results are collected if (noResultsNeededForAggregation != null && noResultsNeededForAggregation.ContainsKey(taskEntryID) && ResultsPerTask[taskEntryID].Count < noResultsNeededForAggregation[taskEntryID]) { continue; } string aggResultString = SingleObjectLabelingAggregator.GetAggregatedResultString(ResultsPerTask[taskEntryID], MinResults, MaxResults, MajorityThreshold); //SingleObjectLabelingAggregatedResult aggResult = SingleObjectLabelingAggregator.getAggregatedResult(ResultEntriesPerTask[taskEntryID], MinResults, MaxResults, MajorityThreshold); if (aggResultString == null) { continue; } SatyamAggregatedResultsTableEntry aggEntry = new SatyamAggregatedResultsTableEntry(); aggEntry.JobGUID = ResultsPerTask[taskEntryID][0].JobGUID; aggEntry.JobTemplateType = ResultsPerTask[taskEntryID][0].JobTemplateType; aggEntry.SatyamTaskTableEntryID = ResultsPerTask[taskEntryID][0].SatyamTaskTableEntryID; aggEntry.UserID = ResultsPerTask[taskEntryID][0].UserID; aggEntry.ResultString = aggResultString; /// aggregation happen // record logs if (noResultsNeededForAggregation == null || !noResultsNeededForAggregation.ContainsKey(taskEntryID)) { noResultsNeededForAggregation_new.Add(taskEntryID, ResultsPerTask[taskEntryID].Count); } /// //if (aggResult.Category == "None of the Above") //{ // continue; //} aggEntries.Add(aggEntry); SatyamSaveAggregatedDataSatyam data = new SatyamSaveAggregatedDataSatyam(aggEntry); String resultString = data.AggregatedResultString; SingleObjectLabelingAggregatedResult aggResult = JSonUtils.ConvertJSonToObject <SingleObjectLabelingAggregatedResult>(resultString); if (!confusionMatrix_res_groundtruth.ContainsKey(aggResult.Category)) { confusionMatrix_res_groundtruth.Add(aggResult.Category, new Dictionary <string, int>() { { GroundTruth[imageCategoryName], 0 } }); } else { if (!confusionMatrix_res_groundtruth[aggResult.Category].ContainsKey(GroundTruth[imageCategoryName])) { confusionMatrix_res_groundtruth[aggResult.Category].Add(GroundTruth[imageCategoryName], 0); } } if (!confusionMatrix_groundtruth_res.ContainsKey(GroundTruth[imageCategoryName])) { confusionMatrix_groundtruth_res.Add(GroundTruth[imageCategoryName], new Dictionary <string, int>() { { aggResult.Category, 0 } }); } else { if (!confusionMatrix_groundtruth_res[GroundTruth[imageCategoryName]].ContainsKey(aggResult.Category)) { confusionMatrix_groundtruth_res[GroundTruth[imageCategoryName]].Add(aggResult.Category, 0); } } if (aggResult.Category.Equals(GroundTruth[imageCategoryName], StringComparison.InvariantCultureIgnoreCase)) { noCorrect++; } else { //Console.WriteLine("{0}, Groundtruth: {1}, Aggregated: {2}, Votes: {3}", // fileName, GroundTruth[imageCategoryName], aggResult.Category, // JSonUtils.ConvertObjectToJSon(aggResult.metaData)); String record = String.Format("<p>{0}, Groundtruth: {1}, Aggregated: {2}, Votes: {3}</p>\n", fileName, GroundTruth[imageCategoryName], aggResult.Category, JSonUtils.ConvertObjectToJSon(aggResult.metaData)); String img = String.Format("<img src=\"{0}\" >\n", task.SatyamURI); s.Append(record); s.Append(img); } noTotalConverged++; if (ResultsPerTask[taskEntryID].Count >= MaxResults) { noTerminatedTasks++; } confusionMatrix_res_groundtruth[aggResult.Category][GroundTruth[imageCategoryName]]++; confusionMatrix_groundtruth_res[GroundTruth[imageCategoryName]][aggResult.Category]++; aggregatedTasks.Add(taskEntryID); } } SatyamResultsAnalysis.RecordAggregationLog(noResultsNeededForAggregation_new, configString, guid); s.Append("</body>\n"); s.Append("</html>\n"); string dataToBeSaved = s.ToString(); SatyamJobStorageAccountAccess storage = new SatyamJobStorageAccountAccess(); string FileName = String.Format("AggregatedIncorrectResults-{0}_Min{1}Max{2}Thresh{3}.html", guid, MinResults, MaxResults, MajorityThreshold); storage.SaveATextFile("singleobjectlabeling", guid, FileName, dataToBeSaved); s.Clear(); s.Append("<!DOCTYPE html>\n"); s.Append("<html>\n"); s.Append("<body>\n"); s.Append(title); string resultSummary = String.Format("<p>Result: {0}/{1}, precision: {2}</p>\n", noCorrect, noTotalConverged, (double)noCorrect / noTotalConverged); resultSummary += String.Format("<p>Terminated: {0}, Not Enough Results: {1}</p>\n", noTerminatedTasks, ResultsPerTask.Count - noTotalConverged); Console.WriteLine(resultSummary); s.Append(resultSummary); // write the confusion matrix s.Append("<p>"); String row = "\t\t"; foreach (string resultCategory in confusionMatrix_res_groundtruth.Keys) { row += resultCategory + "\t"; } row += "<br>\n"; s.Append(row); Console.WriteLine(row); string matString = ""; foreach (string groundTruthCategory in confusionMatrix_groundtruth_res.Keys) { row = groundTruthCategory + "\t"; foreach (string resultCategory in confusionMatrix_res_groundtruth.Keys) { if (confusionMatrix_groundtruth_res[groundTruthCategory].ContainsKey(resultCategory)) { row += confusionMatrix_groundtruth_res[groundTruthCategory][resultCategory].ToString(); } else { row += "0"; } row += "\t"; } row += "<br>\n"; s.Append(row); Console.WriteLine(row); matString += row; } s.Append("</p>\n"); s.Append("</body>\n"); s.Append("</html>\n"); string summaryToBeSaved = s.ToString(); FileName = String.Format("Aggregated_Summary-{0}_Min{1}Max{2}Thresh{3}.html", guid, MinResults, MaxResults, MajorityThreshold); storage.SaveATextFile("singleobjectlabeling", guid, FileName, summaryToBeSaved); /// local file string outputString = String.Format("{0} {1} {2} {3} {4} {5}\n", configString, noCorrect, noTotalConverged, (double)noCorrect / noTotalConverged, noTerminatedTasks, ResultsPerTask.Count - noTotalConverged); string outputfile = DirectoryConstants.defaultTempDirectory + guid + "\\resultSummary.txt"; File.AppendAllText(outputfile, outputString); string outputmatFile = DirectoryConstants.defaultTempDirectory + guid + "\\" + configString + "_mat.txt"; File.WriteAllText(outputmatFile, matString); //for (double prob = 0; prob < 1;prob +=0.2) //{ // SatyamResultsAnalysis.AnalyzeApprovalRate(aggEntries, entries, guid, configString, anotherChanceProbablity: prob); //} //for (double ratio = 0; ratio < 1; ratio += 0.2) //{ // SatyamResultsAnalysis.AnalyzeApprovalRate(aggEntries, entriesBySubmitTime, noResultsNeededForAggregation, noResultsNeededForAggregation_new, guid, configString, approvalRatioThreshold: ratio); //} SatyamResultsAnalysis.AggregationAnalysis(aggEntries, entriesBySubmitTime, noResultsNeededForAggregation, noResultsNeededForAggregation_new, guid, configString); }
public static void AggregateWithParameterAndValidateObjectCountingResultByGUID(string guid, //bool saveImage = false, bool prepareTrainingSet = false, string outputDirectory = null, int MinHeight = TaskConstants.OBJECT_COUNTING_VALIDATION_MIN_HEIGHT, int MaxOcclusion = TaskConstants.OBJECT_COUNTING_VALIDATION_MAX_OCCLUSION, double Max_Truncation = TaskConstants.OBJECT_COUNTING_VALIDATION_MIN_TRUNCATION, int MinResults = TaskConstants.OBJECT_COUNTING_MTURK_MIN_RESULTS_TO_AGGREGATE, int MaxResults = TaskConstants.OBJECT_COUNTING_MTURK_MAX_RESULTS_TO_AGGREGATE, double MAX_ABSOLUTE_COUNT_DEVIATION_LOWERBOUND = TaskConstants.OBJECT_COUNTING_MTURK_MAX_ABSOLUTE_COUNT_DEVIATION_LOWERBOUND, double MAX_DEVIATION_FRACTION = TaskConstants.OBJECT_COUNTING_MTURK_MAX_DEVIATION_FRACTION, double SUPER_MAJORITY_VALUE = TaskConstants.OBJECT_COUNTING_MTURK_SUPER_MAJORITY_VALUE ) { if (!Directory.Exists(DirectoryConstants.defaultTempDirectory + guid)) { Directory.CreateDirectory(DirectoryConstants.defaultTempDirectory + guid); } string configString = "Min_" + MinResults + "_Max_" + MaxResults + "_Dev_" + MAX_DEVIATION_FRACTION + "_Majority_" + SUPER_MAJORITY_VALUE; Console.WriteLine("Aggregating for param set " + configString); int noTerminatedTasks = 0; SatyamResultsTableAccess resultsDB = new SatyamResultsTableAccess(); List <SatyamResultsTableEntry> entries = resultsDB.getEntriesByGUIDOrderByID(guid); resultsDB.close(); SortedDictionary <DateTime, List <SatyamResultsTableEntry> > entriesBySubmitTime = SatyamResultValidationToolKit.SortResultsBySubmitTime(entries); int noTotalConverged = 0; //int noCorrect = 0; Dictionary <int, List <SatyamResultsTableEntry> > ResultsPerTask = new Dictionary <int, List <SatyamResultsTableEntry> >(); List <int> aggregatedTasks = new List <int>(); List <SatyamAggregatedResultsTableEntry> aggEntries = new List <SatyamAggregatedResultsTableEntry>(); Dictionary <int, int> noResultsNeededForAggregation = SatyamResultsAnalysis.getNoResultsNeededForAggregationFromLog(configString, guid); Dictionary <int, int> noResultsNeededForAggregation_new = new Dictionary <int, int>(); // play back by time foreach (DateTime t in entriesBySubmitTime.Keys) { //Console.WriteLine("Processing Results of time: {0}", t); foreach (SatyamResultsTableEntry entry in entriesBySubmitTime[t]) { SatyamResult satyamResult = JSonUtils.ConvertJSonToObject <SatyamResult>(entry.ResultString); SatyamTask task = JSonUtils.ConvertJSonToObject <SatyamTask>(satyamResult.TaskParametersString); SatyamJob job = task.jobEntry; string fileName = URIUtilities.filenameFromURI(task.SatyamURI); int taskEntryID = entry.SatyamTaskTableEntryID; if (aggregatedTasks.Contains(taskEntryID)) { continue; } if (!ResultsPerTask.ContainsKey(taskEntryID)) { ResultsPerTask.Add(taskEntryID, new List <SatyamResultsTableEntry>()); } ResultsPerTask[taskEntryID].Add(entry); // check log if enough results are collected if (noResultsNeededForAggregation != null && noResultsNeededForAggregation.ContainsKey(taskEntryID) && ResultsPerTask[taskEntryID].Count < noResultsNeededForAggregation[taskEntryID]) { continue; } string aggResultString = ObjectCountingAggregator.GetAggregatedResultString(ResultsPerTask[taskEntryID], MinResults, MaxResults, MAX_ABSOLUTE_COUNT_DEVIATION_LOWERBOUND, MAX_DEVIATION_FRACTION, SUPER_MAJORITY_VALUE); if (aggResultString == null) { continue; } SatyamAggregatedResultsTableEntry aggEntry = new SatyamAggregatedResultsTableEntry(); aggEntry.JobGUID = ResultsPerTask[taskEntryID][0].JobGUID; aggEntry.JobTemplateType = ResultsPerTask[taskEntryID][0].JobTemplateType; aggEntry.SatyamTaskTableEntryID = taskEntryID; aggEntry.UserID = ResultsPerTask[taskEntryID][0].UserID; aggEntry.ResultString = aggResultString; /// aggregation happen /// // record logs // record logs if (noResultsNeededForAggregation == null || !noResultsNeededForAggregation.ContainsKey(taskEntryID)) { noResultsNeededForAggregation_new.Add(taskEntryID, ResultsPerTask[taskEntryID].Count); } aggEntries.Add(aggEntry); noTotalConverged++; if (ResultsPerTask[taskEntryID].Count >= MaxResults) { noTerminatedTasks++; } aggregatedTasks.Add(taskEntryID); } } SatyamResultsAnalysis.RecordAggregationLog(noResultsNeededForAggregation_new, configString, guid); double totalError, totalGroundtruth; //ValidateSatyamKITTIObjectCountingAggregationResult(aggEntries, out totalError, out totalGroundtruth, // MinHeight, MaxOcclusion, Max_Truncation); ValidateSatyamCARPKObjectCountingAggregationResult(aggEntries, out totalError, out totalGroundtruth); Console.WriteLine("noTotalConverged {0}", noTotalConverged); Console.WriteLine("noTerminatedTasks {0}", noTerminatedTasks); /// local file string outputString = String.Format("{0} {1} {2} {3} {4} {5} {6}\n", configString, totalError, totalGroundtruth, totalError / totalGroundtruth, noTotalConverged, noTerminatedTasks, ResultsPerTask.Count - noTotalConverged); string outputfile = DirectoryConstants.defaultTempDirectory + guid + "\\resultSummary.txt"; File.AppendAllText(outputfile, outputString); string approvalString = configString + "_PayDev_" + TaskConstants.OBJECT_COUNTING_MTURK_MAX_ABSOLUTE_COUNT_DEVIATION_FOR_PAYMENT + "_PayFrac_" + TaskConstants.OBJECT_COUNTING_MTURK_MAX_DEVIATION_FRACTION_FOR_PAYMENT; //for (double prob = 0; prob < 1;prob +=0.2) //{ // SatyamResultsAnalysis.AnalyzeApprovalRate(aggEntries, entries, guid, approvalString, anotherChanceProbablity: prob); //} //for (double ratio = 0; ratio < 1; ratio += 0.2) //{ // SatyamResultsAnalysis.AnalyzeApprovalRate(aggEntries, entriesBySubmitTime, noResultsNeededForAggregation, noResultsNeededForAggregation_new, guid, configString, approvalRatioThreshold: ratio); //} SatyamResultsAnalysis.AggregationAnalysis(aggEntries, entriesBySubmitTime, noResultsNeededForAggregation, noResultsNeededForAggregation_new, guid, configString); }
public static void ValidateSameTaskSameWorkerMultipleResultsEffect(string jobGUID, string confusingImageListFilePath = null) { SatyamResultsTableAccess resultsDB = new SatyamResultsTableAccess(); List <SatyamResultsTableEntry> entries = resultsDB.getEntriesByGUIDOrderByID(jobGUID); resultsDB.close(); Dictionary <int, int> noResultsPerTask = new Dictionary <int, int>(); Dictionary <int, int> noAcceptedResultsPerTask = new Dictionary <int, int>(); Dictionary <int, int> noPaidResultsPerTask = new Dictionary <int, int>(); Dictionary <int, double> moneyPaidPerTask = new Dictionary <int, double>(); SortedDictionary <int, Dictionary <string, List <string> > > ResultsPerWorkerPerTask = new SortedDictionary <int, Dictionary <string, List <string> > >(); SortedDictionary <int, Dictionary <string, int> > noDifferentDuplicateResultsPerWorkerPerTask = new SortedDictionary <int, Dictionary <string, int> >(); Dictionary <int, string> taskSatyamUri = new Dictionary <int, string>(); Dictionary <int, double> timeTakenTillAggregationPerTask = new Dictionary <int, double>(); Dictionary <int, DateTime> finalTaskEndTime = new Dictionary <int, DateTime>(); Dictionary <string, double> finalHITEndTime = new Dictionary <string, double>(); int totalResults = 0; int noTasksWithDuplicateResultsFromSameWorker = 0; int noTasksWithMixedResultsFromSameWorker = 0; int noTaskWithDuplicateResultsAsMajority = 0; int noTaskWhoseDuplicateResultsChangedAggregation = 0; int noTaskWhoseDuplicateResultsChangedAggregationIncorrectly = 0; int noCorrectDecisionsAmongDuplicates = 0; //noCorrectDuplicateResultsOfSameWorkerSameTask = 0; int noTasksWithCorrectDuplicateResultsOfSameWorkerSameTask = 0; int noWorkerSwitchedToCorrect = 0; int noWorkerSwitchedToCorrectAndMaintained = 0; int noTasksWithWorkerSwitchedToCorrect = 0; int noTasksWithWorkerSwitchedToCorrectAndMaintained = 0; List <double> timeTakenPerResult = new List <double>(); List <double> acceptedTimeTakenPerResult = new List <double>(); List <int> SequenceNumberOfResultPerTask = new List <int>(); List <int> SequenceNumberOfResultPerWorker = new List <int>(); string JobGUID = entries[0].JobGUID; Dictionary <string, int> noJobsPerWorker = new Dictionary <string, int>(); Dictionary <string, int> noAcceptedJobsPerWorker = new Dictionary <string, int>(); Dictionary <string, int> noPaidJobsPerWorker = new Dictionary <string, int>(); List <double> resultArrivalTimes = new List <double>(); totalResults = entries.Count; ///imagenet only Dictionary <string, ConfusingReason> imageBlackListReason = new Dictionary <string, ConfusingReason>(); if (confusingImageListFilePath != null) { imageBlackListReason = getConfusingImageList(confusingImageListFilePath); } for (int i = 0; i < entries.Count; i++) { SatyamResultsTableEntry entry = entries[i]; SatyamResult satyamResult = JSonUtils.ConvertJSonToObject <SatyamResult>(entry.ResultString); SatyamTask task = JSonUtils.ConvertJSonToObject <SatyamTask>(satyamResult.TaskParametersString); SatyamJob job = task.jobEntry; if (IsBlackListed(task.SatyamURI, imageBlackListReason)) { continue; } int taskEntryID = entry.SatyamTaskTableEntryID; if (!noResultsPerTask.ContainsKey(taskEntryID)) { noResultsPerTask.Add(taskEntryID, 0); noAcceptedResultsPerTask.Add(taskEntryID, 0); noPaidResultsPerTask.Add(taskEntryID, 0); moneyPaidPerTask.Add(taskEntryID, 0); finalTaskEndTime.Add(taskEntryID, entry.SubmitTime); ResultsPerWorkerPerTask.Add(taskEntryID, new Dictionary <string, List <string> >()); noDifferentDuplicateResultsPerWorkerPerTask.Add(taskEntryID, new Dictionary <string, int>()); taskSatyamUri.Add(taskEntryID, task.SatyamURI); } if (!ResultsPerWorkerPerTask[taskEntryID].ContainsKey(satyamResult.amazonInfo.WorkerID)) { ResultsPerWorkerPerTask[taskEntryID].Add(satyamResult.amazonInfo.WorkerID, new List <string>()); noDifferentDuplicateResultsPerWorkerPerTask[taskEntryID].Add(satyamResult.amazonInfo.WorkerID, 0); } if (!ResultsPerWorkerPerTask[taskEntryID][satyamResult.amazonInfo.WorkerID].Contains(satyamResult.TaskResult)) { noDifferentDuplicateResultsPerWorkerPerTask[taskEntryID][satyamResult.amazonInfo.WorkerID]++; } ResultsPerWorkerPerTask[taskEntryID][satyamResult.amazonInfo.WorkerID].Add(satyamResult.TaskResult); } ///////////////////////////////////// Per Task Analysis ////////////////////////////////////// SortedDictionary <int, int> resultsPerTaskHistogram = new SortedDictionary <int, int>(); SortedDictionary <int, int> resultsAcceptedPerTaskHistogram = new SortedDictionary <int, int>(); SortedDictionary <int, int> resultsPaidPerTaskHistogram = new SortedDictionary <int, int>(); SortedDictionary <int, int> moneyPaidPerTaskHistogram = new SortedDictionary <int, int>(); //cents SortedDictionary <int, int> ResultsPerWorkerPerTaskHistogram = new SortedDictionary <int, int>(); List <int> taskIDs = noResultsPerTask.Keys.ToList(); foreach (int taskID in taskIDs) { if (ResultsPerWorkerPerTask[taskID].Count != noResultsPerTask[taskID]) { //has multiple results from same turker noTasksWithDuplicateResultsFromSameWorker++; // the aggregation result List <SingleObjectLabelingResult> allResultsPerTask = new List <SingleObjectLabelingResult>(); SingleObjectLabelingAggregatedResult aggregatedResultPerTask = new SingleObjectLabelingAggregatedResult(); // the aggregation if without duplicate results List <SingleObjectLabelingResult> OnlyFirstResultOfEachTurkerPerTask = new List <SingleObjectLabelingResult>(); SingleObjectLabelingAggregatedResult aggregatedOnlyFirstResultOfTurkerPerTask = new SingleObjectLabelingAggregatedResult(); foreach (List <string> ResultsStringsPerWorkerPerTask in ResultsPerWorkerPerTask[taskID].Values) { foreach (string resultString in ResultsStringsPerWorkerPerTask) { allResultsPerTask.Add(JSonUtils.ConvertJSonToObject <SingleObjectLabelingResult>(resultString)); } OnlyFirstResultOfEachTurkerPerTask.Add(JSonUtils.ConvertJSonToObject <SingleObjectLabelingResult>(ResultsStringsPerWorkerPerTask[0])); } aggregatedResultPerTask = SingleObjectLabelingAggregator.getAggregatedResult(allResultsPerTask); aggregatedOnlyFirstResultOfTurkerPerTask = SingleObjectLabelingAggregator.getAggregatedResult(OnlyFirstResultOfEachTurkerPerTask); if (aggregatedResultPerTask != null) { if (aggregatedOnlyFirstResultOfTurkerPerTask == null || aggregatedResultPerTask.Category != aggregatedOnlyFirstResultOfTurkerPerTask.Category) { noTaskWhoseDuplicateResultsChangedAggregation++; if (AggregatedResultEqualsGroundTruth(taskSatyamUri[taskID], JSonUtils.ConvertObjectToJSon(aggregatedResultPerTask))) { noTaskWhoseDuplicateResultsChangedAggregationIncorrectly++; } } } bool hasMixedResults = false; bool hasCorrectResult = false; bool atLeastOneShiftedFromIncorrectToCorrect = false; bool atLeastOneShiftedFromIncorrectToCorrectAndMaintained = false; foreach (List <string> ResultsStringsPerWorkerPerTask in ResultsPerWorkerPerTask[taskID].Values) { if (!ResultsPerWorkerPerTaskHistogram.ContainsKey(ResultsStringsPerWorkerPerTask.Count)) { ResultsPerWorkerPerTaskHistogram.Add(ResultsStringsPerWorkerPerTask.Count, 0); } ResultsPerWorkerPerTaskHistogram[ResultsStringsPerWorkerPerTask.Count]++; double superMajority = 0.6; // tunable if (((double)ResultsStringsPerWorkerPerTask.Count + 1) / ((double)noResultsPerTask[taskID] + 2) > superMajority) { noTaskWithDuplicateResultsAsMajority++; } ///////////// imagenet only ////////////////// if (ResultsStringsPerWorkerPerTask.Distinct().Count() > 1) { // multiple choices given hasMixedResults = true; bool incorrect = false; bool switchToCorrect = false; foreach (string resultString in ResultsStringsPerWorkerPerTask) { if (SingleObjectLabelingResultEqualsGroundTruth(taskSatyamUri[taskID], resultString)) { hasCorrectResult = true; noCorrectDecisionsAmongDuplicates++; if (incorrect) { switchToCorrect = true; } incorrect = false; } else { incorrect = true; } } if (switchToCorrect) { // been incorrect and swithed to correct noWorkerSwitchedToCorrect++; atLeastOneShiftedFromIncorrectToCorrect = true; if (!incorrect) { //switch to correct and maintain till the end noWorkerSwitchedToCorrectAndMaintained++; atLeastOneShiftedFromIncorrectToCorrectAndMaintained = true; } } } //////////////////////////////////////////////// } if (hasMixedResults) { noTasksWithMixedResultsFromSameWorker++; } if (hasCorrectResult) { noTasksWithCorrectDuplicateResultsOfSameWorkerSameTask++; } if (atLeastOneShiftedFromIncorrectToCorrect) { noTasksWithWorkerSwitchedToCorrect++; } if (atLeastOneShiftedFromIncorrectToCorrectAndMaintained) { noTasksWithWorkerSwitchedToCorrectAndMaintained++; } } } Console.WriteLine("DuplicateResultsHistogram"); foreach (int no in ResultsPerWorkerPerTaskHistogram.Keys) { Console.WriteLine("{0}, {1}", no, ResultsPerWorkerPerTaskHistogram[no]); } Console.WriteLine( //"{0} images (in total {1}({2} of which are correct)) has duplicate results from same turker, \n" + //"\t{3} images have >=1 workers making more than majority number of results.\n" + //"\t{11} images aggregation is changed ({12} of which incorrectly) by duplicate results from same turker.\n" + //"\t{4} images(In total {5} times) with duplicate results include mixed(>= 2) decisions from same worker.\n" + "\t\t{6} images has correct decision among duplicate decisions.\n" + "\t\t{7} images({8} times) a worker has switched from incorrect to correct choice\n" + "\t\t\t{9} images({10} times) of which maintained the correct choice till the last time of their job on that image.", //noTasksWithDuplicateResultsFromSameWorker, noDuplicateResultsFromSameWorker, noCorrectDecisionsAmongDuplicates, //noTaskWithDuplicateResultsAsMajority, //noTasksWithMixedResultsFromSameWorker, noDecisionChangeAmongDuplicateResultsOfSameWorkerSameTask, //noTasksWithCorrectDuplicateResultsOfSameWorkerSameTask, noTasksWithWorkerSwitchedToCorrect, noWorkerSwitchedToCorrect, noTasksWithWorkerSwitchedToCorrectAndMaintained, noWorkerSwitchedToCorrectAndMaintained, noTaskWhoseDuplicateResultsChangedAggregation, noTaskWhoseDuplicateResultsChangedAggregationIncorrectly ); }
protected void SubmitButton_Click(object sender, EventArgs e) { DateTime SubmitTime = DateTime.Now; DateTime PageLoadTime = Convert.ToDateTime(Hidden_PageLoadTime.Value); int count; bool isValidCount = int.TryParse(CountTextBox.Text, out count); if (!isValidCount) { ErrorLabel.Text = "Error : Ivalid Count."; ErrorLabel.ForeColor = System.Drawing.Color.Red; ErrorLabel.Font.Bold = true; } else if (count < 0) { ErrorLabel.Text = "Error : Count cannot be less than zero."; ErrorLabel.ForeColor = System.Drawing.Color.Red; ErrorLabel.Font.Bold = true; } else { ErrorLabel.Text = ""; CountTextBox.Text = ""; SatyamTaskTableEntry taskEntry = JSonUtils.ConvertJSonToObject <SatyamTaskTableEntry>(Hidden_TaskEntryString.Value); SatyamResult result = new SatyamResult(); result.TaskParametersString = taskEntry.TaskParametersString; result.TaskStartTime = PageLoadTime; result.TaskEndTime = SubmitTime; result.TaskTableEntryID = taskEntry.ID; AmazonTaskResultInfo amazonInfo = new AmazonTaskResultInfo(); amazonInfo.AssignmentID = Hidden_AmazonAssignmentID.Value; amazonInfo.WorkerID = Hidden_AmazonWorkerID.Value; amazonInfo.HITID = Hidden_HITID.Value; amazonInfo.PricePerHIT = Convert.ToDouble(Hidden_Price.Value); result.amazonInfo = amazonInfo; ObjectCountingResult sresult = new ObjectCountingResult(); sresult.Count = count; string sresultString = JSonUtils.ConvertObjectToJSon <ObjectCountingResult>(sresult); result.TaskResult = sresultString; string resultString = JSonUtils.ConvertObjectToJSon <SatyamResult>(result); SatyamResultsTableAccess resultdb = new SatyamResultsTableAccess(); resultdb.AddEntry(taskEntry.JobTemplateType, taskEntry.UserID, taskEntry.JobGUID, resultString, taskEntry.ID, PageLoadTime, SubmitTime); resultdb.close(); SatyamTaskTableManagement.UpdateResultNumber(taskEntry.ID); //SatyamTaskTableAccess taskDB = new SatyamTaskTableAccess(); //taskDB.IncrementDoneScore(taskEntry.ID); int noDone = Convert.ToInt32(Hidden_NoImagesDone.Value); noDone++; Hidden_NoImagesDone.Value = noDone.ToString(); bool NotDone = false; int noTasksPerJob = Convert.ToInt32(Hidden_TasksPerJob.Value); if (noDone < noTasksPerJob) { NotDone = getNewRandomJob(); } if (NotDone == false) { if (!Testing) { SatyamAmazonHITTableAccess HITdb = new SatyamAmazonHITTableAccess(); string HITID = result.amazonInfo.HITID; HITdb.UpdateStatusByHITID(HITID, HitStatus.submitted); HITdb.close(); AmazonMTurkNotification.submitAmazonTurkHit(result.amazonInfo.AssignmentID, result.amazonInfo.WorkerID, false); } Response.Redirect("MTurkTaskDonePage.aspx"); } } }
public static MultiObjectTrackingResult stitchAllChunksOfOneVideo(List <SatyamAggregatedResultsTableEntry> entries, out List <string> ImageURLs, out int fps) { ImageURLs = new List <string>(); fps = 0; if (entries.Count == 0) { return(null); } SatyamJobStorageAccountAccess satyamStorage = new SatyamJobStorageAccountAccess(); MultiObjectTrackingResult stitched = new MultiObjectTrackingResult(); int totalFrameCounts = 0; // ensure the order is correct SortedDictionary <int, SatyamAggregatedResultsTableEntry> sortedEntries = new SortedDictionary <int, SatyamAggregatedResultsTableEntry>(); List <int> idx = new List <int>(); for (int i = 0; i < entries.Count; i++) { SatyamAggregatedResultsTableEntry entry = entries[i]; SatyamAggregatedResult satyamAggResult = JSonUtils.ConvertJSonToObject <SatyamAggregatedResult>(entry.ResultString); SatyamTask aggTask = JSonUtils.ConvertJSonToObject <SatyamTask>(satyamAggResult.TaskParameters); string video = URIUtilities.localDirectoryNameFromURI(aggTask.SatyamURI); string[] fields = video.Split('_'); int startingFrame = Convert.ToInt32(fields[fields.Length - 1]); if (!sortedEntries.ContainsKey(startingFrame)) { sortedEntries.Add(startingFrame, entries[i]); idx.Add(startingFrame); } } idx.Sort(); for (int i = 0; i < idx.Count; i++) { //SatyamAggregatedResultsTableEntry entry = entries[i]; SatyamAggregatedResultsTableEntry entry = sortedEntries[idx[i]]; SatyamAggregatedResult satyamAggResult = JSonUtils.ConvertJSonToObject <SatyamAggregatedResult>(entry.ResultString); SatyamTask aggTask = JSonUtils.ConvertJSonToObject <SatyamTask>(satyamAggResult.TaskParameters); MultiObjectTrackingSubmittedJob job = JSonUtils.ConvertJSonToObject <MultiObjectTrackingSubmittedJob>(aggTask.jobEntry.JobParameters); MultiObjectTrackingAggregatedResult aggresult = JSonUtils.ConvertJSonToObject <MultiObjectTrackingAggregatedResult>(satyamAggResult.AggregatedResultString); int noFramesOverlap = 0; if (job.ChunkOverlap != 0.0) { noFramesOverlap = (int)(job.ChunkOverlap * job.FrameRate); } List <string> TraceURLs = satyamStorage.getURLListOfSubDirectoryByURL(aggTask.SatyamURI); if (i == 0) { ImageURLs.AddRange(TraceURLs); string[] names = aggTask.SatyamURI.Split('/'); fps = job.FrameRate; stitched = aggresult.tracklets; totalFrameCounts += TraceURLs.Count; } else { int noNewFrames = 0; for (int j = noFramesOverlap; j < TraceURLs.Count; j++) { ImageURLs.Add(TraceURLs[j]); noNewFrames++; } //stitched = stitchTwoTracesByOneFrameBoundingBoxes(stitched, aggresult.tracklets, totalFrameCounts, totalFrameCounts+ noNewFrames, noFramesOverlap,fps); stitched = stitchTwoTracesByTubeletsOfOverlappingVideoChunk(stitched, aggresult.tracklets, totalFrameCounts, totalFrameCounts + noNewFrames, noFramesOverlap, fps); totalFrameCounts += noNewFrames; } //debug //generateVideoForEvaluation(ImageURLs, stitched, directoryName + "_" + i, videoName, fps); } return(stitched); }