public static Image DrawImageDetectionResult(MultiObjectLocalizationAndLabelingResult res, string image_url, List <string> Categories) { Image originalImage = ImageUtilities.getImageFromURI(image_url); List <Rectangle> rectangles = new List <Rectangle>(); List <Color> colors = new List <Color>(); List <string> ids = new List <string>(); List <bool> dashed = new List <bool>(); for (int j = 0; j < res.objects.Count; j++) { MultiObjectLocalizationAndLabelingResultSingleEntry box = res.objects[j]; int x = box.boundingBox.tlx; int y = box.boundingBox.tly; int width = box.boundingBox.brx - box.boundingBox.tlx; int height = box.boundingBox.bry - box.boundingBox.tly; Rectangle r = new Rectangle(x, y, width, height); rectangles.Add(r); string category = box.Category; int colorIndex = Categories.IndexOf(category); colors.Add(DrawingBoxesAndLinesOnImages.Colors[colorIndex]); string id = j + "-" + category; ids.Add(id); dashed.Add(false); } Image imageWithBoxes = DrawingBoxesAndLinesOnImages.addRectanglesToImage(originalImage, rectangles, colors, ids, dashed); return(imageWithBoxes); }
public static MultiObjectLocalizationAndLabelingResult GetImageDetectionResult(string image_url, List <string> Categories) { MultiObjectLocalizationAndLabelingResult ret = new MultiObjectLocalizationAndLabelingResult(); int height = 0; int width = 0; PredictResponse res = ImageDetectionRequest(image_url, out height, out width); //float score_threshold = int class_counts = res.Outputs["detection_classes"].FloatVal.Count; float[] classes = new float[class_counts]; res.Outputs["detection_classes"].FloatVal.CopyTo(classes, 0); int box_counts = res.Outputs["detection_boxes"].FloatVal.Count; float[] boxes = new float[box_counts]; res.Outputs["detection_boxes"].FloatVal.CopyTo(boxes, 0); int score_counts = res.Outputs["detection_scores"].FloatVal.Count; float[] scores = new float[score_counts]; res.Outputs["detection_scores"].FloatVal.CopyTo(scores, 0); double score_threshold = 0.5; for (int i = 0; i < score_counts; i++) { float s = scores[i]; if (s == 0) { break; } if (s < score_threshold) { continue; } int c = (int)classes[i] - 1; // comes in as ymin, xmin, ymax, xmax int tlx = (int)(boxes[4 * i + 1] * width); int tly = (int)(boxes[4 * i] * height); int brx = (int)(boxes[4 * i + 3] * width); int bry = (int)(boxes[4 * i + 2] * height); MultiObjectLocalizationAndLabelingResultSingleEntry e = new MultiObjectLocalizationAndLabelingResultSingleEntry(); e.boundingBox = new BoundingBox(tlx, tly, brx, bry); e.Category = Categories[c]; ret.objects.Add(e); } ret.imageHeight = height; ret.imageWidth = width; return(ret); }
private MultiObjectLocalizationAndLabelingResult LoadTFServingResult(SatyamTaskTableEntry entry) { SatyamTask task = JSonUtils.ConvertJSonToObject <SatyamTask>(entry.TaskParametersString); string image_url = task.SatyamURI; SatyamJob jobDefinitionEntry = task.jobEntry; MultiObjectLocalizationAndLabelingSubmittedJob job = JSonUtils.ConvertJSonToObject <MultiObjectLocalizationAndLabelingSubmittedJob>(jobDefinitionEntry.JobParameters); List <string> categories = job.Categories; MultiObjectLocalizationAndLabelingResult res = TensorflowServingClient.GetImageDetectionResult(image_url, categories); return(res); }
public static void TestDetectionRequest() { string image_url = "https://satyamresearchjobstorage.blob.core.windows.net/longdurationblob/SeattleLive-5-Westlake-NS/SeattleLive-5-Westlake-NS_2017-09-28-15-20-19-000_2017-09-28-15-20-22-000/SeattleLive-5-Westlake-NS-000001.jpg"; //TFServingClient.ImageDetectionRequest(image_url); List <string> categories = new List <string>(); categories.Add("Car"); categories.Add("Ped"); MultiObjectLocalizationAndLabelingResult res = TensorflowServingClient.GetImageDetectionResult(image_url, categories); Image im = MultiObjectLabelingAndLocalizationAnalysis.DrawImageDetectionResult(res, image_url, categories); ImageUtilities.saveImage(im, Constants.DirectoryConstants.DefaultResultDirectory, "test"); }
private MultiObjectLocalizationAndLabelingResult LoadLatestProgressiveAggregationResult(SatyamTaskTableEntry entry) { SatyamAggregatedProgressiveResultsTableAccess aggDB = new SatyamAggregatedProgressiveResultsTableAccess(); SatyamAggregatedProgressiveResultsTableEntry aggEntry = aggDB.getLatestEntryWithMostResultsAggregatedByTaskID(entry.ID); if (aggEntry == null) { return(null); } Hidden_PrevResultID.Value = aggEntry.ID.ToString(); SatyamAggregatedResult satyamResult = JSonUtils.ConvertJSonToObject <SatyamAggregatedResult>(aggEntry.ResultString); MultiObjectLocalizationAndLabelingAggregatedResult aggRes = JSonUtils.ConvertJSonToObject <MultiObjectLocalizationAndLabelingAggregatedResult>(satyamResult.AggregatedResultString); MultiObjectLocalizationAndLabelingResult res = aggRes.boxesAndCategories; return(res); }
public static void testJSONString() { MultiObjectLocalizationAndLabelingResult res = new MultiObjectLocalizationAndLabelingResult(); res.objects = new List <MultiObjectLocalizationAndLabelingResultSingleEntry>(); MultiObjectLocalizationAndLabelingResultSingleEntry entry = new MultiObjectLocalizationAndLabelingResultSingleEntry(); entry.boundingBox = new BoundingBox(5, 15, 25, 35); entry.Category = "Car"; res.objects.Add(entry); entry = new MultiObjectLocalizationAndLabelingResultSingleEntry(); entry.boundingBox = new BoundingBox(45, 55, 65, 75); entry.Category = "Bus"; res.objects.Add(entry); string jsonString = JSonUtils.ConvertObjectToJSon <MultiObjectLocalizationAndLabelingResult>(res); }
private MultiObjectLocalizationAndLabelingResult LoadLatestTurkerResult(SatyamTaskTableEntry entry) { SatyamResultsTableAccess resultsDB = new SatyamResultsTableAccess(); List <SatyamResultsTableEntry> entries = resultsDB.getEntriesByGUIDAndTaskID(entry.JobGUID, entry.ID); if (entries.Count == 0) { return(null); } //organized sequentially SatyamResultsTableEntry prevResult = entries[entries.Count - 1]; Hidden_PrevResultID.Value = prevResult.ID.ToString(); SatyamResult satyamResult = JSonUtils.ConvertJSonToObject <SatyamResult>(prevResult.ResultString); MultiObjectLocalizationAndLabelingResult res = JSonUtils.ConvertJSonToObject <MultiObjectLocalizationAndLabelingResult>(satyamResult.TaskResult); return(res); }
public static MultiObjectLabelingAndLocalizationAnalysisPerJob Analyse(List <SatyamResultsTableEntry> entries) { MultiObjectLabelingAndLocalizationAnalysisPerJob ana = new MultiObjectLabelingAndLocalizationAnalysisPerJob(); ana.jobTemplateType = entries[0].JobTemplateType; for (int i = 0; i < entries.Count; i++) { SatyamResultsTableEntry entry = entries[i]; SatyamResult satyamResult = JSonUtils.ConvertJSonToObject <SatyamResult>(entry.ResultString); SatyamTask task = JSonUtils.ConvertJSonToObject <SatyamTask>(satyamResult.TaskParametersString); SatyamJob job = task.jobEntry; MultiObjectLocalizationAndLabelingResult res = JSonUtils.ConvertJSonToObject <MultiObjectLocalizationAndLabelingResult>(satyamResult.TaskResult); } return(ana); }
private bool getNewRandomJob() { double price = 0; bool success = Double.TryParse(Hidden_Price.Value, out price); if (!success) { price = 0; } SatyamTaskTableAccess taskTableDB = new SatyamTaskTableAccess(); SatyamTaskTableEntry entry = null; //Thread.Sleep(5000); int maxDoneScore = TaskConstants.MULTI_OBJECT_LOCALIZATION_AND_LABLING_MAX_DONE_SCORE; if (SubmitButton.Enabled == true) { //entry = taskTableDB.getMinimumTriedNewEntryForWorkerIDByTemplateAndPrice(Hidden_AmazonWorkerID.Value, // TaskConstants.Detection_Image_MTurk, price); entry = taskTableDB.getMinimumTriedNewEntryForWorkerIDByTemplateAndPriceAndMaxDoneScore(Hidden_AmazonWorkerID.Value, TaskConstants.Detection_Image_MTurk, price, MaxDoneScore: maxDoneScore); } else { //entry = taskTableDB.getMinimumTriedEntryByTemplate(TaskConstants.Detection_Image_MTurk); entry = taskTableDB.getMinimumTriedEntryByTemplateAndMaxDoneScore(TaskConstants.Detection_Image_MTurk, MaxDoneScore: maxDoneScore); } if (entry == null) { taskTableDB.close(); return(false); } //doneScore = entry.DoneScore; //taskTableDB.IncrementDoneScore(entry.ID); taskTableDB.close(); SatyamTask task = JSonUtils.ConvertJSonToObject <SatyamTask>(entry.TaskParametersString); string uri = task.SatyamURI; DisplayImage.ImageUrl = uri; SatyamJob jobDefinitionEntry = task.jobEntry; MultiObjectLocalizationAndLabelingSubmittedJob job = JSonUtils.ConvertJSonToObject <MultiObjectLocalizationAndLabelingSubmittedJob>(jobDefinitionEntry.JobParameters); List <string> categories = job.Categories; CategorySelection_RadioButtonList.Items.Clear(); for (int i = 0; i < categories.Count; i++) { ListItem l = new ListItem(categories[i]); CategorySelection_RadioButtonList.Items.Add(l); } if (job.Description != "") { DescriptionPanel.Visible = true; DescriptionTextPanel.Controls.Add(new LiteralControl(job.Description)); } Hidden_BoundaryLines.Value = JSonUtils.ConvertObjectToJSon(job.BoundaryLines); Hidden_TaskEntryString.Value = JSonUtils.ConvertObjectToJSon <SatyamTaskTableEntry>(entry); Hidden_PageLoadTime.Value = DateTime.Now.ToString(); NoLabeled.Text = Hidden_NoImagesDone.Value; Hidden_TasksPerJob.Value = jobDefinitionEntry.TasksPerJob.ToString(); /////////////////////////////////Load Previous Turker Results //MultiObjectLocalizationAndLabelingResult res = LoadLatestTurkerResult(entry); ///////////////////////////////Load Previous Aggregation Results MultiObjectLocalizationAndLabelingResult res = LoadLatestProgressiveAggregationResult(entry); //TFServing Backend if (res == null && TFServingBackend) { res = LoadTFServingResult(entry); Hidden_PrevResultID.Value = "-1"; // means TF } if (res == null) { Hidden_PrevResults.Value = "[]"; Hidden_PrevResultID.Value = 0.ToString(); } else { string prevBoxes = JSonUtils.ConvertObjectToJSon(res.objects); Hidden_ImageHeight.Value = res.imageHeight.ToString(); Hidden_ImageWidth.Value = res.imageWidth.ToString(); Hidden_PrevResults.Value = prevBoxes; } return(true); }
public static double ACCEPTANCE_NUMBER_OF_BOXES_THRESHOLD = TaskConstants.MULTI_OBJECT_LOCALIZATION_AND_LABLING_MTURK_OBJECT_COVERAGE_THRESHOLD_FOR_PAYMENT; //the person must have made at least 80% of the boxes public static bool IsAcceptable(SatyamAggregatedResultsTableEntry aggResultEntry, SatyamResultsTableEntry resultEntry, double DeviationPixelThreshold = TaskConstants.MULTI_OBJECT_LOCALIZATION_AND_LABLING_MTURK_DEVIATION_THRESHOLD_FOR_PAYMENT) { //most boxes should be within limits //most categories should be right SatyamAggregatedResult satyamAggResult = JSonUtils.ConvertJSonToObject <SatyamAggregatedResult>(aggResultEntry.ResultString); MultiObjectLocalizationAndLabelingAggregatedResult aggresult = JSonUtils.ConvertJSonToObject <MultiObjectLocalizationAndLabelingAggregatedResult>(satyamAggResult.AggregatedResultString); SatyamResult satyamResult = JSonUtils.ConvertJSonToObject <SatyamResult>(resultEntry.ResultString); MultiObjectLocalizationAndLabelingResult result = JSonUtils.ConvertJSonToObject <MultiObjectLocalizationAndLabelingResult>(satyamResult.TaskResult); if (result == null) { return(false); } //first check if the number of boxes are within limit int boxLimit = (int)Math.Ceiling((double)aggresult.boxesAndCategories.objects.Count * ACCEPTANCE_NUMBER_OF_BOXES_THRESHOLD); if (result.objects.Count < boxLimit) { return(false); } double minboxdimension_threshold_x = MIN_BOX_DIMENSION_FOR_CONSIDERATION * result.displayScaleReductionX; double minboxdimension_threshold_y = MIN_BOX_DIMENSION_FOR_CONSIDERATION * result.displayScaleReductionY; //We fist do a bipartitte matching to find the best assocaition for the boxes List <List <BoundingBox> > allboxes = new List <List <BoundingBox> >(); allboxes.Add(new List <BoundingBox>()); foreach (MultiObjectLocalizationAndLabelingResultSingleEntry entry in result.objects) { allboxes[0].Add(entry.boundingBox); } allboxes.Add(new List <BoundingBox>()); List <bool> tooSmallToIgnore = new List <bool>(); foreach (MultiObjectLocalizationAndLabelingResultSingleEntry entry in aggresult.boxesAndCategories.objects) { allboxes[1].Add(entry.boundingBox); if (entry.boundingBox.getWidth() < minboxdimension_threshold_x && entry.boundingBox.getHeight() < minboxdimension_threshold_y) { tooSmallToIgnore.Add(true); } else { tooSmallToIgnore.Add(false); } } List <MultipartiteWeightedMatch> boxAssociation = BoundingBoxAssociation.computeBoundingBoxAssociations(allboxes); //now find how many of the results match aggregated results int noAccepted = 0; double deviation_threshold_x = DeviationPixelThreshold * result.displayScaleReductionX; double deviation_threshold_y = DeviationPixelThreshold * result.displayScaleReductionY; int noIgnorable = 0; foreach (MultipartiteWeightedMatch match in boxAssociation) { if (match.elementList.ContainsKey(1)) // this contains an aggregated box { if (match.elementList.ContainsKey(0)) // a result box has been associated { BoundingBox aggregatedBoundingBox = allboxes[1][match.elementList[1]]; BoundingBox resultBoundingBox = allboxes[0][match.elementList[0]]; //double deviation = aggregatedBoundingBox.ComputeMaxDeviationMetric(resultBoundingBox); double deviation = aggregatedBoundingBox.ComputeNormalizedMaxDeviationMetric(resultBoundingBox, deviation_threshold_x, deviation_threshold_y); if (deviation <= 1) //deviation test passed { //now check category if (result.objects[match.elementList[0]].Category == aggresult.boxesAndCategories.objects[match.elementList[1]].Category) { //both category and bounding box tests have passed noAccepted++; } } else { if (tooSmallToIgnore[match.elementList[1]]) { noIgnorable++; } } } } } if (noAccepted >= boxLimit - noIgnorable) { return(true); } return(false); }
public static string GetAggregatedResultString(List <SatyamResultsTableEntry> results) { if (results.Count == 0) { return(null); } List <MultiObjectLocalizationAndLabelingResult> resultList = new List <MultiObjectLocalizationAndLabelingResult>(); List <string> WorkersPerTask = new List <string>(); foreach (SatyamResultsTableEntry entry in results) { SatyamResult res = JSonUtils.ConvertJSonToObject <SatyamResult>(entry.ResultString); // remove duplicate workers result // "" works for internal test!!! string workerID = res.amazonInfo.WorkerID; if (workerID != "" && WorkersPerTask.Contains(workerID)) { continue; } //enclose only non-duplicate results, one per each worker. WorkersPerTask.Add(workerID); MultiObjectLocalizationAndLabelingResult taskr = JSonUtils.ConvertJSonToObject <MultiObjectLocalizationAndLabelingResult>(res.TaskResult); resultList.Add(taskr); } /// Store Progressive results in progressive table, for consensus loading on turker task pages. // check if there are any new valid results first SatyamAggregatedProgressiveResultsTableAccess progDBtemp = new SatyamAggregatedProgressiveResultsTableAccess(); int LatestResultsAggregated = progDBtemp.getLatestNoResultsAggregatedByTaskID(results[0].SatyamTaskTableEntryID);// default value is -1 progDBtemp.close(); if (LatestResultsAggregated >= resultList.Count) { return(null); } /// safely return null here, since if it's aggregated by standard, it won't be executed again. /// it will be only executed when there is no standard aggregation result. /// so return null when no new results came in to even produce a progress aggregation result. int MaxCount = Math.Min(resultList.Count, TaskConstants.MULTI_OBJECT_LOCALIZATION_AND_LABLING_MTURK_MAX_RESULTS_TO_AGGREGATE); MultiObjectLocalizationAndLabelingAggregatedResult tempr = getAggregatedResult(resultList, MinResults: 2); if (tempr != null) { string rString = JSonUtils.ConvertObjectToJSon <MultiObjectLocalizationAndLabelingAggregatedResult>(tempr); SatyamAggregatedResult aggResult = new SatyamAggregatedResult(); aggResult.SatyamTaskTableEntryID = results[0].SatyamTaskTableEntryID; aggResult.AggregatedResultString = rString; SatyamResult res = JSonUtils.ConvertJSonToObject <SatyamResult>(results[0].ResultString); aggResult.TaskParameters = res.TaskParametersString; string resultString = JSonUtils.ConvertObjectToJSon <SatyamAggregatedResult>(aggResult); // create progressive entry SatyamAggregatedProgressiveResultsTableEntry aggProgEntry = new SatyamAggregatedProgressiveResultsTableEntry(); aggProgEntry.JobGUID = results[0].JobGUID; aggProgEntry.JobTemplateType = results[0].JobTemplateType; aggProgEntry.SatyamTaskTableEntryID = results[0].SatyamTaskTableEntryID; aggProgEntry.UserID = results[0].UserID; aggProgEntry.ResultString = resultString; aggProgEntry.ResultsAggregated = resultList.Count; // use the actual valid non-duplicate results number here. SatyamAggregatedProgressiveResultsTableAccess aggProgDB = new SatyamAggregatedProgressiveResultsTableAccess(); aggProgDB.AddEntry(aggProgEntry); aggProgDB.close(); } else { return(null); } MultiObjectLocalizationAndLabelingAggregatedResult r = getAggregatedResult(resultList); if (r != null) { string rString = JSonUtils.ConvertObjectToJSon <MultiObjectLocalizationAndLabelingAggregatedResult>(r); SatyamAggregatedResult aggResult = new SatyamAggregatedResult(); aggResult.SatyamTaskTableEntryID = results[0].SatyamTaskTableEntryID; aggResult.AggregatedResultString = rString; SatyamResult res = JSonUtils.ConvertJSonToObject <SatyamResult>(results[0].ResultString); aggResult.TaskParameters = res.TaskParametersString; string resultString = JSonUtils.ConvertObjectToJSon <SatyamAggregatedResult>(aggResult); return(resultString); } else { return(null); } }
public static void SaveResultImagesLocally(List <SatyamResultsTableEntry> entries, string directoryName) { if (!Directory.Exists(directoryName)) { Directory.CreateDirectory(directoryName); } directoryName = directoryName + "\\Raw"; if (!Directory.Exists(directoryName)) { Directory.CreateDirectory(directoryName); } for (int i = 0; i < entries.Count; i++) { SatyamResultsTableEntry entry = entries[i]; SatyamResult satyamResult = JSonUtils.ConvertJSonToObject <SatyamResult>(entry.ResultString); SatyamTask task = JSonUtils.ConvertJSonToObject <SatyamTask>(satyamResult.TaskParametersString); SatyamJob job = task.jobEntry; MultiObjectLocalizationAndLabelingResult res = JSonUtils.ConvertJSonToObject <MultiObjectLocalizationAndLabelingResult>(satyamResult.TaskResult); string ofilename = URIUtilities.filenameFromURI(task.SatyamURI); string[] fields = ofilename.Split('.'); string fileName = fields[0]; Image originalImage = ImageUtilities.getImageFromURI(task.SatyamURI); MultiObjectLocalizationAndLabelingSubmittedJob jobDefinition = JSonUtils.ConvertJSonToObject <MultiObjectLocalizationAndLabelingSubmittedJob>(job.JobParameters); Image imageWithBoundary = DrawingBoxesAndLinesOnImages.addLinesToImage(originalImage, jobDefinition.BoundaryLines, Color.Red, true); List <Rectangle> rectangles = new List <Rectangle>(); List <Color> colors = new List <Color>(); List <string> ids = new List <string>(); List <bool> dashed = new List <bool>(); for (int j = 0; j < res.objects.Count; j++) { MultiObjectLocalizationAndLabelingResultSingleEntry box = res.objects[j]; int x = box.boundingBox.tlx; int y = box.boundingBox.tly; int width = box.boundingBox.brx - box.boundingBox.tlx; int height = box.boundingBox.bry - box.boundingBox.tly; Rectangle r = new Rectangle(x, y, width, height); rectangles.Add(r); string category = box.Category; int colorIndex = jobDefinition.Categories.IndexOf(category); colors.Add(DrawingBoxesAndLinesOnImages.Colors[colorIndex]); string id = j + "-" + category; ids.Add(id); dashed.Add(false); } Image imageWithBoxesAndBoundary = DrawingBoxesAndLinesOnImages.addRectanglesToImage(imageWithBoundary, rectangles, colors, ids, dashed); fileName = fileName + "-Result"; if (satyamResult.amazonInfo.AssignmentID != "") { //fileName = fileName + "-" + satyamResult.amazonInfo.AssignmentID + "_" + entry.ID; fileName = fileName + "-" + entry.ID + "_" + satyamResult.PrevResultID; } ImageUtilities.saveImage(imageWithBoxesAndBoundary, directoryName, fileName); } }