public static Image DrawImageDetectionResult(MultiObjectLocalizationAndLabelingResult res, string image_url, List <string> Categories)
{
Image originalImage = ImageUtilities.getImageFromURI(image_url);
List <Rectangle> rectangles = new List <Rectangle>();
List <Color> colors = new List <Color>();
List <string> ids = new List <string>();
List <bool> dashed = new List <bool>();
for (int j = 0; j < res.objects.Count; j++)
{
MultiObjectLocalizationAndLabelingResultSingleEntry box = res.objects[j];
int x = box.boundingBox.tlx;
int y = box.boundingBox.tly;
int width = box.boundingBox.brx - box.boundingBox.tlx;
int height = box.boundingBox.bry - box.boundingBox.tly;
Rectangle r = new Rectangle(x, y, width, height);
rectangles.Add(r);
string category = box.Category;
int colorIndex = Categories.IndexOf(category);
colors.Add(DrawingBoxesAndLinesOnImages.Colors[colorIndex]);
string id = j + "-" + category;
ids.Add(id);
dashed.Add(false);
}
Image imageWithBoxes = DrawingBoxesAndLinesOnImages.addRectanglesToImage(originalImage, rectangles, colors, ids, dashed);
return(imageWithBoxes);
}
public static MultiObjectLocalizationAndLabelingResult GetImageDetectionResult(string image_url, List <string> Categories)
{
MultiObjectLocalizationAndLabelingResult ret = new MultiObjectLocalizationAndLabelingResult();
int height = 0;
int width = 0;
PredictResponse res = ImageDetectionRequest(image_url, out height, out width);
//float score_threshold =
int class_counts = res.Outputs["detection_classes"].FloatVal.Count;
float[] classes = new float[class_counts];
res.Outputs["detection_classes"].FloatVal.CopyTo(classes, 0);
int box_counts = res.Outputs["detection_boxes"].FloatVal.Count;
float[] boxes = new float[box_counts];
res.Outputs["detection_boxes"].FloatVal.CopyTo(boxes, 0);
int score_counts = res.Outputs["detection_scores"].FloatVal.Count;
float[] scores = new float[score_counts];
res.Outputs["detection_scores"].FloatVal.CopyTo(scores, 0);
double score_threshold = 0.5;
for (int i = 0; i < score_counts; i++)
{
float s = scores[i];
if (s == 0)
{
break;
}
if (s < score_threshold)
{
continue;
}
int c = (int)classes[i] - 1;
// comes in as ymin, xmin, ymax, xmax
int tlx = (int)(boxes[4 * i + 1] * width);
int tly = (int)(boxes[4 * i] * height);
int brx = (int)(boxes[4 * i + 3] * width);
int bry = (int)(boxes[4 * i + 2] * height);
MultiObjectLocalizationAndLabelingResultSingleEntry e = new MultiObjectLocalizationAndLabelingResultSingleEntry();
e.boundingBox = new BoundingBox(tlx, tly, brx, bry);
e.Category = Categories[c];
ret.objects.Add(e);
}
ret.imageHeight = height;
ret.imageWidth = width;
return(ret);
}
private MultiObjectLocalizationAndLabelingResult LoadTFServingResult(SatyamTaskTableEntry entry)
{
SatyamTask task = JSonUtils.ConvertJSonToObject <SatyamTask>(entry.TaskParametersString);
string image_url = task.SatyamURI;
SatyamJob jobDefinitionEntry = task.jobEntry;
MultiObjectLocalizationAndLabelingSubmittedJob job = JSonUtils.ConvertJSonToObject <MultiObjectLocalizationAndLabelingSubmittedJob>(jobDefinitionEntry.JobParameters);
List <string> categories = job.Categories;
MultiObjectLocalizationAndLabelingResult res = TensorflowServingClient.GetImageDetectionResult(image_url, categories);
return(res);
}
public static void TestDetectionRequest()
{
string image_url = "https://satyamresearchjobstorage.blob.core.windows.net/longdurationblob/SeattleLive-5-Westlake-NS/SeattleLive-5-Westlake-NS_2017-09-28-15-20-19-000_2017-09-28-15-20-22-000/SeattleLive-5-Westlake-NS-000001.jpg";
//TFServingClient.ImageDetectionRequest(image_url);
List <string> categories = new List <string>();
categories.Add("Car");
categories.Add("Ped");
MultiObjectLocalizationAndLabelingResult res = TensorflowServingClient.GetImageDetectionResult(image_url, categories);
Image im = MultiObjectLabelingAndLocalizationAnalysis.DrawImageDetectionResult(res, image_url, categories);
ImageUtilities.saveImage(im, Constants.DirectoryConstants.DefaultResultDirectory, "test");
}
private MultiObjectLocalizationAndLabelingResult LoadLatestProgressiveAggregationResult(SatyamTaskTableEntry entry)
{
SatyamAggregatedProgressiveResultsTableAccess aggDB = new SatyamAggregatedProgressiveResultsTableAccess();
SatyamAggregatedProgressiveResultsTableEntry aggEntry = aggDB.getLatestEntryWithMostResultsAggregatedByTaskID(entry.ID);
if (aggEntry == null)
{
return(null);
}
Hidden_PrevResultID.Value = aggEntry.ID.ToString();
SatyamAggregatedResult satyamResult = JSonUtils.ConvertJSonToObject <SatyamAggregatedResult>(aggEntry.ResultString);
MultiObjectLocalizationAndLabelingAggregatedResult aggRes = JSonUtils.ConvertJSonToObject <MultiObjectLocalizationAndLabelingAggregatedResult>(satyamResult.AggregatedResultString);
MultiObjectLocalizationAndLabelingResult res = aggRes.boxesAndCategories;
return(res);
}
public static void testJSONString()
{
MultiObjectLocalizationAndLabelingResult res = new MultiObjectLocalizationAndLabelingResult();
res.objects = new List <MultiObjectLocalizationAndLabelingResultSingleEntry>();
MultiObjectLocalizationAndLabelingResultSingleEntry entry = new MultiObjectLocalizationAndLabelingResultSingleEntry();
entry.boundingBox = new BoundingBox(5, 15, 25, 35);
entry.Category = "Car";
res.objects.Add(entry);
entry = new MultiObjectLocalizationAndLabelingResultSingleEntry();
entry.boundingBox = new BoundingBox(45, 55, 65, 75);
entry.Category = "Bus";
res.objects.Add(entry);
string jsonString = JSonUtils.ConvertObjectToJSon <MultiObjectLocalizationAndLabelingResult>(res);
}
private MultiObjectLocalizationAndLabelingResult LoadLatestTurkerResult(SatyamTaskTableEntry entry)
{
SatyamResultsTableAccess resultsDB = new SatyamResultsTableAccess();
List <SatyamResultsTableEntry> entries = resultsDB.getEntriesByGUIDAndTaskID(entry.JobGUID, entry.ID);
if (entries.Count == 0)
{
return(null);
}
//organized sequentially
SatyamResultsTableEntry prevResult = entries[entries.Count - 1];
Hidden_PrevResultID.Value = prevResult.ID.ToString();
SatyamResult satyamResult = JSonUtils.ConvertJSonToObject <SatyamResult>(prevResult.ResultString);
MultiObjectLocalizationAndLabelingResult res = JSonUtils.ConvertJSonToObject <MultiObjectLocalizationAndLabelingResult>(satyamResult.TaskResult);
return(res);
}
public static MultiObjectLabelingAndLocalizationAnalysisPerJob Analyse(List <SatyamResultsTableEntry> entries)
{
MultiObjectLabelingAndLocalizationAnalysisPerJob ana = new MultiObjectLabelingAndLocalizationAnalysisPerJob();
ana.jobTemplateType = entries[0].JobTemplateType;
for (int i = 0; i < entries.Count; i++)
{
SatyamResultsTableEntry entry = entries[i];
SatyamResult satyamResult = JSonUtils.ConvertJSonToObject <SatyamResult>(entry.ResultString);
SatyamTask task = JSonUtils.ConvertJSonToObject <SatyamTask>(satyamResult.TaskParametersString);
SatyamJob job = task.jobEntry;
MultiObjectLocalizationAndLabelingResult res = JSonUtils.ConvertJSonToObject <MultiObjectLocalizationAndLabelingResult>(satyamResult.TaskResult);
}
return(ana);
}
private bool getNewRandomJob()
{
double price = 0;
bool success = Double.TryParse(Hidden_Price.Value, out price);
if (!success)
{
price = 0;
}
SatyamTaskTableAccess taskTableDB = new SatyamTaskTableAccess();
SatyamTaskTableEntry entry = null;
//Thread.Sleep(5000);
int maxDoneScore = TaskConstants.MULTI_OBJECT_LOCALIZATION_AND_LABLING_MAX_DONE_SCORE;
if (SubmitButton.Enabled == true)
{
//entry = taskTableDB.getMinimumTriedNewEntryForWorkerIDByTemplateAndPrice(Hidden_AmazonWorkerID.Value,
// TaskConstants.Detection_Image_MTurk, price);
entry = taskTableDB.getMinimumTriedNewEntryForWorkerIDByTemplateAndPriceAndMaxDoneScore(Hidden_AmazonWorkerID.Value,
TaskConstants.Detection_Image_MTurk, price, MaxDoneScore: maxDoneScore);
}
else
{
//entry = taskTableDB.getMinimumTriedEntryByTemplate(TaskConstants.Detection_Image_MTurk);
entry = taskTableDB.getMinimumTriedEntryByTemplateAndMaxDoneScore(TaskConstants.Detection_Image_MTurk, MaxDoneScore: maxDoneScore);
}
if (entry == null)
{
taskTableDB.close();
return(false);
}
//doneScore = entry.DoneScore;
//taskTableDB.IncrementDoneScore(entry.ID);
taskTableDB.close();
SatyamTask task = JSonUtils.ConvertJSonToObject <SatyamTask>(entry.TaskParametersString);
string uri = task.SatyamURI;
DisplayImage.ImageUrl = uri;
SatyamJob jobDefinitionEntry = task.jobEntry;
MultiObjectLocalizationAndLabelingSubmittedJob job = JSonUtils.ConvertJSonToObject <MultiObjectLocalizationAndLabelingSubmittedJob>(jobDefinitionEntry.JobParameters);
List <string> categories = job.Categories;
CategorySelection_RadioButtonList.Items.Clear();
for (int i = 0; i < categories.Count; i++)
{
ListItem l = new ListItem(categories[i]);
CategorySelection_RadioButtonList.Items.Add(l);
}
if (job.Description != "")
{
DescriptionPanel.Visible = true;
DescriptionTextPanel.Controls.Add(new LiteralControl(job.Description));
}
Hidden_BoundaryLines.Value = JSonUtils.ConvertObjectToJSon(job.BoundaryLines);
Hidden_TaskEntryString.Value = JSonUtils.ConvertObjectToJSon <SatyamTaskTableEntry>(entry);
Hidden_PageLoadTime.Value = DateTime.Now.ToString();
NoLabeled.Text = Hidden_NoImagesDone.Value;
Hidden_TasksPerJob.Value = jobDefinitionEntry.TasksPerJob.ToString();
/////////////////////////////////Load Previous Turker Results
//MultiObjectLocalizationAndLabelingResult res = LoadLatestTurkerResult(entry);
///////////////////////////////Load Previous Aggregation Results
MultiObjectLocalizationAndLabelingResult res = LoadLatestProgressiveAggregationResult(entry);
//TFServing Backend
if (res == null && TFServingBackend)
{
res = LoadTFServingResult(entry);
Hidden_PrevResultID.Value = "-1"; // means TF
}
if (res == null)
{
Hidden_PrevResults.Value = "[]";
Hidden_PrevResultID.Value = 0.ToString();
}
else
{
string prevBoxes = JSonUtils.ConvertObjectToJSon(res.objects);
Hidden_ImageHeight.Value = res.imageHeight.ToString();
Hidden_ImageWidth.Value = res.imageWidth.ToString();
Hidden_PrevResults.Value = prevBoxes;
}
return(true);
}
public static double ACCEPTANCE_NUMBER_OF_BOXES_THRESHOLD = TaskConstants.MULTI_OBJECT_LOCALIZATION_AND_LABLING_MTURK_OBJECT_COVERAGE_THRESHOLD_FOR_PAYMENT; //the person must have made at least 80% of the boxes
public static bool IsAcceptable(SatyamAggregatedResultsTableEntry aggResultEntry, SatyamResultsTableEntry resultEntry,
double DeviationPixelThreshold = TaskConstants.MULTI_OBJECT_LOCALIZATION_AND_LABLING_MTURK_DEVIATION_THRESHOLD_FOR_PAYMENT)
{
//most boxes should be within limits
//most categories should be right
SatyamAggregatedResult satyamAggResult = JSonUtils.ConvertJSonToObject <SatyamAggregatedResult>(aggResultEntry.ResultString);
MultiObjectLocalizationAndLabelingAggregatedResult aggresult = JSonUtils.ConvertJSonToObject <MultiObjectLocalizationAndLabelingAggregatedResult>(satyamAggResult.AggregatedResultString);
SatyamResult satyamResult = JSonUtils.ConvertJSonToObject <SatyamResult>(resultEntry.ResultString);
MultiObjectLocalizationAndLabelingResult result = JSonUtils.ConvertJSonToObject <MultiObjectLocalizationAndLabelingResult>(satyamResult.TaskResult);
if (result == null)
{
return(false);
}
//first check if the number of boxes are within limit
int boxLimit = (int)Math.Ceiling((double)aggresult.boxesAndCategories.objects.Count * ACCEPTANCE_NUMBER_OF_BOXES_THRESHOLD);
if (result.objects.Count < boxLimit)
{
return(false);
}
double minboxdimension_threshold_x = MIN_BOX_DIMENSION_FOR_CONSIDERATION * result.displayScaleReductionX;
double minboxdimension_threshold_y = MIN_BOX_DIMENSION_FOR_CONSIDERATION * result.displayScaleReductionY;
//We fist do a bipartitte matching to find the best assocaition for the boxes
List <List <BoundingBox> > allboxes = new List <List <BoundingBox> >();
allboxes.Add(new List <BoundingBox>());
foreach (MultiObjectLocalizationAndLabelingResultSingleEntry entry in result.objects)
{
allboxes[0].Add(entry.boundingBox);
}
allboxes.Add(new List <BoundingBox>());
List <bool> tooSmallToIgnore = new List <bool>();
foreach (MultiObjectLocalizationAndLabelingResultSingleEntry entry in aggresult.boxesAndCategories.objects)
{
allboxes[1].Add(entry.boundingBox);
if (entry.boundingBox.getWidth() < minboxdimension_threshold_x && entry.boundingBox.getHeight() < minboxdimension_threshold_y)
{
tooSmallToIgnore.Add(true);
}
else
{
tooSmallToIgnore.Add(false);
}
}
List <MultipartiteWeightedMatch> boxAssociation = BoundingBoxAssociation.computeBoundingBoxAssociations(allboxes);
//now find how many of the results match aggregated results
int noAccepted = 0;
double deviation_threshold_x = DeviationPixelThreshold * result.displayScaleReductionX;
double deviation_threshold_y = DeviationPixelThreshold * result.displayScaleReductionY;
int noIgnorable = 0;
foreach (MultipartiteWeightedMatch match in boxAssociation)
{
if (match.elementList.ContainsKey(1)) // this contains an aggregated box
{
if (match.elementList.ContainsKey(0)) // a result box has been associated
{
BoundingBox aggregatedBoundingBox = allboxes[1][match.elementList[1]];
BoundingBox resultBoundingBox = allboxes[0][match.elementList[0]];
//double deviation = aggregatedBoundingBox.ComputeMaxDeviationMetric(resultBoundingBox);
double deviation = aggregatedBoundingBox.ComputeNormalizedMaxDeviationMetric(resultBoundingBox, deviation_threshold_x, deviation_threshold_y);
if (deviation <= 1) //deviation test passed
{
//now check category
if (result.objects[match.elementList[0]].Category == aggresult.boxesAndCategories.objects[match.elementList[1]].Category)
{
//both category and bounding box tests have passed
noAccepted++;
}
}
else
{
if (tooSmallToIgnore[match.elementList[1]])
{
noIgnorable++;
}
}
}
}
}
if (noAccepted >= boxLimit - noIgnorable)
{
return(true);
}
return(false);
}
public static string GetAggregatedResultString(List <SatyamResultsTableEntry> results)
{
if (results.Count == 0)
{
return(null);
}
List <MultiObjectLocalizationAndLabelingResult> resultList = new List <MultiObjectLocalizationAndLabelingResult>();
List <string> WorkersPerTask = new List <string>();
foreach (SatyamResultsTableEntry entry in results)
{
SatyamResult res = JSonUtils.ConvertJSonToObject <SatyamResult>(entry.ResultString);
// remove duplicate workers result
// "" works for internal test!!!
string workerID = res.amazonInfo.WorkerID;
if (workerID != "" && WorkersPerTask.Contains(workerID))
{
continue;
}
//enclose only non-duplicate results, one per each worker.
WorkersPerTask.Add(workerID);
MultiObjectLocalizationAndLabelingResult taskr = JSonUtils.ConvertJSonToObject <MultiObjectLocalizationAndLabelingResult>(res.TaskResult);
resultList.Add(taskr);
}
/// Store Progressive results in progressive table, for consensus loading on turker task pages.
// check if there are any new valid results first
SatyamAggregatedProgressiveResultsTableAccess progDBtemp = new SatyamAggregatedProgressiveResultsTableAccess();
int LatestResultsAggregated = progDBtemp.getLatestNoResultsAggregatedByTaskID(results[0].SatyamTaskTableEntryID);// default value is -1
progDBtemp.close();
if (LatestResultsAggregated >= resultList.Count)
{
return(null);
}
/// safely return null here, since if it's aggregated by standard, it won't be executed again.
/// it will be only executed when there is no standard aggregation result.
/// so return null when no new results came in to even produce a progress aggregation result.
int MaxCount = Math.Min(resultList.Count, TaskConstants.MULTI_OBJECT_LOCALIZATION_AND_LABLING_MTURK_MAX_RESULTS_TO_AGGREGATE);
MultiObjectLocalizationAndLabelingAggregatedResult tempr = getAggregatedResult(resultList, MinResults: 2);
if (tempr != null)
{
string rString = JSonUtils.ConvertObjectToJSon <MultiObjectLocalizationAndLabelingAggregatedResult>(tempr);
SatyamAggregatedResult aggResult = new SatyamAggregatedResult();
aggResult.SatyamTaskTableEntryID = results[0].SatyamTaskTableEntryID;
aggResult.AggregatedResultString = rString;
SatyamResult res = JSonUtils.ConvertJSonToObject <SatyamResult>(results[0].ResultString);
aggResult.TaskParameters = res.TaskParametersString;
string resultString = JSonUtils.ConvertObjectToJSon <SatyamAggregatedResult>(aggResult);
// create progressive entry
SatyamAggregatedProgressiveResultsTableEntry aggProgEntry = new SatyamAggregatedProgressiveResultsTableEntry();
aggProgEntry.JobGUID = results[0].JobGUID;
aggProgEntry.JobTemplateType = results[0].JobTemplateType;
aggProgEntry.SatyamTaskTableEntryID = results[0].SatyamTaskTableEntryID;
aggProgEntry.UserID = results[0].UserID;
aggProgEntry.ResultString = resultString;
aggProgEntry.ResultsAggregated = resultList.Count; // use the actual valid non-duplicate results number here.
SatyamAggregatedProgressiveResultsTableAccess aggProgDB = new SatyamAggregatedProgressiveResultsTableAccess();
aggProgDB.AddEntry(aggProgEntry);
aggProgDB.close();
}
else
{
return(null);
}
MultiObjectLocalizationAndLabelingAggregatedResult r = getAggregatedResult(resultList);
if (r != null)
{
string rString = JSonUtils.ConvertObjectToJSon <MultiObjectLocalizationAndLabelingAggregatedResult>(r);
SatyamAggregatedResult aggResult = new SatyamAggregatedResult();
aggResult.SatyamTaskTableEntryID = results[0].SatyamTaskTableEntryID;
aggResult.AggregatedResultString = rString;
SatyamResult res = JSonUtils.ConvertJSonToObject <SatyamResult>(results[0].ResultString);
aggResult.TaskParameters = res.TaskParametersString;
string resultString = JSonUtils.ConvertObjectToJSon <SatyamAggregatedResult>(aggResult);
return(resultString);
}
else
{
return(null);
}
}
public static void SaveResultImagesLocally(List <SatyamResultsTableEntry> entries, string directoryName)
{
if (!Directory.Exists(directoryName))
{
Directory.CreateDirectory(directoryName);
}
directoryName = directoryName + "\\Raw";
if (!Directory.Exists(directoryName))
{
Directory.CreateDirectory(directoryName);
}
for (int i = 0; i < entries.Count; i++)
{
SatyamResultsTableEntry entry = entries[i];
SatyamResult satyamResult = JSonUtils.ConvertJSonToObject <SatyamResult>(entry.ResultString);
SatyamTask task = JSonUtils.ConvertJSonToObject <SatyamTask>(satyamResult.TaskParametersString);
SatyamJob job = task.jobEntry;
MultiObjectLocalizationAndLabelingResult res = JSonUtils.ConvertJSonToObject <MultiObjectLocalizationAndLabelingResult>(satyamResult.TaskResult);
string ofilename = URIUtilities.filenameFromURI(task.SatyamURI);
string[] fields = ofilename.Split('.');
string fileName = fields[0];
Image originalImage = ImageUtilities.getImageFromURI(task.SatyamURI);
MultiObjectLocalizationAndLabelingSubmittedJob jobDefinition = JSonUtils.ConvertJSonToObject <MultiObjectLocalizationAndLabelingSubmittedJob>(job.JobParameters);
Image imageWithBoundary = DrawingBoxesAndLinesOnImages.addLinesToImage(originalImage, jobDefinition.BoundaryLines, Color.Red, true);
List <Rectangle> rectangles = new List <Rectangle>();
List <Color> colors = new List <Color>();
List <string> ids = new List <string>();
List <bool> dashed = new List <bool>();
for (int j = 0; j < res.objects.Count; j++)
{
MultiObjectLocalizationAndLabelingResultSingleEntry box = res.objects[j];
int x = box.boundingBox.tlx;
int y = box.boundingBox.tly;
int width = box.boundingBox.brx - box.boundingBox.tlx;
int height = box.boundingBox.bry - box.boundingBox.tly;
Rectangle r = new Rectangle(x, y, width, height);
rectangles.Add(r);
string category = box.Category;
int colorIndex = jobDefinition.Categories.IndexOf(category);
colors.Add(DrawingBoxesAndLinesOnImages.Colors[colorIndex]);
string id = j + "-" + category;
ids.Add(id);
dashed.Add(false);
}
Image imageWithBoxesAndBoundary = DrawingBoxesAndLinesOnImages.addRectanglesToImage(imageWithBoundary, rectangles, colors, ids, dashed);
fileName = fileName + "-Result";
if (satyamResult.amazonInfo.AssignmentID != "")
{
//fileName = fileName + "-" + satyamResult.amazonInfo.AssignmentID + "_" + entry.ID;
fileName = fileName + "-" + entry.ID + "_" + satyamResult.PrevResultID;
}
ImageUtilities.saveImage(imageWithBoxesAndBoundary, directoryName, fileName);
}
}
