private static void ValidateSatyamCARPKObjectCountingAggregationResult(List <SatyamAggregatedResultsTableEntry> aggResultEntries,
out double totalError, out double totalGroundTruth)
{
List <double> aggResultCounts = new List <double>();
List <int> GroundTruthCounts = new List <int>();
List <double> abs_errors = new List <double>();
List <double> errors = new List <double>();
foreach (SatyamAggregatedResultsTableEntry aggResultEntry in aggResultEntries)
{
SatyamSaveAggregatedDataSatyam data = new SatyamSaveAggregatedDataSatyam(aggResultEntry);
string fileName = URIUtilities.filenameFromURINoExtension(data.SatyamURI);
string labelFilePath = DirectoryConstants.CARPKCountingLabels + fileName + ".txt";
string[] labels = File.ReadAllLines(labelFilePath);
int GroundtruthCount = labels.Length;
string jobGUID = aggResultEntry.JobGUID;
int taskID = aggResultEntry.SatyamTaskTableEntryID;
String resultString = data.AggregatedResultString;
ObjectCountingAggregatedResult result = JSonUtils.ConvertJSonToObject <ObjectCountingAggregatedResult>(resultString);
aggResultCounts.Add(result.Count);
GroundTruthCounts.Add(GroundtruthCount);
abs_errors.Add(Math.Abs(GroundtruthCount - result.Count));
errors.Add(GroundtruthCount - result.Count);
}
totalError = abs_errors.Sum();
totalGroundTruth = GroundTruthCounts.Sum();
double totalErrorRatio = totalError / totalGroundTruth;
Console.WriteLine("Error: {0} / {1} = {2}", totalError, totalGroundTruth, totalErrorRatio);
Console.WriteLine("Total Aggregated {0}", aggResultEntries.Count);
}
public static bool AggregatedResultEqualsGroundTruth(string satyamUri, string resultString)
{
string fileName = URIUtilities.filenameFromURINoExtension(satyamUri);
string VideoCategory = getVideoCategoryFromFileName(fileName);
if (VideoCategory == "")
{
Console.WriteLine("this video doesn't have valid category in filename");
}
SingleObjectLabelingAggregatedResult result = JSonUtils.ConvertJSonToObject <SingleObjectLabelingAggregatedResult>(resultString);
return(result.Category.Equals(VideoCategory, StringComparison.InvariantCultureIgnoreCase));
}
private static void ValidateSatyamKITTIObjectCountingAggregationResult(List <SatyamAggregatedResultsTableEntry> aggResultEntries,
//bool saveImage = false,
out double totalError,
out double totalGroundTruth,
int MinHeight = TaskConstants.OBJECT_COUNTING_VALIDATION_MIN_HEIGHT,
int MaxOcclusion = TaskConstants.OBJECT_COUNTING_VALIDATION_MAX_OCCLUSION,
double Max_Truncation = TaskConstants.OBJECT_COUNTING_VALIDATION_MIN_TRUNCATION)
{
List <double> aggResultCounts = new List <double>();
List <int> GroundTruthCounts = new List <int>();
List <double> errors = new List <double>();
foreach (SatyamAggregatedResultsTableEntry aggResultEntry in aggResultEntries)
{
SatyamSaveAggregatedDataSatyam data = new SatyamSaveAggregatedDataSatyam(aggResultEntry);
string fileName = URIUtilities.filenameFromURINoExtension(data.SatyamURI);
KITTIDetectionGroundTruth GroundTruthObjects = new KITTIDetectionGroundTruth(KITTIDetectionResultValidation.GroundTruthLabelDirectory + fileName + ".txt", MinHeight, MaxOcclusion, Max_Truncation);
string jobGUID = aggResultEntry.JobGUID;
int taskID = aggResultEntry.SatyamTaskTableEntryID;
String resultString = data.AggregatedResultString;
ObjectCountingAggregatedResult result = JSonUtils.ConvertJSonToObject <ObjectCountingAggregatedResult>(resultString);
int GroundtruthCount = 0;
for (int i = 0; i < GroundTruthObjects.objects.Count; i++)
{
MultiObjectLocalizationAndLabelingResultSingleEntry obj = GroundTruthObjects.objects[i];
//if (obj.Category == "Car" || obj.Category == "Van" || obj.Category == "DontCare")
if (obj.Category == "Car" || obj.Category == "Van")
{
//if (!GroundTruthObjects.BlackListed[i])
//{
GroundtruthCount++;
//}
}
}
aggResultCounts.Add(result.Count);
GroundTruthCounts.Add(GroundtruthCount);
errors.Add(Math.Abs(GroundtruthCount - result.Count));
}
totalError = errors.Sum();
totalGroundTruth = GroundTruthCounts.Sum();
double totalErrorRatio = totalError / totalGroundTruth;
Console.WriteLine("Error: {0} / {1} = {2}", totalError, totalGroundTruth, totalErrorRatio);
Console.WriteLine("Total Aggregated {0}", aggResultEntries.Count);
}
public static ImageSegmentationAggregatedResult_NoHoles getAggregatedResult(List <ImageSegmentationResult_NoHoles> results,
string SatyamURL,
string guid,
int MinResults = TaskConstants.IMAGE_SEGMENTATION_MTURK_MIN_RESULTS_TO_AGGREGATE,
int MaxResults = TaskConstants.IMAGE_SEGMENTATION_MTURK_MAX_RESULTS_TO_AGGREGATE,
double CategoryMajorityThreshold = TaskConstants.IMAGE_SEGMENTATION_MTURK_MAJORITY_CATEGORY_THRESHOLD,
double PolygonBoundaryMajorityThreshold = TaskConstants.IMAGE_SEGMENTATION_MTURK_MAJORITY_POLYGON_BOUNDARY_THRESHOLD,
double ObjectsCoverageThreshold = TaskConstants.IMAGE_SEGMENTATION_MTURK_OBJECT_COVERAGE_THRESHOLD_FOR_AGGREGATION_TERMINATION
)
{
if (results.Count < MinResults) //need at least three results!
{
return(null);
}
int ImageWidth = results[0].imageWidth;
int ImageHeight = results[0].imageHeight;
byte[] PNG = new byte[ImageHeight * ImageWidth];
ImageSegmentationAggregatedResult_NoHoles aggResult = new ImageSegmentationAggregatedResult_NoHoles();
aggResult.metaData = new ImageSegmentationAggregatedResultMetaData_NoHoles();
aggResult.metaData.TotalCount = results.Count;
for (int i = 0; i < PNG.Length; i++)
{
PNG[i] = 0;
}
aggResult.boxesAndCategories = new ImageSegmentationResult_NoHoles();
aggResult.boxesAndCategories.objects = new List <ImageSegmentationResultSingleEntry_NoHoles>();
aggResult.boxesAndCategories.displayScaleReductionX = results[0].displayScaleReductionX;
aggResult.boxesAndCategories.displayScaleReductionY = results[0].displayScaleReductionY;
aggResult.boxesAndCategories.imageHeight = ImageHeight;
aggResult.boxesAndCategories.imageWidth = ImageWidth;
//first use multipartitie wieghted matching to associated the boxes disregarding the labels since
//people might make mistake with lables but boxes are usually right
List <List <GenericPolygon> > AllPolygons = new List <List <GenericPolygon> >();
List <int> noPolygonsPerResult = new List <int>();
foreach (ImageSegmentationResult_NoHoles res in results)
{
AllPolygons.Add(new List <GenericPolygon>());
if (res == null)
{
noPolygonsPerResult.Add(0);
continue;
}
List <GenericPolygon> polygonList = AllPolygons[AllPolygons.Count - 1];
foreach (ImageSegmentationResultSingleEntry_NoHoles entry in res.objects)
{
polygonList.Add(entry.polygon);
}
noPolygonsPerResult.Add(polygonList.Count);
}
//now associate boxes across the various results
List <MultipartiteWeightedMatch> polyAssociation = PolygonAssociation.computeGenericPolygonAssociations(AllPolygons);
int noObjects = polyAssociation.Count;
int noAssociatedPolygons = 0;
//how many of the drawn boxes for each result were actually associated by two or more people for each user?
SortedDictionary <int, int> noMultipleAssociatedPolygonsPerResult = new SortedDictionary <int, int>();
for (int i = 0; i < results.Count; i++)
{
noMultipleAssociatedPolygonsPerResult.Add(i, 0);
}
foreach (MultipartiteWeightedMatch match in polyAssociation)
{
if (match.elementList.Count > 1) //this has been corroborated by two people
{
noAssociatedPolygons++;
foreach (KeyValuePair <int, int> entry in match.elementList)
{
noMultipleAssociatedPolygonsPerResult[entry.Key]++;
}
}
}
//count how many people have a high association ratio
int noHighAssociationRatio = 0;
for (int i = 0; i < results.Count; i++)
{
if (noPolygonsPerResult[i] == 0)
{
continue;
}
double ratio = (double)noMultipleAssociatedPolygonsPerResult[i] / (double)noPolygonsPerResult[i];
if (ratio > ObjectsCoverageThreshold)
{
noHighAssociationRatio++;
}
}
if (noHighAssociationRatio < MinResults) //at least three people should have all their boxes highly corroborated by one other person
{
return(null);
}
//int noAcceptedPolygons = 0;
for (int idx = 0; idx < polyAssociation.Count; idx++)
{
MultipartiteWeightedMatch match = polyAssociation[idx];
List <GenericPolygon> polyList = new List <GenericPolygon>();
List <string> identifiers = new List <string>();
foreach (KeyValuePair <int, int> entry in match.elementList)
{
polyList.Add(AllPolygons[entry.Key][entry.Value]);
identifiers.Add(entry.Key + "_" + entry.Value);
}
//GenericPolygon aggregatedPolygon = GetAggregatedGenericPolygon_Relaxation(polyList, ImageWidth, ImageHeight, PolygonBoundaryMajorityThreshold);
//GenericPolygon aggregatedPolygon = GetAggregatedGenericPolygon_MajorityEdge(polyList, ImageWidth, ImageHeight, PolygonBoundaryMajorityThreshold);
GenericPolygon aggregatedPolygon = new GenericPolygon();// dummy polygon
byte[] png = GetAggregatedGenericPolygon_PixelSweep(polyList, ImageWidth, ImageHeight, PolygonBoundaryMajorityThreshold);
Dictionary <string, int> categoryNames = new Dictionary <string, int>();
int totalCount = match.elementList.Count;
int maxCount = 0;
string maxCategory = "";
foreach (KeyValuePair <int, int> entry in match.elementList)
{
string category = results[entry.Key].objects[entry.Value].Category;
if (!categoryNames.ContainsKey(category))
{
categoryNames.Add(category, 0);
}
categoryNames[category]++;
if (maxCount < categoryNames[category])
{
maxCount = categoryNames[category];
maxCategory = category;
}
}
double probability = ((double)maxCount + 1) / ((double)totalCount + 2);
if (probability < CategoryMajorityThreshold && results.Count < MaxResults) //this is not a valid category need more work
{
return(null);
}
// now we have one segment ready
ImageSegmentationResultSingleEntry_NoHoles aggregated = new ImageSegmentationResultSingleEntry_NoHoles();
aggregated.polygon = aggregatedPolygon;
aggregated.Category = maxCategory;
aggResult.boxesAndCategories.objects.Add(aggregated);
for (int i = 0; i < PNG.Length; i++)
{
if (PNG[i] == 0 && png[i] != 0)
{
PNG[i] = (byte)(idx + 1);
}
}
}
// save and upload to azure
string filename = URIUtilities.filenameFromURINoExtension(SatyamURL);
string filepath = DirectoryConstants.defaultTempDirectory + filename + "_aggregated.PNG";
ImageUtilities.savePNGRawData(filepath, ImageWidth, ImageHeight, PNG);
SatyamJobStorageAccountAccess blob = new SatyamJobStorageAccountAccess();
string container = SatyamTaskGenerator.JobTemplateToSatyamContainerNameMap[TaskConstants.Segmentation_Image_MTurk];
string directoryPath = guid + "_aggregated";
blob.UploadALocalFile(filepath, container, directoryPath);
aggResult.metaData.PNG_URL = TaskConstants.AzureBlobURL + container + "/" + directoryPath + "/" + filename + "_aggregated.PNG";
return(aggResult);
}
/// <summary>
/// Input should be a folder of videos and corresponding annotation file with the same name
/// </summary>
/// <param name="jobEntry"></param>
public static bool ProcessAndUploadToAzureBlob(SatyamJobSubmissionsTableAccessEntry jobEntry)
{
// if the input is a folder of folders of frames, then copy directly
SatyamJobStorageAccountAccess satyamStorage = new SatyamJobStorageAccountAccess();
string satyamContainerName = SatyamTaskGenerator.JobTemplateToSatyamContainerNameMap[jobEntry.JobTemplateType];
string GUID = jobEntry.JobGUID;
string satyamDirectoryName = GUID;
SatyamJob job = JSonUtils.ConvertJSonToObject <SatyamJob>(jobEntry.JobParametersString);
MultiObjectTrackingSubmittedJob jobParams = JSonUtils.ConvertJSonToObject <MultiObjectTrackingSubmittedJob>(job.JobParameters);
BlobContainerManager bcm = new BlobContainerManager();
string status = bcm.Connect(job.azureInformation.AzureBlobStorageConnectionString);
List <string> FileTypes = SatyamTaskGenerator.ValidFileTypesByTemplate[job.JobTemplateType];
if (status != "SUCCESS")
{
return(false);
}
string guidFolder = DirectoryConstants.defaultTempDirectory + "\\" + GUID;
Directory.CreateDirectory(guidFolder);
int chunkLength = jobParams.ChunkDuration; // sec
int outputFPS = jobParams.FrameRate;
double chunkOverlap = jobParams.ChunkOverlap; // sec
var client = new WebClient();
// sample to frames
int noFramePerChunk = (int)(chunkLength * outputFPS);
int noFrameOverlap = (int)(chunkOverlap * outputFPS);
List <string> videoUrls = bcm.getURLList(job.azureInformation.AzureBlobStorageContainerName, job.azureInformation.AzureBlobStorageContainerDirectoryName);
Dictionary <string, List <string> > videos = new Dictionary <string, List <string> >();
foreach (string videolink in videoUrls)
{
string videoName = URIUtilities.filenameFromURINoExtension(videolink);
if (!videos.ContainsKey(videoName))
{
videos.Add(videoName, new List <string>());
}
videos[videoName].Add(videolink);
}
foreach (string videoName in videos.Keys)
{
// filter out those that doesn't provide a annotation file with it....
if (videos[videoName].Count != 2)
{
Console.WriteLine("Warning: Not 2 files provided for {0}.", videoName);
//Directory.Delete(guidFolder, true);
//return false;
continue;
}
string videoURL = "";
string annotationURL = "";
foreach (string fileLink in videos[videoName])
{
string extension = URIUtilities.fileExtensionFromURI(fileLink);
if (extension != "txt")
{
videoURL = fileLink;
}
else
{
annotationURL = fileLink;
}
}
string outputDirectory = guidFolder + "\\" + videoName;
Directory.CreateDirectory(outputDirectory);
string videoNameWithExtension = URIUtilities.filenameFromURI(videoURL);
client.DownloadFile(videoURL, outputDirectory + "\\" + videoNameWithExtension);
FFMpegWrappers.ExtractFrames(DirectoryConstants.ffmpeg, outputDirectory + "\\" + videoNameWithExtension, outputDirectory, videoName, DateTime.Now, outputFPS);
File.Delete(outputDirectory + "\\" + videoNameWithExtension);
List <string> chunkFolders = TrackingDataPreprocessor.GroupFramesIntoChunks(outputDirectory, noFramePerChunk);
//parse VIRAT annotation file
string annotationNameWithExtension = URIUtilities.filenameFromURI(annotationURL);
client.DownloadFile(annotationURL, outputDirectory + "\\" + annotationNameWithExtension);
parseAnnotationFileIntoChunkFolders(chunkFolders,
outputDirectory + "\\" + annotationNameWithExtension,
noFramePerChunk, outputFPS);
//upload
for (int i = 0; i < chunkFolders.Count; i++)
{
string subDir = chunkFolders[i];
satyamStorage.uploadALocalFolder(subDir, satyamContainerName, satyamDirectoryName + "/Video_" + videoName + "_startingFrame_" + noFramePerChunk * i);
}
Directory.Delete(outputDirectory, true);
}
return(true);
}
public static void AggregateWithParameter(string guid,
int MinResults = TaskConstants.TRACKLET_LABELING_MTURK_MIN_RESULTS_TO_AGGREGATE,
int MaxResults = TaskConstants.TRACKLET_LABELING_MTURK_MAX_RESULTS_TO_AGGREGATE,
double boxToleranceThreshold = TaskConstants.TRACKLET_LABELING_BOX_DEVIATION_THRESHOLD,
double ObjectCoverageApprovalThresholdPerVideo = TaskConstants.TRACKLET_LABELING_APPROVALRATIO_PER_VIDEO,
double BoxCoverageApprovalThresholdPerTrack = TaskConstants.TRACKLET_LABELING_APPROVALRATIO_PER_TRACK,
int consensusNumber = TaskConstants.TRACKLET_LABELING_MIN_RESULTS_FOR_CONSENSUS,
double minTubeletIoUSimilarityThreshold = TaskConstants.TRACKLET_LABELING_MIN_TUBELET_SIMILARITY_THRESHOLD,
double attributeMajority = TaskConstants.TRACKLET_LABELING_MTURK_ATTRIBUTE_MAJORITY_THRESHOLD,
bool allFalseAttributeInvalid = false
)
{
string configString = "Min_" + MinResults + "_Max_" + MaxResults + "_IoU_" + minTubeletIoUSimilarityThreshold + "_Ratio_" + ObjectCoverageApprovalThresholdPerVideo;
Console.WriteLine("Aggregating for " + guid + " with param set " + configString);
SatyamResultsTableAccess resultsDB = new SatyamResultsTableAccess();
List <SatyamResultsTableEntry> entries = resultsDB.getEntriesByGUID(guid);
resultsDB.close();
SortedDictionary <DateTime, List <SatyamResultsTableEntry> > entriesBySubmitTime = SatyamResultValidationToolKit.SortResultsBySubmitTime_OneResultPerTurkerPerTask(entries);
Dictionary <int, List <MultiObjectTrackingResult> > ResultsPerTask = new Dictionary <int, List <MultiObjectTrackingResult> >();
List <int> aggregatedTasks = new List <int>();
int noTotalConverged = 0;
//int noCorrect = 0;
int noTerminatedTasks = 0;
List <SatyamAggregatedResultsTableEntry> aggEntries = new List <SatyamAggregatedResultsTableEntry>();
Dictionary <int, int> noResultsNeededForAggregation = SatyamResultsAnalysis.getNoResultsNeededForAggregationFromLog(configString, guid);
Dictionary <int, int> noResultsNeededForAggregation_new = new Dictionary <int, int>();
// play back by time
Dictionary <int, List <string> > WorkersPerTask = new Dictionary <int, List <string> >();
foreach (DateTime t in entriesBySubmitTime.Keys)
{
//Console.WriteLine("Processing Results of time: {0}", t);
List <SatyamResultsTableEntry> ResultEntries = entriesBySubmitTime[t];
foreach (SatyamResultsTableEntry entry in ResultEntries)
{
SatyamResult satyamResult = JSonUtils.ConvertJSonToObject <SatyamResult>(entry.ResultString);
SatyamTask task = JSonUtils.ConvertJSonToObject <SatyamTask>(satyamResult.TaskParametersString);
MultiObjectTrackingSubmittedJob job = JSonUtils.ConvertJSonToObject <MultiObjectTrackingSubmittedJob>(task.jobEntry.JobParameters);
string fileName = URIUtilities.filenameFromURINoExtension(task.SatyamURI);
int taskEntryID = entry.SatyamTaskTableEntryID;
if (aggregatedTasks.Contains(taskEntryID))
{
continue;
}
if (!ResultsPerTask.ContainsKey(taskEntryID))
{
ResultsPerTask.Add(taskEntryID, new List <MultiObjectTrackingResult>());
WorkersPerTask.Add(taskEntryID, new List <string>());
}
// remove duplicate workers result
string workerID = satyamResult.amazonInfo.WorkerID;
if (WorkersPerTask[taskEntryID].Contains(workerID))
{
continue;
}
//enclose only non-duplicate results, one per each worker.
WorkersPerTask[taskEntryID].Add(workerID);
string videoName = URIUtilities.localDirectoryNameFromURI(task.SatyamURI);
string[] fields = videoName.Split('_');
int startingFrame = Convert.ToInt32(fields[fields.Length - 1]);
int maxChunkEndFrame = startingFrame + job.ChunkDuration * job.FrameRate;
int noFrameOverlap = (int)(job.ChunkOverlap * job.FrameRate);
if (startingFrame != 0)
{
startingFrame -= noFrameOverlap;
}
string blobDir = URIUtilities.localDirectoryFullPathFromURI(task.SatyamURI);
VATIC_DVA_CrowdsourcedResult taskr = TrackletLabelingAggregator.createVATIC_DVA_CrowdsourcedResultUsingSatyamBlobImageCount(satyamResult.TaskResult, blobDir, entry.SatyamTaskTableEntryID.ToString(), job.FrameRate);
//MultiObjectTrackingResult result_tmp = vatic_tmp.getCompressedTracksInTimeSegment();
//VATIC_DVA_CrowdsourcedResult taskr = new VATIC_DVA_CrowdsourcedResult(satyamResult.TaskResult, videoName, entry.ID.ToString(), ImageURLs.Count, job.FrameRate);
MultiObjectTrackingResult res = taskr.getCompressedTracksInTimeSegment();
if (allFalseAttributeInvalid)
{
if (TrackletLabelingAggregator.AllAttributeAllFalse(res))
{
continue;
}
}
ResultsPerTask[taskEntryID].Add(res);
// check log if enough results are collected
if (noResultsNeededForAggregation != null && noResultsNeededForAggregation.ContainsKey(taskEntryID) &&
ResultsPerTask[taskEntryID].Count < noResultsNeededForAggregation[taskEntryID])
{
continue;
}
// hack for masters
int tempMin = MinResults;
if (TaskConstants.masterGUIDs.Contains(entry.JobGUID))
{
tempMin = 1;
}
TrackletLabelingAggregatedResult aggResult = TrackletLabelingAggregator.getAggregatedResult(ResultsPerTask[taskEntryID],
guid, task.SatyamURI,
job.FrameRate, job.BoundaryLines,
MinResults, MaxResults, boxToleranceThreshold,
ObjectCoverageApprovalThresholdPerVideo,
BoxCoverageApprovalThresholdPerTrack,
consensusNumber, minTubeletIoUSimilarityThreshold,
attributeMajority);
if (aggResult == null)
{
continue;
}
//////////////// aggregation happen
// record logs
if (noResultsNeededForAggregation == null || !noResultsNeededForAggregation.ContainsKey(taskEntryID))
{
noResultsNeededForAggregation_new.Add(taskEntryID, ResultsPerTask[taskEntryID].Count);
}
aggregatedTasks.Add(taskEntryID);
noTotalConverged++;
if (ResultsPerTask[taskEntryID].Count >= MaxResults)
{
noTerminatedTasks++;
}
SatyamAggregatedResult SatyamAggResult = new SatyamAggregatedResult();
SatyamAggResult.SatyamTaskTableEntryID = taskEntryID;
SatyamAggResult.AggregatedResultString = JSonUtils.ConvertObjectToJSon <TrackletLabelingAggregatedResult>(aggResult);
SatyamAggResult.TaskParameters = JSonUtils.ConvertJSonToObject <SatyamResult>(entry.ResultString).TaskParametersString;
SatyamAggregatedResultsTableEntry aggEntry = new SatyamAggregatedResultsTableEntry();
aggEntry.SatyamTaskTableEntryID = taskEntryID;
aggEntry.JobGUID = entry.JobGUID;
aggEntry.ResultString = JSonUtils.ConvertObjectToJSon <SatyamAggregatedResult>(SatyamAggResult);
aggEntries.Add(aggEntry);
}
}
Console.WriteLine("Total_Aggregated_Tasks: {0}", noTotalConverged);
Console.WriteLine("Total_Terminated_Tasks: {0}", noTerminatedTasks);
SatyamResultsAnalysis.RecordAggregationLog(noResultsNeededForAggregation_new, configString, guid);
TrackletLabelingAnalyzer.GroupEntriesByVideoNameAndStitchAndSaveAggregatedResultVideosLocally(aggEntries, DirectoryConstants.defaultTempDirectory + guid);
}
public static void SaveAggregatedResultVideosLocally(List <SatyamAggregatedResultsTableEntry> entries, string directoryName)
{
directoryName = directoryName + "\\Aggregated";
if (!Directory.Exists(directoryName))
{
Directory.CreateDirectory(directoryName);
}
SatyamJobStorageAccountAccess satyamStorage = new SatyamJobStorageAccountAccess();
for (int i = 0; i < entries.Count; i++)
{
SatyamAggregatedResultsTableEntry entry = entries[i];
SatyamAggregatedResult satyamAggResult = JSonUtils.ConvertJSonToObject <SatyamAggregatedResult>(entry.ResultString);
SatyamTask aggTask = JSonUtils.ConvertJSonToObject <SatyamTask>(satyamAggResult.TaskParameters);
MultiObjectTrackingSubmittedJob job = JSonUtils.ConvertJSonToObject <MultiObjectTrackingSubmittedJob>(aggTask.jobEntry.JobParameters);
TrackletLabelingAggregatedResult aggresult = JSonUtils.ConvertJSonToObject <TrackletLabelingAggregatedResult>(satyamAggResult.AggregatedResultString);
WebClient wb = new WebClient();
Stream aggTrackStream = wb.OpenRead(aggresult.AggregatedTrackletsString_URL);
StreamReader reader = new StreamReader(aggTrackStream);
String aggTrackString = reader.ReadToEnd();
MultiObjectTrackingResult aggTracks = JSonUtils.ConvertJSonToObject <MultiObjectTrackingResult>(aggTrackString);
string blobDir = URIUtilities.localDirectoryFullPathFromURI(aggTask.SatyamURI);
List <string> ImageURLs = satyamStorage.getURLListOfSpecificExtensionUnderSubDirectoryByURI(blobDir, new List <string>()
{
"jpg", "png"
});
string videoName = URIUtilities.localDirectoryNameFromURI(blobDir) + "_" + URIUtilities.filenameFromURINoExtension(aggTask.SatyamURI) + "_" + entry.ID;
MultiObjectTrackingAnalyzer.generateVideoForEvaluation(ImageURLs, aggTracks, directoryName, videoName, job.FrameRate);
}
}
public static void SaveResultVideosLocally(List <SatyamResultsTableEntry> entries, string directoryName, int fps = 10)
{
directoryName = directoryName + "\\Raw";
if (!Directory.Exists(directoryName))
{
Directory.CreateDirectory(directoryName);
}
SatyamJobStorageAccountAccess satyamStorage = new SatyamJobStorageAccountAccess();
for (int i = 0; i < entries.Count; i++)
{
SatyamResultsTableEntry entry = entries[i];
SatyamResult satyamResult = JSonUtils.ConvertJSonToObject <SatyamResult>(entry.ResultString);
SatyamTask task = JSonUtils.ConvertJSonToObject <SatyamTask>(satyamResult.TaskParametersString);
MultiObjectTrackingSubmittedJob job = JSonUtils.ConvertJSonToObject <MultiObjectTrackingSubmittedJob>(task.jobEntry.JobParameters);
string blobDir = URIUtilities.localDirectoryFullPathFromURI(task.SatyamURI);
//VATIC_DVA_CrowdsourcedResult taskr = new VATIC_DVA_CrowdsourcedResult(satyamResult.TaskResult, task.SatyamURI, start, end, task.SatyamJobSubmissionsTableEntryID.ToString());
VATIC_DVA_CrowdsourcedResult taskr = MultiObjectTrackingAggregator.createVATIC_DVA_CrowdsourcedResultUsingSatyamBlobImageCount(satyamResult.TaskResult, blobDir, entry.ID.ToString(), job.FrameRate);
MultiObjectTrackingResult res = taskr.getCompressedTracksInTimeSegment();
List <string> ImageURLs = satyamStorage.getURLListOfSpecificExtensionUnderSubDirectoryByURI(blobDir, new List <string>()
{
"jpg", "png"
});
string videoName = URIUtilities.localDirectoryNameFromURI(blobDir) + "_" + URIUtilities.filenameFromURINoExtension(task.SatyamURI) + "_" + entry.ID;
MultiObjectTrackingAnalyzer.generateVideoForEvaluation(ImageURLs, res, directoryName, videoName, job.FrameRate);
}
}
public static MultiObjectTrackingResult stitchAllChunksAllObjectsOfOneVideo(List <SatyamAggregatedResultsTableEntry> entries, out List <string> ImageURLs, out int fps)
{
ImageURLs = new List <string>();
fps = 0;
if (entries.Count == 0)
{
return(null);
}
SatyamJobStorageAccountAccess satyamStorage = new SatyamJobStorageAccountAccess();
MultiObjectTrackingResult stitched = new MultiObjectTrackingResult();
int totalFrameCounts = 0;
// ensure the order is correct
SortedDictionary <int, List <SatyamAggregatedResultsTableEntry> > sortedEntries = new SortedDictionary <int, List <SatyamAggregatedResultsTableEntry> >();
List <int> idx = new List <int>();
for (int i = 0; i < entries.Count; i++)
{
SatyamAggregatedResultsTableEntry entry = entries[i];
SatyamAggregatedResult satyamAggResult = JSonUtils.ConvertJSonToObject <SatyamAggregatedResult>(entry.ResultString);
SatyamTask aggTask = JSonUtils.ConvertJSonToObject <SatyamTask>(satyamAggResult.TaskParameters);
string video = URIUtilities.localDirectoryNameFromURI(aggTask.SatyamURI);
string[] fields = video.Split('_');
int startingFrame = Convert.ToInt32(fields[fields.Length - 1]);
if (!sortedEntries.ContainsKey(startingFrame))
{
sortedEntries.Add(startingFrame, new List <SatyamAggregatedResultsTableEntry>());
idx.Add(startingFrame);
}
sortedEntries[startingFrame].Add(entries[i]);
}
idx.Sort();
List <string> AggObjIds = new List <string>();
for (int i = 0; i < idx.Count; i++)
{
int noFramesOverlap = 0;
string blobDir = "";
// grouping all objects that belong to the same chunk
MultiObjectTrackingResult aggTracksOfAllObjectsPerChunk = new MultiObjectTrackingResult();
List <string> objIds = new List <string>();
for (int j = 0; j < sortedEntries[idx[i]].Count; j++)
{
SatyamAggregatedResultsTableEntry entry = sortedEntries[idx[i]][j];
SatyamAggregatedResult satyamAggResult = JSonUtils.ConvertJSonToObject <SatyamAggregatedResult>(entry.ResultString);
SatyamTask aggTask = JSonUtils.ConvertJSonToObject <SatyamTask>(satyamAggResult.TaskParameters);
MultiObjectTrackingSubmittedJob job = JSonUtils.ConvertJSonToObject <MultiObjectTrackingSubmittedJob>(aggTask.jobEntry.JobParameters);
if (job.ChunkOverlap != 0.0)
{
noFramesOverlap = (int)(job.ChunkOverlap * job.FrameRate);
}
fps = job.FrameRate;
blobDir = URIUtilities.localDirectoryFullPathFromURI(aggTask.SatyamURI);
string objId = URIUtilities.filenameFromURINoExtension(aggTask.SatyamURI);
objIds.Add(objId);
TrackletLabelingAggregatedResult aggresult = JSonUtils.ConvertJSonToObject <TrackletLabelingAggregatedResult>(satyamAggResult.AggregatedResultString);
WebClient wb = new WebClient();
Stream aggTrackStream = wb.OpenRead(aggresult.AggregatedTrackletsString_URL);
StreamReader reader = new StreamReader(aggTrackStream);
String aggTrackString = reader.ReadToEnd();
MultiObjectTrackingResult aggTracks = JSonUtils.ConvertJSonToObject <MultiObjectTrackingResult>(aggTrackString);
if (aggTracksOfAllObjectsPerChunk.tracks.Count == 0)
{
aggTracksOfAllObjectsPerChunk = aggTracks;
}
else
{
for (int k = 0; k < aggTracks.tracks.Count; k++)
{
aggTracksOfAllObjectsPerChunk.tracks.Add(aggTracks.tracks[k]);
}
}
}
List <string> TraceURLs = satyamStorage.getURLListOfSpecificExtensionUnderSubDirectoryByURI(blobDir, new List <string>()
{
"jpg", "png"
});
if (i == 0)
{
ImageURLs.AddRange(TraceURLs);
stitched = aggTracksOfAllObjectsPerChunk;
totalFrameCounts += TraceURLs.Count;
AggObjIds = objIds;
}
else
{
int noNewFrames = 0;
for (int j = noFramesOverlap; j < TraceURLs.Count; j++)
{
ImageURLs.Add(TraceURLs[j]);
noNewFrames++;
}
//stitched = MultiObjectTrackingAnalyzer.stitchTwoTracesByTubeletsOfOverlappingVideoChunk(stitched,
// aggTracksOfAllObjectsPerChunk, totalFrameCounts, totalFrameCounts + noNewFrames, noFramesOverlap, fps);
// postpone the agg trace
double frameTimeInMiliSeconds = (double)1000 / (double)fps;
double timeToPostponeInMilliSeconds = (double)(totalFrameCounts - noFramesOverlap) * frameTimeInMiliSeconds;
//TimeSpan timeSpanToPostponeInSeconds = new TimeSpan(0,0,0,0, (int)timeToPostponeInMilliSeconds);
TimeSpan timeSpanToPostponeInSeconds = DateTimeUtilities.getTimeSpanFromTotalMilliSeconds((int)timeToPostponeInMilliSeconds);
aggTracksOfAllObjectsPerChunk.postpone(timeSpanToPostponeInSeconds);
// overlap must be 0
for (int k = 0; k < aggTracksOfAllObjectsPerChunk.tracks.Count; k++)
{
if (!AggObjIds.Contains(objIds[k]))
{
stitched.tracks.Add(aggTracksOfAllObjectsPerChunk.tracks[k]);
AggObjIds.Add(objIds[k]);
}
else
{
// stitch the track for the same id
int tckIdx = AggObjIds.IndexOf(objIds[k]);
stitched.tracks[tckIdx] = CompressedTrack.stitchTwoAdjacentTrack(stitched.tracks[tckIdx], aggTracksOfAllObjectsPerChunk.tracks[k]);
}
}
totalFrameCounts += noNewFrames;
}
//debug
//generateVideoForEvaluation(ImageURLs, stitched, directoryName + "_" + i, videoName, fps);
}
return(stitched);
}
public static void SaveResultImagesLocally(List <SatyamResultsTableEntry> entries, string directoryName)
{
if (!Directory.Exists(directoryName))
{
Directory.CreateDirectory(directoryName);
}
directoryName = directoryName + "\\Raw";
if (!Directory.Exists(directoryName))
{
Directory.CreateDirectory(directoryName);
}
// sort by task id
SortedDictionary <int, List <SatyamResultsTableEntry> > taskResults = new SortedDictionary <int, List <SatyamResultsTableEntry> >();
for (int i = 0; i < entries.Count; i++)
{
if (!taskResults.ContainsKey(entries[i].SatyamTaskTableEntryID))
{
taskResults.Add(entries[i].SatyamTaskTableEntryID, new List <SatyamResultsTableEntry>());
}
taskResults[entries[i].SatyamTaskTableEntryID].Add(entries[i]);
}
foreach (int taskID in taskResults.Keys)
{
for (int i = 0; i < taskResults[taskID].Count; i++)
{
SatyamResultsTableEntry entry = taskResults[taskID][i];
SatyamResult satyamResult = JSonUtils.ConvertJSonToObject <SatyamResult>(entry.ResultString);
SatyamTask task = JSonUtils.ConvertJSonToObject <SatyamTask>(satyamResult.TaskParametersString);
SatyamJob job = task.jobEntry;
ImageSegmentationResult res = JSonUtils.ConvertJSonToObject <ImageSegmentationResult>(satyamResult.TaskResult);
if (res == null)
{
continue;
}
string fileName = URIUtilities.filenameFromURINoExtension(task.SatyamURI);
fileName = fileName + "-Result";
if (satyamResult.amazonInfo.AssignmentID != "")
{
fileName = fileName + "-" + satyamResult.amazonInfo.AssignmentID;
}
fileName += "-" + entry.ID;
if (File.Exists(directoryName + "\\" + fileName + ".jpg"))
{
continue;
}
Console.WriteLine("Saving Turker Result {0}", fileName);
Image originalImage = ImageUtilities.getImageFromURI(task.SatyamURI);
Image ResultImage = DrawResultsOnImage(res, originalImage);
byte[] png = ImageSegmentationResult.PolygonResult2Bitmap(res);
if (ResultImage == null)
{
continue;
}
ImageUtilities.saveImage(ResultImage, directoryName, fileName);
ImageUtilities.savePNGRawData(directoryName + "\\" + fileName + "_bitmap.jpg", originalImage.Width, originalImage.Height, png);
}
}
}
public static bool ProcessAndUploadToAzureBlob(SatyamJobSubmissionsTableAccessEntry jobEntry)
{
// if the input is a folder of folders of frames, then copy directly
SatyamJobStorageAccountAccess satyamStorage = new SatyamJobStorageAccountAccess();
string satyamContainerName = SatyamTaskGenerator.JobTemplateToSatyamContainerNameMap[jobEntry.JobTemplateType];
string GUID = jobEntry.JobGUID;
string satyamDirectoryName = GUID;
SatyamJob job = JSonUtils.ConvertJSonToObject <SatyamJob>(jobEntry.JobParametersString);
MultiObjectTrackingSubmittedJob jobParams = JSonUtils.ConvertJSonToObject <MultiObjectTrackingSubmittedJob>(job.JobParameters);
BlobContainerManager bcm = new BlobContainerManager();
string status = bcm.Connect(job.azureInformation.AzureBlobStorageConnectionString);
List <string> FileTypes = SatyamTaskGenerator.ValidFileTypesByTemplate[job.JobTemplateType];
if (status != "SUCCESS")
{
return(false);
}
string guidFolder = DirectoryConstants.defaultTempDirectory + "\\" + GUID;
Directory.CreateDirectory(guidFolder);
int chunkLength = jobParams.ChunkDuration; // sec
int outputFPS = jobParams.FrameRate;
double chunkOverlap = jobParams.ChunkOverlap; // sec
var client = new WebClient();
if (jobParams.DataSrcFormat == DataFormat.Video)
{
// sample to frames
int noFramePerChunk = (int)(chunkLength * outputFPS);
int noFrameOverlap = (int)(chunkOverlap * outputFPS);
List <string> videoUrls = bcm.getURLList(job.azureInformation.AzureBlobStorageContainerName, job.azureInformation.AzureBlobStorageContainerDirectoryName);
foreach (string video in videoUrls)
{
string videoName = URIUtilities.filenameFromURINoExtension(video);
string videonameExtension = URIUtilities.filenameFromURI(video);
string outputDirectory = guidFolder + "\\" + videoName;
Directory.CreateDirectory(outputDirectory);
client.DownloadFile(video, outputDirectory + "\\" + videonameExtension);
FFMpegWrappers.ExtractFrames(DirectoryConstants.ffmpeg, outputDirectory + "\\" + videonameExtension, outputDirectory, videoName, DateTime.Now, outputFPS);
Console.WriteLine("deleting downloaded file...");
File.Delete(outputDirectory + "\\" + videonameExtension);
GroupFramesIntoChunksAndUploadChunks(videoName, outputDirectory, noFramePerChunk, noFrameOverlap, satyamContainerName, satyamDirectoryName);
Directory.Delete(outputDirectory, true);
}
}
if (jobParams.DataSrcFormat == DataFormat.VideoFrame)
{
int noFramePerChunk = (int)(chunkLength * outputFPS);//just use one fps for now, assume input should've already downsampled
int noFrameOverlap = (int)(chunkOverlap * outputFPS);
// chunk according to parameters
List <string> frameUrls = bcm.getURLList(job.azureInformation.AzureBlobStorageContainerName, job.azureInformation.AzureBlobStorageContainerDirectoryName);
Dictionary <string, List <string> > framesPerVideo = new Dictionary <string, List <string> >();
foreach (string url in frameUrls)
{
// assumed hierarchy: blob/directory.../videoname/frameid.jpg
//string frameName = URIUtilities.filenameFromURINoExtension(url);
string[] urlparts = url.Split('/');
string videoName = urlparts[urlparts.Length - 2];
if (!framesPerVideo.ContainsKey(videoName))
{
framesPerVideo.Add(videoName, new List <string>());
}
framesPerVideo[videoName].Add(url);
}
foreach (string video in framesPerVideo.Keys)
{
string outputDirectory = guidFolder + "\\" + video;
Directory.CreateDirectory(outputDirectory);
foreach (string frameURL in framesPerVideo[video])
{
string frameName = URIUtilities.filenameFromURI(frameURL);
client.DownloadFile(frameURL, outputDirectory + "\\" + frameName);
}
GroupFramesIntoChunksAndUploadChunks(video, outputDirectory, noFramePerChunk, noFrameOverlap, satyamContainerName, satyamDirectoryName);
Directory.Delete(outputDirectory, true);
}
}
return(true);
}
public static ImageSegmentationAggregatedResult getAggregatedResult(List <ImageSegmentationResult> originalResults,
string SatyamURL,
string guid,
int MinResults = TaskConstants.IMAGE_SEGMENTATION_MTURK_MIN_RESULTS_TO_AGGREGATE,
int MaxResults = TaskConstants.IMAGE_SEGMENTATION_MTURK_MAX_RESULTS_TO_AGGREGATE,
double CategoryMajorityThreshold = TaskConstants.IMAGE_SEGMENTATION_MTURK_MAJORITY_CATEGORY_THRESHOLD,
double PolygonBoundaryMajorityThreshold = TaskConstants.IMAGE_SEGMENTATION_MTURK_MAJORITY_POLYGON_BOUNDARY_THRESHOLD,
double ObjectsCoverageThreshold = TaskConstants.IMAGE_SEGMENTATION_MTURK_OBJECT_COVERAGE_THRESHOLD_FOR_AGGREGATION_TERMINATION,
double minSimilarityThreshold = TaskConstants.IMAGE_SEGMENTATION_MTURK_POLYGON_IOU_THRESHOLD,
int minResultsForConsensus = TaskConstants.IMAGE_SEGMENTATION_MTURK_MIN_RESULTS_FOR_CONSENSUS
)
{
List <ImageSegmentationResult> results = new List <ImageSegmentationResult>();
// Warning: filter empty results: strong assumption that there has to be something for now!!!
for (int i = 0; i < originalResults.Count; i++)
{
if (originalResults[i] != null && originalResults[i].objects.Count != 0)
{
results.Add(originalResults[i]);
}
}
Console.WriteLine("Filetered Results: {0}", results.Count);
if (results.Count < MinResults) //need at least three results!
{
return(null);
}
int ImageWidth = results[0].imageWidth;
int ImageHeight = results[0].imageHeight;
//auto padding data to make the stride a multiple of 4. required by bmpdata for output
int paddedWidth = ImageWidth;
if (paddedWidth % 4 != 0)
{
paddedWidth = (paddedWidth / 4 + 1) * 4;
}
byte[] PNG = new byte[ImageHeight * paddedWidth];
ImageSegmentationAggregatedResult aggResult = new ImageSegmentationAggregatedResult();
aggResult.metaData = new ImageSegmentationAggregatedResultMetaData();
aggResult.metaData.TotalCount = results.Count;
for (int i = 0; i < PNG.Length; i++)
{
PNG[i] = 0;
}
aggResult.boxesAndCategories = new ImageSegmentationResult();
aggResult.boxesAndCategories.objects = new List <ImageSegmentationResultSingleEntry>();
aggResult.boxesAndCategories.displayScaleReductionX = results[0].displayScaleReductionX;
aggResult.boxesAndCategories.displayScaleReductionY = results[0].displayScaleReductionY;
aggResult.boxesAndCategories.imageHeight = ImageHeight;
aggResult.boxesAndCategories.imageWidth = ImageWidth;
//first use multipartitie wieghted matching to associated the boxes disregarding the labels since
//people might make mistake with lables but boxes are usually right
List <List <Segment> > AllPolygons = new List <List <Segment> >();
List <int> noPolygonsPerResult = new List <int>();
foreach (ImageSegmentationResult res in results)
{
AllPolygons.Add(new List <Segment>());
if (res == null)
{
noPolygonsPerResult.Add(0);
continue;
}
List <Segment> polygonList = AllPolygons[AllPolygons.Count - 1];
foreach (ImageSegmentationResultSingleEntry entry in res.objects)
{
polygonList.Add(entry.segment);
}
noPolygonsPerResult.Add(polygonList.Count);
}
//now associate boxes across the various results
//List<MultipartiteWeightedMatch> polyAssociation = PolygonAssociation.computeGenericPolygonAssociations(AllPolygons);
List <MultipartiteWeightedMatch> polyAssociation = GenericObjectAssociation.computeAssociations <Segment>(AllPolygons, Segment.computeIoU_PixelSweep);
int noObjects = polyAssociation.Count;
int noAssociatedPolygons = 0;
//how many of the drawn boxes for each result were actually associated by two or more people for each user?
SortedDictionary <int, int> noMultipleAssociatedPolygonsPerResult = new SortedDictionary <int, int>();
for (int i = 0; i < results.Count; i++)
{
noMultipleAssociatedPolygonsPerResult.Add(i, 0);
}
foreach (MultipartiteWeightedMatch match in polyAssociation)
{
if (match.elementList.Count > 1) //this has been corroborated by two people
{
noAssociatedPolygons++;
foreach (KeyValuePair <int, int> entry in match.elementList)
{
noMultipleAssociatedPolygonsPerResult[entry.Key]++;
}
}
}
//count how many people have a high association ratio
int noHighAssociationRatio = 0;
for (int i = 0; i < results.Count; i++)
{
if (noPolygonsPerResult[i] == 0)
{
continue;
}
double ratio = (double)noMultipleAssociatedPolygonsPerResult[i] / (double)noPolygonsPerResult[i];
if (ratio > ObjectsCoverageThreshold)
{
noHighAssociationRatio++;
}
}
if (noHighAssociationRatio < MinResults && results.Count < MaxResults) //at least three people should have all their boxes highly corroborated by one other person
{
return(null);
}
//int noAcceptedPolygons = 0;
SortedDictionary <int, List <int> > noHighQualityAssociation = new SortedDictionary <int, List <int> >();
for (int idx = 0; idx < polyAssociation.Count; idx++)
{
MultipartiteWeightedMatch match = polyAssociation[idx];
List <Segment> polyList = new List <Segment>();
List <string> identifiers = new List <string>();
foreach (KeyValuePair <int, int> entry in match.elementList)
{
polyList.Add(AllPolygons[entry.Key][entry.Value]);
identifiers.Add(entry.Key + "_" + entry.Value);
}
SegmentGroup majorityGroup = getMajorityGroupPolygons(identifiers, polyList, minSimilarityThreshold, match.weightMatrix);
if (majorityGroup == null)
{
continue;
}
if (majorityGroup.segments.Count < minResultsForConsensus)
{
continue;
}
Segment aggregatedPolygon = new Segment();// dummy polygon
byte[] png = GetAggregatedGenericPolygon_PixelSweep(majorityGroup.segments, ImageWidth, ImageHeight, PolygonBoundaryMajorityThreshold);
// log where the results come from
foreach (string id in majorityGroup.identifiers)
{
string[] fields = id.Split('_');
int k = Convert.ToInt32(fields[0]);
int v = Convert.ToInt32(fields[1]);
if (!noHighQualityAssociation.ContainsKey(k))
{
noHighQualityAssociation.Add(k, new List <int>());
}
noHighQualityAssociation[k].Add(v);
}
// category aggregation
Dictionary <string, int> categoryNames = new Dictionary <string, int>();
int totalCount = match.elementList.Count;
int maxCount = 0;
string maxCategory = "";
foreach (KeyValuePair <int, int> entry in match.elementList)
{
string category = results[entry.Key].objects[entry.Value].Category;
if (!categoryNames.ContainsKey(category))
{
categoryNames.Add(category, 0);
}
categoryNames[category]++;
if (maxCount < categoryNames[category])
{
maxCount = categoryNames[category];
maxCategory = category;
}
}
double probability = ((double)maxCount + 1) / ((double)totalCount + 2);
if (probability < CategoryMajorityThreshold && results.Count < MaxResults) //this is not a valid category need more work
{
return(null);
}
// now we have one segment ready
ImageSegmentationResultSingleEntry aggregated = new ImageSegmentationResultSingleEntry();
aggregated.segment = aggregatedPolygon;
aggregated.Category = maxCategory;
aggResult.boxesAndCategories.objects.Add(aggregated);
for (int i = 0; i < ImageWidth; i++)
{
for (int j = 0; j < ImageHeight; j++)
{
int pospng = j * ImageWidth + i;
int pos = j * paddedWidth + i;
if (PNG[pos] == 0 && png[pospng] != 0)
{
PNG[pos] = (byte)(idx + 1);
}
}
}
}
// check no. of high association results
int noResultsWithHighQualityObjectCoverage = 0;
for (int i = 0; i < results.Count; i++)
{
if (noPolygonsPerResult[i] == 0)
{
continue;
}
if (!noHighQualityAssociation.ContainsKey(i))
{
continue;
}
double ratio = ((double)noHighQualityAssociation[i].Count) / (double)noPolygonsPerResult[i];
if (ratio > ObjectsCoverageThreshold)
{
noResultsWithHighQualityObjectCoverage++;
}
}
if (noResultsWithHighQualityObjectCoverage < MinResults && results.Count < MaxResults) //at least three people should have most of their boxes highly corroborated by one other person
{
return(null);
}
// save and upload to azure
string filename = URIUtilities.filenameFromURINoExtension(SatyamURL);
//string filepath = DirectoryConstants.defaultTempDirectory + filename + "_aggregated.PNG";
string filepath = DirectoryConstants.defaultAzureTempDirectory + filename + "_aggregated.PNG";
ImageUtilities.savePNGRawData(filepath, ImageWidth, ImageHeight, PNG);
SatyamJobStorageAccountAccess blob = new SatyamJobStorageAccountAccess();
string container = SatyamTaskGenerator.JobTemplateToSatyamContainerNameMap[TaskConstants.Segmentation_Image_MTurk];
string directoryPath = guid + "_aggregated";
blob.UploadALocalFile(filepath, container, directoryPath);
aggResult.metaData.PNG_URL = TaskConstants.AzureBlobURL + container + "/" + directoryPath + "/" + filename + "_aggregated.PNG";
//clean up
File.Delete(filepath);
return(aggResult);
}
/// <summary>
/// function generalizable.. TODO... to all tasks
/// </summary>
/// <param name="guids"></param>
/// <param name="IoUTreshold"></param>
/// <param name="saveImage"></param>
/// <param name="outputDirectory"></param>
/// <param name="MinResults"></param>
/// <param name="MaxResults"></param>
/// <param name="CategoryMajorityThreshold"></param>
/// <param name="PolygonBoundaryMajorityThreshold"></param>
/// <param name="ObjectsCoverageThreshold"></param>
/// <param name="overwrite"></param>
/// <param name="approvalAnalysis"></param>
public static void AggregateWithParameterAndValidatePascalVOCImageSegmentationByGUID(List <string> guids,
double IoUTreshold,
bool saveImage = false, string outputDirectory = null,
int MinResults = TaskConstants.IMAGE_SEGMENTATION_MTURK_MIN_RESULTS_TO_AGGREGATE,
int MaxResults = TaskConstants.IMAGE_SEGMENTATION_MTURK_MAX_RESULTS_TO_AGGREGATE,
double CategoryMajorityThreshold = TaskConstants.IMAGE_SEGMENTATION_MTURK_MAJORITY_CATEGORY_THRESHOLD,
double PolygonBoundaryMajorityThreshold = TaskConstants.IMAGE_SEGMENTATION_MTURK_MAJORITY_POLYGON_BOUNDARY_THRESHOLD,
double ObjectsCoverageThreshold = TaskConstants.IMAGE_SEGMENTATION_MTURK_OBJECT_COVERAGE_THRESHOLD_FOR_AGGREGATION_TERMINATION,
double minSimilarityThreshold = TaskConstants.IMAGE_SEGMENTATION_MTURK_POLYGON_IOU_THRESHOLD,
int minResultsForConsensus = TaskConstants.IMAGE_SEGMENTATION_MTURK_MIN_RESULTS_FOR_CONSENSUS,
bool overwrite = false,
bool approvalAnalysis = false)
{
string configString = "Min_" + MinResults + "_Max_" + MaxResults + "_Majority_" + PolygonBoundaryMajorityThreshold + "_Ratio_" + ObjectsCoverageThreshold;
Console.WriteLine("Aggregating for param set " + configString);
if (!overwrite && File.Exists(DirectoryConstants.defaultTempDirectory + "\\ImageSegmentationResult\\" + configString + ".txt"))
{
return;
}
SatyamResultsTableAccess resultsDB = new SatyamResultsTableAccess();
Dictionary <int, List <string> > WorkersPerTask = new Dictionary <int, List <string> >();
List <SatyamResultsTableEntry> entries = new List <SatyamResultsTableEntry>();
foreach (string guid in guids)
{
entries.AddRange(resultsDB.getEntriesByGUIDOrderByID(guid));
}
resultsDB.close();
SortedDictionary <DateTime, List <SatyamResultsTableEntry> > entriesBySubmitTime =
SatyamResultValidationToolKit.SortResultsBySubmitTime_OneResultPerTurkerPerTask(entries);
Dictionary <int, List <ImageSegmentationResult> > ResultsPerTask = new Dictionary <int, List <ImageSegmentationResult> >();
List <int> aggregatedTasks = new List <int>();
int noTotalConverged = 0;
//int noCorrect = 0;
int noTerminatedTasks = 0;
List <SatyamAggregatedResultsTableEntry> aggEntries = new List <SatyamAggregatedResultsTableEntry>();
Dictionary <int, int> noResultsNeededForAggregation = SatyamResultsAnalysis.getNoResultsNeededForAggregationFromLog(configString, guids[0]);
Dictionary <int, int> noResultsNeededForAggregation_new = new Dictionary <int, int>();
// play back by time
foreach (DateTime t in entriesBySubmitTime.Keys)
{
//Console.WriteLine("Processing Results of time: {0}", t);
List <SatyamResultsTableEntry> ResultEntries = entriesBySubmitTime[t];
foreach (SatyamResultsTableEntry entry in ResultEntries)
{
SatyamResult satyamResult = JSonUtils.ConvertJSonToObject <SatyamResult>(entry.ResultString);
SatyamTask task = JSonUtils.ConvertJSonToObject <SatyamTask>(satyamResult.TaskParametersString);
SatyamJob job = task.jobEntry;
string fileName = URIUtilities.filenameFromURINoExtension(task.SatyamURI);
int taskEntryID = entry.SatyamTaskTableEntryID;
if (aggregatedTasks.Contains(taskEntryID))
{
continue;
}
if (!ResultsPerTask.ContainsKey(taskEntryID))
{
ResultsPerTask.Add(taskEntryID, new List <ImageSegmentationResult>());
}
ResultsPerTask[taskEntryID].Add(JSonUtils.ConvertJSonToObject <ImageSegmentationResult>(satyamResult.TaskResult));
// check log if enough results are collected
if (noResultsNeededForAggregation != null && noResultsNeededForAggregation.ContainsKey(taskEntryID) &&
ResultsPerTask[taskEntryID].Count < noResultsNeededForAggregation[taskEntryID])
{
continue;
}
ImageSegmentationAggregatedResult aggResult = ImageSegmentationAggregator.getAggregatedResult(
ResultsPerTask[taskEntryID], task.SatyamURI, entry.JobGUID,
MinResults, MaxResults, CategoryMajorityThreshold,
PolygonBoundaryMajorityThreshold, ObjectsCoverageThreshold,
minSimilarityThreshold, minResultsForConsensus);
if (aggResult == null)
{
continue;
}
// aggregation happen
// record logs
if (noResultsNeededForAggregation == null || !noResultsNeededForAggregation.ContainsKey(taskEntryID))
{
noResultsNeededForAggregation_new.Add(taskEntryID, ResultsPerTask[taskEntryID].Count);
}
aggregatedTasks.Add(taskEntryID);
noTotalConverged++;
if (ResultsPerTask[taskEntryID].Count >= MaxResults)
{
noTerminatedTasks++;
}
SatyamAggregatedResult SatyamAggResult = new SatyamAggregatedResult();
SatyamAggResult.SatyamTaskTableEntryID = taskEntryID;
SatyamAggResult.AggregatedResultString = JSonUtils.ConvertObjectToJSon <ImageSegmentationAggregatedResult>(aggResult);
SatyamAggResult.TaskParameters = JSonUtils.ConvertJSonToObject <SatyamResult>(entry.ResultString).TaskParametersString;
SatyamAggregatedResultsTableEntry aggEntry = new SatyamAggregatedResultsTableEntry();
aggEntry.SatyamTaskTableEntryID = taskEntryID;
aggEntry.JobGUID = entry.JobGUID;
aggEntry.ResultString = JSonUtils.ConvertObjectToJSon <SatyamAggregatedResult>(SatyamAggResult);
aggEntries.Add(aggEntry);
List <SatyamAggregatedResultsTableEntry> tmpEntries = new List <SatyamAggregatedResultsTableEntry>();
tmpEntries.Add(aggEntry);
//ValidateSatyamKITTIDetectionAggregationResult(tmpEntries, saveImage, MinHeight, MaxOcclusion, Max_Truncation);
}
}
Console.WriteLine("Total_Aggregated_Tasks: {0}", noTotalConverged);
Console.WriteLine("Total_Terminated_Tasks: {0}", noTerminatedTasks);
SatyamResultsAnalysis.RecordAggregationLog(noResultsNeededForAggregation_new, configString, guids[0]);
string r = ValidatePascalVOCImageSegmentationResult_InstanceLevel(aggEntries, IoUTreshold);
r = noTotalConverged + " " + noTerminatedTasks + " " + r;
File.WriteAllText(DirectoryConstants.defaultTempDirectory + "\\ImageSegmentationResult\\" + configString + ".txt", r);
if (approvalAnalysis)
{
string approvalString = configString + "_PayCover_" + TaskConstants.IMAGE_SEGMENTATION_MTURK_OBJECT_COVERAGE_THRESHOLD_FOR_PAYMENT +
"_PayIoU_" + TaskConstants.IMAGE_SEGMENTATION_MTURK_POLYGON_IOU_THRESHOLD_FOR_PAYMENT;
//for (double ratio = 0; ratio < 1; ratio += 0.2)
//{
// SatyamResultsAnalysis.AnalyzeApprovalRate(aggEntries, entriesBySubmitTime, noResultsNeededForAggregation, noResultsNeededForAggregation_new, guids[0], configString, approvalRatioThreshold: ratio);
//}
SatyamResultsAnalysis.AggregationAnalysis(aggEntries, entriesBySubmitTime, noResultsNeededForAggregation, noResultsNeededForAggregation_new, guids[0], configString);
}
}
public static void StaticOfflineAggregationWithParameterAndValidation(List <string> guids,
double IoUTreshold,
int MinResults = TaskConstants.IMAGE_SEGMENTATION_MTURK_MIN_RESULTS_TO_AGGREGATE,
int MaxResults = TaskConstants.IMAGE_SEGMENTATION_MTURK_MAX_RESULTS_TO_AGGREGATE,
double CategoryMajorityThreshold = TaskConstants.IMAGE_SEGMENTATION_MTURK_MAJORITY_CATEGORY_THRESHOLD,
double PolygonBoundaryMajorityThreshold = TaskConstants.IMAGE_SEGMENTATION_MTURK_MAJORITY_POLYGON_BOUNDARY_THRESHOLD,
double ObjectsCoverageThreshold = TaskConstants.IMAGE_SEGMENTATION_MTURK_OBJECT_COVERAGE_THRESHOLD_FOR_AGGREGATION_TERMINATION,
double minSimilarityThreshold = TaskConstants.IMAGE_SEGMENTATION_MTURK_POLYGON_IOU_THRESHOLD,
int minResultsForConsensus = TaskConstants.IMAGE_SEGMENTATION_MTURK_MIN_RESULTS_FOR_CONSENSUS)
{
SatyamResultsTableAccess resultsDB = new SatyamResultsTableAccess();
Dictionary <int, List <string> > WorkersPerTask = new Dictionary <int, List <string> >();
List <SatyamResultsTableEntry> entries = new List <SatyamResultsTableEntry>();
foreach (string guid in guids)
{
entries.AddRange(resultsDB.getEntriesByGUIDOrderByID(guid));
}
resultsDB.close();
SortedDictionary <int, List <SatyamResultsTableEntry> > EntriesPerTask = new SortedDictionary <int, List <SatyamResultsTableEntry> >();
SortedDictionary <int, List <ImageSegmentationResult> > ResultsPerTask = new SortedDictionary <int, List <ImageSegmentationResult> >();
List <int> aggregatedTasks = new List <int>();
int noTotalConverged = 0;
//int noCorrect = 0;
int noTerminatedTasks = 0;
List <SatyamAggregatedResultsTableEntry> aggEntries = new List <SatyamAggregatedResultsTableEntry>();
// Organize by taskID
foreach (SatyamResultsTableEntry entry in entries)
{
SatyamResult satyamResult = JSonUtils.ConvertJSonToObject <SatyamResult>(entry.ResultString);
int taskEntryID = entry.SatyamTaskTableEntryID;
if (!EntriesPerTask.ContainsKey(taskEntryID))
{
EntriesPerTask.Add(taskEntryID, new List <SatyamResultsTableEntry>());
ResultsPerTask.Add(taskEntryID, new List <ImageSegmentationResult>());
}
EntriesPerTask[taskEntryID].Add(entry);
ResultsPerTask[taskEntryID].Add(JSonUtils.ConvertJSonToObject <ImageSegmentationResult>(satyamResult.TaskResult));
}
foreach (int taskEntryID in EntriesPerTask.Keys)
{
SatyamResult satyamResult = JSonUtils.ConvertJSonToObject <SatyamResult>(EntriesPerTask[taskEntryID][0].ResultString);
SatyamTask task = JSonUtils.ConvertJSonToObject <SatyamTask>(satyamResult.TaskParametersString);
SatyamJob job = task.jobEntry;
string fileName = URIUtilities.filenameFromURINoExtension(task.SatyamURI);
List <int> taskidfilter = new List <int>()
{
40430,
40432,
40433,
40434,
40440,
40447,
40451,
40460,
};
//if (fileName != "2007_000549") continue;
if (!taskidfilter.Contains(satyamResult.TaskTableEntryID))
{
continue;
}
Console.WriteLine("Aggregating task {0}: {1} results", taskEntryID, EntriesPerTask[taskEntryID].Count);
ImageSegmentationAggregatedResult aggResult = ImageSegmentationAggregator.getAggregatedResult(
ResultsPerTask[taskEntryID], task.SatyamURI, job.JobGUIDString,
MinResults, MaxResults, CategoryMajorityThreshold,
PolygonBoundaryMajorityThreshold, ObjectsCoverageThreshold,
minSimilarityThreshold, minResultsForConsensus);
if (aggResult == null)
{
continue;
}
// aggregation happen
aggregatedTasks.Add(taskEntryID);
noTotalConverged++;
if (ResultsPerTask[taskEntryID].Count >= MaxResults)
{
noTerminatedTasks++;
}
SatyamAggregatedResult SatyamAggResult = new SatyamAggregatedResult();
SatyamAggResult.SatyamTaskTableEntryID = taskEntryID;
SatyamAggResult.AggregatedResultString = JSonUtils.ConvertObjectToJSon <ImageSegmentationAggregatedResult>(aggResult);
SatyamAggResult.TaskParameters = satyamResult.TaskParametersString;
SatyamAggregatedResultsTableEntry aggEntry = new SatyamAggregatedResultsTableEntry();
aggEntry.SatyamTaskTableEntryID = taskEntryID;
aggEntry.JobGUID = job.JobGUIDString;
aggEntry.ResultString = JSonUtils.ConvertObjectToJSon <SatyamAggregatedResult>(SatyamAggResult);
aggEntries.Add(aggEntry);
List <SatyamAggregatedResultsTableEntry> tmpEntries = new List <SatyamAggregatedResultsTableEntry>();
tmpEntries.Add(aggEntry);
ValidatePascalVOCImageSegmentationResult_InstanceLevel(tmpEntries, IoUTreshold);
}
Console.WriteLine("Total_Aggregated_Tasks: {0}", noTotalConverged);
Console.WriteLine("Total_Terminated_Tasks: {0}", noTerminatedTasks);
string r = ValidatePascalVOCImageSegmentationResult_InstanceLevel(aggEntries, IoUTreshold);
}
public static void ValidatePascalVOCImageSegmentationResult_ClassLevel(
List <SatyamAggregatedResultsTableEntry> resultsPerImage,
List <double> IoUTresholds)
{
SortedDictionary <int, SatyamAggregatedResultsTableEntry> aggResults = new SortedDictionary <int, SatyamAggregatedResultsTableEntry>();
for (int i = 0; i < resultsPerImage.Count; i++)
{
int taskID = resultsPerImage[i].SatyamTaskTableEntryID;
aggResults.Add(taskID, resultsPerImage[i]);
}
SortedDictionary <double, int> tpPerIoU = new SortedDictionary <double, int>();
SortedDictionary <double, int> fpPerIoU = new SortedDictionary <double, int>();
SortedDictionary <double, int> fnPerIoU = new SortedDictionary <double, int>();
foreach (double iou in IoUTresholds)
{
tpPerIoU.Add(iou, 0);
fpPerIoU.Add(iou, 0);
fnPerIoU.Add(iou, 0);
}
foreach (int id in aggResults.Keys)
{
SatyamSaveAggregatedDataSatyam data = new SatyamSaveAggregatedDataSatyam(aggResults[id]);
string fileName = URIUtilities.filenameFromURINoExtension(data.SatyamURI);
PascalVOCSegmentationGroundTruth gt = new PascalVOCSegmentationGroundTruth(fileName);
Console.WriteLine("Results for {0}", fileName);
String resultString = data.AggregatedResultString;
ImageSegmentationAggregatedResult result = JSonUtils.ConvertJSonToObject <ImageSegmentationAggregatedResult>(resultString);
string PNG_URL = result.metaData.PNG_URL;
//List<List<double>> IoUs = gt.getIoUs(result.boxesAndCategories);
SortedDictionary <int, int> noGroundTruthPixels = new SortedDictionary <int, int>();
SortedDictionary <int, int> noDetectionPixels = new SortedDictionary <int, int>();
//List<List<double>> IoUs = gt.getIoUs(PNG_URL, out noDetectionPixels, out noGroundTruthPixels);
List <List <double> > IoUs = gt.getGTOverlaps(PNG_URL, out noDetectionPixels, out noGroundTruthPixels);
List <int> DetectionAreas = noDetectionPixels.Values.ToList();
List <int> GroundtruthAreas = noGroundTruthPixels.Values.ToList();
if (IoUs == null)
{
continue;
}
//double smallAreaToIgnore = 1000;//px
//double smallAreaToIgnore = 625;//px
double smallAreaToIgnore = 1600;//px
List <int> matchedDetections = new List <int>();
for (int i = 0; i < IoUs.Count; i++)
{
if (GroundtruthAreas[i] < smallAreaToIgnore)
{
continue;
}
List <double> ious = IoUs[i];
double maxIoU = 0;
int idx = -1;
for (int j = 0; j < ious.Count; j++)
{
if (ious[j] > maxIoU)
{
maxIoU = ious[j];
idx = j;
}
}
if (maxIoU == 0)
{
Console.WriteLine("No Match");
foreach (double iou in IoUTresholds)
{
fnPerIoU[iou]++;
}
continue;
}
if (DetectionAreas[idx] < smallAreaToIgnore)
{
continue;
}
Console.WriteLine("Match: {0}", maxIoU);
matchedDetections.Add(idx);
foreach (double iou in IoUTresholds)
{
if (maxIoU >= iou)
{
tpPerIoU[iou]++;
}
else
{
fpPerIoU[iou]++;
fnPerIoU[iou]++;
}
}
}
for (int j = 0; j < IoUs[0].Count; j++)
{
if (matchedDetections.Contains(j))
{
continue;
}
if (DetectionAreas[j] < smallAreaToIgnore)
{
continue;
}
double maxIoU = 0;
int idx = -1;
for (int i = 0; i < IoUs.Count; i++)
{
if (IoUs[i][j] > maxIoU)
{
maxIoU = IoUs[i][j];
idx = j;
}
}
bool alreadyCounted = false;
foreach (double iou in IoUTresholds)
{
if (maxIoU >= iou)
{
alreadyCounted = true; break;
}
}
if (alreadyCounted)
{
continue;
}
Console.WriteLine("No Groundtruth");
foreach (double iou in IoUTresholds)
{
fpPerIoU[iou]++;
}
}
}
double AvgPrec = 0;
foreach (double iou in IoUTresholds)
{
int tp = tpPerIoU[iou] + missingGroundTruth.Count;
int fp = fpPerIoU[iou] - missingGroundTruth.Count;
int fn = fnPerIoU[iou] - GroundTruthFilter.Count;
double ap = (double)tp / (double)(tp + fp + fn);
double prec = (double)tp / (double)(tp + fp);
double recl = (double)tp / (double)(tp + fn);
string ret = String.Format("TP: {0}, FP: {1}, FN {2}, AP, {3}, Precision, {4}, Recall, {5}", tp, fp, fn, ap, prec, recl);
Console.WriteLine(ret);
AvgPrec += prec;
}
AvgPrec /= IoUTresholds.Count;
Console.WriteLine("AvgPrec :{0}", AvgPrec);
}
public static string ValidatePascalVOCImageSegmentationResult_InstanceLevel(
List <SatyamAggregatedResultsTableEntry> resultsPerImage,
double IoUTreshold)
{
SortedDictionary <int, SatyamAggregatedResultsTableEntry> aggResults = new SortedDictionary <int, SatyamAggregatedResultsTableEntry>();
for (int i = 0; i < resultsPerImage.Count; i++)
{
int taskID = resultsPerImage[i].SatyamTaskTableEntryID;
aggResults.Add(taskID, resultsPerImage[i]);
}
int tp = 0;
int fp = 0;
int fn = 0;
foreach (int id in aggResults.Keys)
{
SatyamSaveAggregatedDataSatyam data = new SatyamSaveAggregatedDataSatyam(aggResults[id]);
string fileName = URIUtilities.filenameFromURINoExtension(data.SatyamURI);
PascalVOCSegmentationGroundTruth gt = new PascalVOCSegmentationGroundTruth(fileName);
Console.WriteLine("Results for {0}", fileName);
String resultString = data.AggregatedResultString;
ImageSegmentationAggregatedResult result = JSonUtils.ConvertJSonToObject <ImageSegmentationAggregatedResult>(resultString);
string PNG_URL = result.metaData.PNG_URL;
//List<List<double>> IoUs = gt.getIoUs(result.boxesAndCategories);
SortedDictionary <int, int> noGroundTruthPixels = new SortedDictionary <int, int>();
SortedDictionary <int, int> noDetectionPixels = new SortedDictionary <int, int>();
List <List <double> > IoUs = gt.getIoUs(PNG_URL, out noDetectionPixels, out noGroundTruthPixels);
List <int> DetectionAreas = noDetectionPixels.Values.ToList();
List <int> GroundtruthAreas = noGroundTruthPixels.Values.ToList();
if (IoUs == null)
{
continue;
}
MultipartiteWeightTensor matches = new MultipartiteWeightTensor(2);
matches.setNumPartitionElements(0, IoUs.Count);
matches.setNumPartitionElements(1, IoUs[0].Count);
double[,] array = new double[IoUs.Count, IoUs[0].Count];
for (int j = 0; j < IoUs.Count; j++)
{
for (int k = 0; k < IoUs[0].Count; k++)
{
array[j, k] = IoUs[j][k];
}
}
matches.setWeightMatrix(0, 1, array);
MultipartiteWeightedMatching.GreedyMean matching = new MultipartiteWeightedMatching.GreedyMean();
List <MultipartiteWeightedMatch> polygonAssociation = matching.getMatching(matches);
//double smallAreaToIgnore = 1000;//px
//double smallAreaToIgnore = 625;//px
double smallAreaToIgnore = 1600;//px
foreach (MultipartiteWeightedMatch match in polygonAssociation)
{
if (match.elementList.ContainsKey(0)) // this contains gt
{
if (GroundtruthAreas[match.elementList[0]] < smallAreaToIgnore)
{
continue;
}
if (match.elementList.ContainsKey(1)) // an aggregated result box has been associated
{
if (DetectionAreas[match.elementList[1]] < smallAreaToIgnore)
{
continue;
}
double IoU = IoUs[match.elementList[0]][match.elementList[1]];
Console.WriteLine("Match: {0}", IoU);
if (IoU >= IoUTreshold)
{
tp++;
}
else
{
fp++;
fn++;
}
}
else
{
Console.WriteLine("No Match");
fn++;
}
}
else
{
if (DetectionAreas[match.elementList[1]] < smallAreaToIgnore)
{
continue;
}
Console.WriteLine("No Groundtruth");
fp++;
}
}
}
double ap = (double)tp / (double)(tp + fp + fn);
double prec = (double)tp / (double)(tp + fp);
double recl = (double)tp / (double)(tp + fn);
string ret = String.Format("TP: {0}, FP: {1}, FN {2}, AP, {3}, Precision, {4}, Recall, {5}", tp, fp, fn, ap, prec, recl);
Console.WriteLine(ret);
return(ret);
}
