private bool getNewRandomJob()
{
double price = 0;
bool success = Double.TryParse(Hidden_Price.Value, out price);
if (!success)
{
price = 0;
}
SatyamTaskTableAccess taskTableDB = new SatyamTaskTableAccess();
SatyamTaskTableEntry entry = null;
//Thread.Sleep(5000);
if (SubmitButton.Enabled == true)
{
//entry = taskTableDB.getMinimumTriedEntryByTemplateAndPrice(TaskConstants.Detection_Image_MTurk, price);
//entry = taskTableDB.getMinimumTriedNewEntryForWorkerIDByTemplateAndPrice(Hidden_AmazonWorkerID.Value,
// TaskConstants.Detection_Image_MTurk, price, doneScore);
entry = taskTableDB.getMinimumTriedNewEntryForWorkerIDByTemplateAndPrice(Hidden_AmazonWorkerID.Value,
TaskConstants.Detection_Image_MTurk, price);
}
else
{
entry = taskTableDB.getMinimumTriedEntryByTemplate(TaskConstants.Detection_Image_MTurk);
}
if (entry != null)
{
//doneScore = entry.DoneScore;
taskTableDB.IncrementDoneScore(entry.ID);
SatyamTask task = JSonUtils.ConvertJSonToObject <SatyamTask>(entry.TaskParametersString);
string uri = task.SatyamURI;
DisplayImage.ImageUrl = uri;
SatyamJob jobDefinitionEntry = task.jobEntry;
MultiObjectLocalizationAndLabelingSubmittedJob job = JSonUtils.ConvertJSonToObject <MultiObjectLocalizationAndLabelingSubmittedJob>(jobDefinitionEntry.JobParameters);
List <string> categories = job.Categories;
CategorySelection_RadioButtonList.Items.Clear();
for (int i = 0; i < categories.Count; i++)
{
ListItem l = new ListItem(categories[i]);
CategorySelection_RadioButtonList.Items.Add(l);
}
if (job.Description != "")
{
DescriptionPanel.Visible = true;
DescriptionTextPanel.Controls.Add(new LiteralControl(job.Description));
}
Hidden_BoundaryLines.Value = JSonUtils.ConvertObjectToJSon(job.BoundaryLines);
Hidden_TaskEntryString.Value = JSonUtils.ConvertObjectToJSon <SatyamTaskTableEntry>(entry);
Hidden_PageLoadTime.Value = DateTime.Now.ToString();
NoLabeled.Text = Hidden_NoImagesDone.Value;
Hidden_TasksPerJob.Value = jobDefinitionEntry.TasksPerJob.ToString();
taskTableDB.close();
return(true);
}
else
{
taskTableDB.close();
return(false);
}
}
public static bool IsAcceptable(
SatyamAggregatedResultsTableEntry aggResultEntry,
SatyamResultsTableEntry resultEntry,
double APPROVAL_RATIO_OF_POLYGONS_THRESHOLD = TaskConstants.IMAGE_SEGMENTATION_MTURK_OBJECT_COVERAGE_THRESHOLD_FOR_PAYMENT, //the person must have made at least 80% of the polygon
double POLYGON_IOU_THRESHOLD = TaskConstants.IMAGE_SEGMENTATION_MTURK_POLYGON_IOU_THRESHOLD_FOR_PAYMENT
)
{
//most boxes should be within limits
//most categories should be right
SatyamAggregatedResult satyamAggResult = JSonUtils.ConvertJSonToObject <SatyamAggregatedResult>(aggResultEntry.ResultString);
ImageSegmentationAggregatedResult aggresult = JSonUtils.ConvertJSonToObject <ImageSegmentationAggregatedResult>(satyamAggResult.AggregatedResultString);
SatyamResult satyamResult = JSonUtils.ConvertJSonToObject <SatyamResult>(resultEntry.ResultString);
ImageSegmentationResult result = JSonUtils.ConvertJSonToObject <ImageSegmentationResult>(satyamResult.TaskResult);
if (result == null)
{
return(false);
}
//first check if the number of boxes are within limit
// the objects are dummy polygons just for counting
int boxLimit = (int)Math.Ceiling((double)aggresult.boxesAndCategories.objects.Count * APPROVAL_RATIO_OF_POLYGONS_THRESHOLD);
if (result.objects.Count < boxLimit)
{
return(false);
}
//now find how many of the results match aggregated results
int noAccepted = 0;
byte[] resPNG = ImageSegmentationResult.PolygonResult2Bitmap(result);
int width = -1; int height = -1;
byte[] aggPNG = ImageUtilities.readPNGRawDataFromURL(aggresult.metaData.PNG_URL, out width, out height);
if (resPNG.Length != aggPNG.Length)
{
Console.WriteLine("res and agg res are different size");
return(false);
}
SortedDictionary <int, SortedDictionary <int, int> > overlapping = new SortedDictionary <int, SortedDictionary <int, int> >();
SortedDictionary <int, int> resObjectArea = new SortedDictionary <int, int>();
SortedDictionary <int, int> aggObjectArea = new SortedDictionary <int, int>();
for (int i = 0; i < width; i++)
{
for (int j = 0; j < height; j++)
{
int idx = j * width + i;
int resObjID = resPNG[idx];
int aggObjID = aggPNG[idx];
if (!overlapping.ContainsKey(resObjID))
{
overlapping.Add(resObjID, new SortedDictionary <int, int>());
}
if (!overlapping[resObjID].ContainsKey(aggObjID))
{
overlapping[resObjID].Add(aggObjID, 0);
}
overlapping[resObjID][aggObjID]++;
if (!resObjectArea.ContainsKey(resObjID))
{
resObjectArea.Add(resObjID, 0);
}
resObjectArea[resObjID]++;
if (!aggObjectArea.ContainsKey(aggObjID))
{
aggObjectArea.Add(aggObjID, 0);
}
aggObjectArea[aggObjID]++;
}
}
foreach (int id in resObjectArea.Keys)
{
if (id == 0)
{
continue;
}
int maxOverlap = -1;
int maxOverlapAggObjID = -1;
SortedDictionary <int, int> overlapArea = overlapping[id];
foreach (int aggid in overlapArea.Keys)
{
if (aggid == 0)
{
continue;
}
if (overlapArea[aggid] > maxOverlap)
{
maxOverlap = overlapArea[aggid];
maxOverlapAggObjID = aggid;
}
}
if (maxOverlapAggObjID == -1)
{
continue;
}
double iou = (double)maxOverlap / (double)(resObjectArea[id] + aggObjectArea[maxOverlapAggObjID] - maxOverlap);
if (iou >= POLYGON_IOU_THRESHOLD)
{
noAccepted++;
////now check category
//if (result.objects[match.elementList[0]].Category == aggresult.boxesAndCategories.objects[match.elementList[1]].Category)
//{
// //both category and bounding box tests have passed
//}
}
}
if (noAccepted >= boxLimit)
{
return(true);
}
return(false);
}
public static bool ProcessAndUploadToAzureBlob(SatyamJobSubmissionsTableAccessEntry jobEntry)
{
// if the input is a folder of folders of frames, then copy directly
SatyamJobStorageAccountAccess satyamStorage = new SatyamJobStorageAccountAccess();
string satyamContainerName = SatyamTaskGenerator.JobTemplateToSatyamContainerNameMap[jobEntry.JobTemplateType];
string GUID = jobEntry.JobGUID;
string satyamDirectoryName = GUID;
SatyamJob job = JSonUtils.ConvertJSonToObject <SatyamJob>(jobEntry.JobParametersString);
MultiObjectTrackingSubmittedJob jobParams = JSonUtils.ConvertJSonToObject <MultiObjectTrackingSubmittedJob>(job.JobParameters);
BlobContainerManager bcm = new BlobContainerManager();
string status = bcm.Connect(job.azureInformation.AzureBlobStorageConnectionString);
List <string> FileTypes = SatyamTaskGenerator.ValidFileTypesByTemplate[job.JobTemplateType];
if (status != "SUCCESS")
{
return(false);
}
string guidFolder = DirectoryConstants.defaultTempDirectory + "\\" + GUID;
Directory.CreateDirectory(guidFolder);
int chunkLength = jobParams.ChunkDuration; // sec
int outputFPS = jobParams.FrameRate;
double chunkOverlap = jobParams.ChunkOverlap; // sec
var client = new WebClient();
if (jobParams.DataSrcFormat == DataFormat.Video)
{
// sample to frames
int noFramePerChunk = (int)(chunkLength * outputFPS);
int noFrameOverlap = (int)(chunkOverlap * outputFPS);
List <string> videoUrls = bcm.getURLList(job.azureInformation.AzureBlobStorageContainerName, job.azureInformation.AzureBlobStorageContainerDirectoryName);
foreach (string video in videoUrls)
{
string videoName = URIUtilities.filenameFromURINoExtension(video);
string videonameExtension = URIUtilities.filenameFromURI(video);
string outputDirectory = guidFolder + "\\" + videoName;
Directory.CreateDirectory(outputDirectory);
client.DownloadFile(video, outputDirectory + "\\" + videonameExtension);
FFMpegWrappers.ExtractFrames(DirectoryConstants.ffmpeg, outputDirectory + "\\" + videonameExtension, outputDirectory, videoName, DateTime.Now, outputFPS);
Console.WriteLine("deleting downloaded file...");
File.Delete(outputDirectory + "\\" + videonameExtension);
GroupFramesIntoChunksAndUploadChunks(videoName, outputDirectory, noFramePerChunk, noFrameOverlap, satyamContainerName, satyamDirectoryName);
Directory.Delete(outputDirectory, true);
}
}
if (jobParams.DataSrcFormat == DataFormat.VideoFrame)
{
int noFramePerChunk = (int)(chunkLength * outputFPS);//just use one fps for now, assume input should've already downsampled
int noFrameOverlap = (int)(chunkOverlap * outputFPS);
// chunk according to parameters
List <string> frameUrls = bcm.getURLList(job.azureInformation.AzureBlobStorageContainerName, job.azureInformation.AzureBlobStorageContainerDirectoryName);
Dictionary <string, List <string> > framesPerVideo = new Dictionary <string, List <string> >();
foreach (string url in frameUrls)
{
// assumed hierarchy: blob/directory.../videoname/frameid.jpg
//string frameName = URIUtilities.filenameFromURINoExtension(url);
string[] urlparts = url.Split('/');
string videoName = urlparts[urlparts.Length - 2];
if (!framesPerVideo.ContainsKey(videoName))
{
framesPerVideo.Add(videoName, new List <string>());
}
framesPerVideo[videoName].Add(url);
}
foreach (string video in framesPerVideo.Keys)
{
string outputDirectory = guidFolder + "\\" + video;
Directory.CreateDirectory(outputDirectory);
foreach (string frameURL in framesPerVideo[video])
{
string frameName = URIUtilities.filenameFromURI(frameURL);
client.DownloadFile(frameURL, outputDirectory + "\\" + frameName);
}
GroupFramesIntoChunksAndUploadChunks(video, outputDirectory, noFramePerChunk, noFrameOverlap, satyamContainerName, satyamDirectoryName);
Directory.Delete(outputDirectory, true);
}
}
return(true);
}
public static void Run([TimerTrigger("0 */5 * * * *")] TimerInfo myTimer, TraceWriter log)
{
DateTime start = DateTime.Now;
bool logging = false;
if (logging)
{
log.Info($"Payment Dispatch executed at: {DateTime.Now}");
}
SatyamDispatchStorageAccountAccess satyamQueue = new SatyamDispatchStorageAccountAccess();
//get IDs of all accepted and rejected results related to MTurk
Dictionary <string, string> AssignmentIDToHITIDMap = new Dictionary <string, string>();
List <SatyamResultsTableEntry> acceptedEntries = new List <SatyamResultsTableEntry>();
foreach (string taskTemplate in TaskConstants.MTurkTaskTemplates)
{
SatyamResultsTableAccess resultsDB = new SatyamResultsTableAccess();
List <SatyamResultsTableEntry> entries = resultsDB.getAllEntriesByJobtemplateTypeAndStatus(taskTemplate, ResultStatus.accepted);
resultsDB.close();
acceptedEntries.AddRange(entries);
}
List <SatyamResultsTableEntry> rejectedEntries = new List <SatyamResultsTableEntry>();
foreach (string taskTemplate in TaskConstants.MTurkTaskTemplates)
{
SatyamResultsTableAccess resultsDB = new SatyamResultsTableAccess();
List <SatyamResultsTableEntry> entries = resultsDB.getAllEntriesByJobtemplateTypeAndStatus(taskTemplate, ResultStatus.rejected);
resultsDB.close();
rejectedEntries.AddRange(entries);
}
if (acceptedEntries.Count == 0 && rejectedEntries.Count == 0) //nothing to do
{
return;
}
// a single assignment may have multiple accepted results
Dictionary <string, List <SatyamResultsTableEntry> > acceptedResultsByAssignmentID = new Dictionary <string, List <SatyamResultsTableEntry> >();
foreach (SatyamResultsTableEntry entry in acceptedEntries)
{
SatyamResult sr = JSonUtils.ConvertJSonToObject <SatyamResult>(entry.ResultString);
string AssignmentID = sr.amazonInfo.AssignmentID;
if (!AssignmentIDToHITIDMap.ContainsKey(AssignmentID))
{
string HITID = sr.amazonInfo.HITID;
AssignmentIDToHITIDMap.Add(AssignmentID, HITID);
}
if (!acceptedResultsByAssignmentID.ContainsKey(AssignmentID))
{
acceptedResultsByAssignmentID.Add(AssignmentID, new List <SatyamResultsTableEntry>());
}
acceptedResultsByAssignmentID[AssignmentID].Add(entry);
}
List <string> assignmentIDList = acceptedResultsByAssignmentID.Keys.ToList();
// a single assignment may have multiple rejected results
Dictionary <string, List <SatyamResultsTableEntry> > rejectedResultsByAssignmentID = new Dictionary <string, List <SatyamResultsTableEntry> >();
foreach (SatyamResultsTableEntry entry in rejectedEntries)
{
SatyamResult sr = JSonUtils.ConvertJSonToObject <SatyamResult>(entry.ResultString);
string AssignmentID = sr.amazonInfo.AssignmentID;
if (!AssignmentIDToHITIDMap.ContainsKey(AssignmentID))
{
string HITID = sr.amazonInfo.HITID;
AssignmentIDToHITIDMap.Add(AssignmentID, HITID);
}
if (!rejectedResultsByAssignmentID.ContainsKey(AssignmentID))
{
rejectedResultsByAssignmentID.Add(AssignmentID, new List <SatyamResultsTableEntry>());
if (!assignmentIDList.Contains(AssignmentID))
{
assignmentIDList.Add(AssignmentID);
}
}
rejectedResultsByAssignmentID[AssignmentID].Add(entry);
}
List <string> rejectedHITs = new List <string>();
List <string> acceptedHITs = new List <string>();
//for each assignment ID we check if it should be paid or not and then pay or reject them.
//pay all accepted entries
foreach (string assignmentID in assignmentIDList)
{
int noAccepted = 0;
SatyamResult r = null;
SatyamResultsTableEntry entry = null;
if (acceptedResultsByAssignmentID.ContainsKey(assignmentID))
{
noAccepted = acceptedResultsByAssignmentID[assignmentID].Count;
entry = acceptedResultsByAssignmentID[assignmentID][0];
r = JSonUtils.ConvertJSonToObject <SatyamResult>(entry.ResultString);
}
int noRejected = 0;
if (rejectedResultsByAssignmentID.ContainsKey(assignmentID))
{
noRejected = rejectedResultsByAssignmentID[assignmentID].Count;
if (r == null)
{
entry = rejectedResultsByAssignmentID[assignmentID][0];
r = JSonUtils.ConvertJSonToObject <SatyamResult>(entry.ResultString);
}
}
double ratio = (double)noAccepted / ((double)noAccepted + (double)noRejected);
SatyamTask task = JSonUtils.ConvertJSonToObject <SatyamTask>(r.TaskParametersString);
string AmazonAccessKeyID = task.jobEntry.amazonHITInformation.AmazonAccessKeyID;
string AmazonSecretAccessKeyID = task.jobEntry.amazonHITInformation.AmazonSecretAccessKeyID;
string m = AmazonAccessKeyID + "_" + AmazonSecretAccessKeyID + "_" + assignmentID;
SatyamResultsTableAccess resultsDB = new SatyamResultsTableAccess();
if (ratio >= AmazonMTurkPayments.assignement_acceptance_threshold) //this is acceptable
{
if (logging)
{
log.Info($"Dispatching Payment for {assignmentID}");
}
string queueName = "payment";
satyamQueue.Enqueue(queueName, m);
if (acceptedResultsByAssignmentID.ContainsKey(assignmentID))
{
foreach (SatyamResultsTableEntry result in acceptedResultsByAssignmentID[assignmentID])
{
resultsDB.UpdateStatusByID(result.ID, ResultStatus.accepted_Paid);
}
}
if (rejectedResultsByAssignmentID.ContainsKey(assignmentID))
{
foreach (SatyamResultsTableEntry result in rejectedResultsByAssignmentID[assignmentID])
{
resultsDB.UpdateStatusByID(result.ID, ResultStatus.rejected_Paid);
}
}
if (AssignmentIDToHITIDMap.ContainsKey(assignmentID))
{
if (!acceptedHITs.Contains(AssignmentIDToHITIDMap[assignmentID]))
{
acceptedHITs.Add(AssignmentIDToHITIDMap[assignmentID]);
SatyamAmazonHITTableAccess hitDB = new SatyamAmazonHITTableAccess();
hitDB.UpdateStatusByHITID(AssignmentIDToHITIDMap[assignmentID], HitStatus.accepted);
hitDB.close();
}
}
}
else
{
if (logging)
{
log.Info($"Dispatching NoPayment for {assignmentID}");
}
string queueName = "nopayment";
satyamQueue.Enqueue(queueName, m);
if (acceptedResultsByAssignmentID.ContainsKey(assignmentID))
{
foreach (SatyamResultsTableEntry result in acceptedResultsByAssignmentID[assignmentID])
{
resultsDB.UpdateStatusByID(result.ID, ResultStatus.accepted_NotPaid);
}
}
if (rejectedResultsByAssignmentID.ContainsKey(assignmentID))
{
foreach (SatyamResultsTableEntry result in rejectedResultsByAssignmentID[assignmentID])
{
resultsDB.UpdateStatusByID(result.ID, ResultStatus.rejected_NotPaid);
}
}
if (AssignmentIDToHITIDMap.ContainsKey(assignmentID))
{
if (!rejectedHITs.Contains(AssignmentIDToHITIDMap[assignmentID]))
{
rejectedHITs.Add(AssignmentIDToHITIDMap[assignmentID]);
SatyamAmazonHITTableAccess hitDB = new SatyamAmazonHITTableAccess();
hitDB.UpdateStatusByHITID(AssignmentIDToHITIDMap[assignmentID], HitStatus.rejected);
hitDB.close();
}
}
}
if ((DateTime.Now - start).TotalSeconds > 280)
{
break;
}
}
}
public static void SaveResultImagesLocally(List <SatyamResultsTableEntry> entries, string directoryName)
{
if (!Directory.Exists(directoryName))
{
Directory.CreateDirectory(directoryName);
}
directoryName = directoryName + "\\Raw";
if (!Directory.Exists(directoryName))
{
Directory.CreateDirectory(directoryName);
}
for (int i = 0; i < entries.Count; i++)
{
SatyamResultsTableEntry entry = entries[i];
SatyamResult satyamResult = JSonUtils.ConvertJSonToObject <SatyamResult>(entry.ResultString);
SatyamTask task = JSonUtils.ConvertJSonToObject <SatyamTask>(satyamResult.TaskParametersString);
SatyamJob job = task.jobEntry;
MultiObjectLocalizationAndLabelingResult res = JSonUtils.ConvertJSonToObject <MultiObjectLocalizationAndLabelingResult>(satyamResult.TaskResult);
string ofilename = URIUtilities.filenameFromURI(task.SatyamURI);
string[] fields = ofilename.Split('.');
string fileName = fields[0];
Image originalImage = ImageUtilities.getImageFromURI(task.SatyamURI);
MultiObjectLocalizationAndLabelingSubmittedJob jobDefinition = JSonUtils.ConvertJSonToObject <MultiObjectLocalizationAndLabelingSubmittedJob>(job.JobParameters);
Image imageWithBoundary = DrawingBoxesAndLinesOnImages.addLinesToImage(originalImage, jobDefinition.BoundaryLines, Color.Red, true);
List <Rectangle> rectangles = new List <Rectangle>();
List <Color> colors = new List <Color>();
List <string> ids = new List <string>();
List <bool> dashed = new List <bool>();
for (int j = 0; j < res.objects.Count; j++)
{
MultiObjectLocalizationAndLabelingResultSingleEntry box = res.objects[j];
int x = box.boundingBox.tlx;
int y = box.boundingBox.tly;
int width = box.boundingBox.brx - box.boundingBox.tlx;
int height = box.boundingBox.bry - box.boundingBox.tly;
Rectangle r = new Rectangle(x, y, width, height);
rectangles.Add(r);
string category = box.Category;
int colorIndex = jobDefinition.Categories.IndexOf(category);
colors.Add(DrawingBoxesAndLinesOnImages.Colors[colorIndex]);
string id = j + "-" + category;
ids.Add(id);
dashed.Add(false);
}
Image imageWithBoxesAndBoundary = DrawingBoxesAndLinesOnImages.addRectanglesToImage(imageWithBoundary, rectangles, colors, ids, dashed);
fileName = fileName + "-Result";
if (satyamResult.amazonInfo.AssignmentID != "")
{
//fileName = fileName + "-" + satyamResult.amazonInfo.AssignmentID + "_" + entry.ID;
fileName = fileName + "-" + entry.ID + "_" + satyamResult.PrevResultID;
}
ImageUtilities.saveImage(imageWithBoxesAndBoundary, directoryName, fileName);
}
}
//go through all the inconclusive results and check with aggregated table, if aggreagted, compare and see if it is acceptable and then
//accept/reject
public static void AcceptRejectResults()
{
//get all inconclusive results and group them by taskIDs
SatyamResultsTableAccess resultsDB = new SatyamResultsTableAccess();
List <SatyamResultsTableEntry> results = resultsDB.getAllEntriesByStatus(ResultStatus.inconclusive);
if (results.Count == 0)
{
resultsDB.close();
return;
}
Dictionary <int, List <SatyamResultsTableEntry> > resultsByTaskID = new Dictionary <int, List <SatyamResultsTableEntry> >();
List <string> pendingGUID = new List <string>();
foreach (SatyamResultsTableEntry result in results)
{
if (!resultsByTaskID.ContainsKey(result.SatyamTaskTableEntryID))
{
resultsByTaskID.Add(result.SatyamTaskTableEntryID, new List <SatyamResultsTableEntry>());
}
resultsByTaskID[result.SatyamTaskTableEntryID].Add(result);
if (!pendingGUID.Contains(result.JobGUID))
{
pendingGUID.Add(result.JobGUID);
}
}
//now check against the aggregated result to see if the result is aceptable
Dictionary <string, Dictionary <int, SatyamAggregatedResultsTableEntry> > aggEntriesPerTaskPerGUID = SatyamAggregatedResultManagement.getAggregatedEntriesPerTaskByGuidList(pendingGUID);
foreach (KeyValuePair <int, List <SatyamResultsTableEntry> > entry in resultsByTaskID)
{
if (entry.Value.Count == 0)
{
continue;
}
int taskEntryID = entry.Key;
string taskGUID = entry.Value[0].JobGUID;
if (aggEntriesPerTaskPerGUID.ContainsKey(taskGUID) && aggEntriesPerTaskPerGUID[taskGUID].ContainsKey(taskEntryID))
{
// this task has been aggregated
foreach (SatyamResultsTableEntry result in resultsByTaskID[taskEntryID]) //now got through each task to see if they satify pass criterion
{
SatyamResult res = JSonUtils.ConvertJSonToObject <SatyamResult>(result.ResultString);
string taskTemplateType = result.JobTemplateType;
string workerID = res.amazonInfo.WorkerID;
DateTime doneTime = res.TaskEndTime;
if (AcceptanceCriterionChecker.IsAcceptable(aggEntriesPerTaskPerGUID[taskGUID][taskEntryID], result))
{
resultsDB.UpdateStatusByID(result.ID, ResultStatus.accepted);
WorkerStatisticsManagement.UpdateWorkerStatistics(workerID, taskTemplateType, true, doneTime);
}
else
{
resultsDB.UpdateStatusByID(result.ID, ResultStatus.rejected);
WorkerStatisticsManagement.UpdateWorkerStatistics(workerID, taskTemplateType, false, doneTime);
}
}
}
}
resultsDB.close();
}
private bool getNewRandomJob()
{
SatyamTaskTableAccess taskTableDB = new SatyamTaskTableAccess();
SatyamTaskTableEntry entry = taskTableDB.getMinimumTriedEntryByTemplate(TaskConstants.TrackletLabeling);
if (entry != null)
{
taskTableDB.IncrementDoneScore(entry.ID);
}
taskTableDB.close();
if (entry != null)
{
SatyamTask task = JSonUtils.ConvertJSonToObject <SatyamTask>(entry.TaskParametersString);
SatyamJobStorageAccountAccess satyamStorage = new SatyamJobStorageAccountAccess();
string videoDir = URIUtilities.localDirectoryFullPathFromURI(task.SatyamURI);
List <string> ImageURLs = satyamStorage.getURLListOfSpecificExtensionUnderSubDirectoryByURI(videoDir, new List <string>()
{
"jpg"
});
string annotationFilePath = task.SatyamURI;
//string urls = "";
//for (int i=0;i<ImageURLs.Count;i++)
//{
// urls += ImageURLs[i];
// if (i == ImageURLs.Count - 1) break;
// urls += ',';
//}
Hidden_ImageURLList.Value = ObjectsToStrings.ListString(ImageURLs, ',');
SatyamJob jobDefinitionEntry = task.jobEntry;
MultiObjectTrackingSubmittedJob job = JSonUtils.ConvertJSonToObject <MultiObjectTrackingSubmittedJob>(jobDefinitionEntry.JobParameters);
Dictionary <string, List <string> > subcategories = job.Categories;
List <string> categories = subcategories.Keys.ToList();
//CategorySelection_RakdioButtonList.Items.Clear();
for (int i = 0; i < categories.Count; i++)
{
ListItem l = new ListItem(categories[i]);
//CategorySelection_RadioButtonList.Items.Add(l);
}
if (job.Description != "")
{
//DescriptionPanel.Visible = true;
//DescriptionTextPanel.Controls.Add(new LiteralControl(job.Description));
}
//Hidden_BoundaryLines.Value = JSonUtils.ConvertObjectToJSon(job.BoundaryLines);
Hidden_TaskEntryString.Value = JSonUtils.ConvertObjectToJSon <SatyamTaskTableEntry>(entry);
Hidden_PageLoadTime.Value = DateTime.Now.ToString();
// pass parameters from old template
Slug_Hidden.Value = "null";
Start_Hidden.Value = "0";
Stop_Hidden.Value = (ImageURLs.Count - 1).ToString();
Skip_Hidden.Value = "0";
PerObject_Hidden.Value = "0.1";
Completion_Hidden.Value = "0.5";
BlowRadius_Hidden.Value = "0";
JobId_Hidden.Value = "1";
LabelString_Hidden.Value = ObjectsToStrings.ListString(categories.ToList(), ',');
Attributes_Hidden.Value = ObjectsToStrings.DictionaryStringListString(subcategories, ',', ':', '_');
Training_Hidden.Value = "0";
fps_Hidden.Value = job.FrameRate.ToString();
Hidden_ChunkDuration.Value = job.ChunkDuration.ToString();
var web = new WebClient();
System.Drawing.Image x = System.Drawing.Image.FromStream(web.OpenRead(ImageURLs[0]));
ImageWidth_Hidden.Value = x.Width.ToString();
ImageHeight_Hidden.Value = x.Height.ToString();
// image boundary for now
//string[] region = new string[] { "0-0-1242-0-1242-375-0-375-0-0" };
string[] region = new string[] { "0-0-" + x.Width + "-0-" + x.Width + "-" + x.Height + "-0-" + x.Height + "-0-0" };
RegionString_Hidden.Value = ObjectsToStrings.ListString(region, ',');
// temp test
List <VATIC_Tracklet> prevTracesTemp = new List <VATIC_Tracklet>();
WebClient client = new WebClient();
Stream stream = client.OpenRead(annotationFilePath);
StreamReader reader = new StreamReader(stream);
List <string> trace = new List <string>();
while (reader.Peek() >= 0)
{
string content = reader.ReadLine();
trace.Add(content);
}
Dictionary <string, VATIC_Tracklet> tracklets = VATIC_Tracklet.ReadTrackletsFromVIRAT(trace);
foreach (string id in tracklets.Keys)
{
//string output = JSonUtils.ConvertObjectToJSon(tracklets[id]);
prevTracesTemp.Add(tracklets[id]);
}
string output = JSonUtils.ConvertObjectToJSon(prevTracesTemp);
PreviousTrackString_Hidden.Value = output;
return(true);
}
else
{
return(false);
}
}
/// <summary>
/// function generalizable.. TODO... to all tasks
/// </summary>
/// <param name="guids"></param>
/// <param name="IoUTreshold"></param>
/// <param name="saveImage"></param>
/// <param name="outputDirectory"></param>
/// <param name="MinResults"></param>
/// <param name="MaxResults"></param>
/// <param name="CategoryMajorityThreshold"></param>
/// <param name="PolygonBoundaryMajorityThreshold"></param>
/// <param name="ObjectsCoverageThreshold"></param>
/// <param name="overwrite"></param>
/// <param name="approvalAnalysis"></param>
public static void AggregateWithParameterAndValidatePascalVOCImageSegmentationByGUID(List <string> guids,
double IoUTreshold,
bool saveImage = false, string outputDirectory = null,
int MinResults = TaskConstants.IMAGE_SEGMENTATION_MTURK_MIN_RESULTS_TO_AGGREGATE,
int MaxResults = TaskConstants.IMAGE_SEGMENTATION_MTURK_MAX_RESULTS_TO_AGGREGATE,
double CategoryMajorityThreshold = TaskConstants.IMAGE_SEGMENTATION_MTURK_MAJORITY_CATEGORY_THRESHOLD,
double PolygonBoundaryMajorityThreshold = TaskConstants.IMAGE_SEGMENTATION_MTURK_MAJORITY_POLYGON_BOUNDARY_THRESHOLD,
double ObjectsCoverageThreshold = TaskConstants.IMAGE_SEGMENTATION_MTURK_OBJECT_COVERAGE_THRESHOLD_FOR_AGGREGATION_TERMINATION,
double minSimilarityThreshold = TaskConstants.IMAGE_SEGMENTATION_MTURK_POLYGON_IOU_THRESHOLD,
int minResultsForConsensus = TaskConstants.IMAGE_SEGMENTATION_MTURK_MIN_RESULTS_FOR_CONSENSUS,
bool overwrite = false,
bool approvalAnalysis = false)
{
string configString = "Min_" + MinResults + "_Max_" + MaxResults + "_Majority_" + PolygonBoundaryMajorityThreshold + "_Ratio_" + ObjectsCoverageThreshold;
Console.WriteLine("Aggregating for param set " + configString);
if (!overwrite && File.Exists(DirectoryConstants.defaultTempDirectory + "\\ImageSegmentationResult\\" + configString + ".txt"))
{
return;
}
SatyamResultsTableAccess resultsDB = new SatyamResultsTableAccess();
Dictionary <int, List <string> > WorkersPerTask = new Dictionary <int, List <string> >();
List <SatyamResultsTableEntry> entries = new List <SatyamResultsTableEntry>();
foreach (string guid in guids)
{
entries.AddRange(resultsDB.getEntriesByGUIDOrderByID(guid));
}
resultsDB.close();
SortedDictionary <DateTime, List <SatyamResultsTableEntry> > entriesBySubmitTime =
SatyamResultValidationToolKit.SortResultsBySubmitTime_OneResultPerTurkerPerTask(entries);
Dictionary <int, List <ImageSegmentationResult> > ResultsPerTask = new Dictionary <int, List <ImageSegmentationResult> >();
List <int> aggregatedTasks = new List <int>();
int noTotalConverged = 0;
//int noCorrect = 0;
int noTerminatedTasks = 0;
List <SatyamAggregatedResultsTableEntry> aggEntries = new List <SatyamAggregatedResultsTableEntry>();
Dictionary <int, int> noResultsNeededForAggregation = SatyamResultsAnalysis.getNoResultsNeededForAggregationFromLog(configString, guids[0]);
Dictionary <int, int> noResultsNeededForAggregation_new = new Dictionary <int, int>();
// play back by time
foreach (DateTime t in entriesBySubmitTime.Keys)
{
//Console.WriteLine("Processing Results of time: {0}", t);
List <SatyamResultsTableEntry> ResultEntries = entriesBySubmitTime[t];
foreach (SatyamResultsTableEntry entry in ResultEntries)
{
SatyamResult satyamResult = JSonUtils.ConvertJSonToObject <SatyamResult>(entry.ResultString);
SatyamTask task = JSonUtils.ConvertJSonToObject <SatyamTask>(satyamResult.TaskParametersString);
SatyamJob job = task.jobEntry;
string fileName = URIUtilities.filenameFromURINoExtension(task.SatyamURI);
int taskEntryID = entry.SatyamTaskTableEntryID;
if (aggregatedTasks.Contains(taskEntryID))
{
continue;
}
if (!ResultsPerTask.ContainsKey(taskEntryID))
{
ResultsPerTask.Add(taskEntryID, new List <ImageSegmentationResult>());
}
ResultsPerTask[taskEntryID].Add(JSonUtils.ConvertJSonToObject <ImageSegmentationResult>(satyamResult.TaskResult));
// check log if enough results are collected
if (noResultsNeededForAggregation != null && noResultsNeededForAggregation.ContainsKey(taskEntryID) &&
ResultsPerTask[taskEntryID].Count < noResultsNeededForAggregation[taskEntryID])
{
continue;
}
ImageSegmentationAggregatedResult aggResult = ImageSegmentationAggregator.getAggregatedResult(
ResultsPerTask[taskEntryID], task.SatyamURI, entry.JobGUID,
MinResults, MaxResults, CategoryMajorityThreshold,
PolygonBoundaryMajorityThreshold, ObjectsCoverageThreshold,
minSimilarityThreshold, minResultsForConsensus);
if (aggResult == null)
{
continue;
}
// aggregation happen
// record logs
if (noResultsNeededForAggregation == null || !noResultsNeededForAggregation.ContainsKey(taskEntryID))
{
noResultsNeededForAggregation_new.Add(taskEntryID, ResultsPerTask[taskEntryID].Count);
}
aggregatedTasks.Add(taskEntryID);
noTotalConverged++;
if (ResultsPerTask[taskEntryID].Count >= MaxResults)
{
noTerminatedTasks++;
}
SatyamAggregatedResult SatyamAggResult = new SatyamAggregatedResult();
SatyamAggResult.SatyamTaskTableEntryID = taskEntryID;
SatyamAggResult.AggregatedResultString = JSonUtils.ConvertObjectToJSon <ImageSegmentationAggregatedResult>(aggResult);
SatyamAggResult.TaskParameters = JSonUtils.ConvertJSonToObject <SatyamResult>(entry.ResultString).TaskParametersString;
SatyamAggregatedResultsTableEntry aggEntry = new SatyamAggregatedResultsTableEntry();
aggEntry.SatyamTaskTableEntryID = taskEntryID;
aggEntry.JobGUID = entry.JobGUID;
aggEntry.ResultString = JSonUtils.ConvertObjectToJSon <SatyamAggregatedResult>(SatyamAggResult);
aggEntries.Add(aggEntry);
List <SatyamAggregatedResultsTableEntry> tmpEntries = new List <SatyamAggregatedResultsTableEntry>();
tmpEntries.Add(aggEntry);
//ValidateSatyamKITTIDetectionAggregationResult(tmpEntries, saveImage, MinHeight, MaxOcclusion, Max_Truncation);
}
}
Console.WriteLine("Total_Aggregated_Tasks: {0}", noTotalConverged);
Console.WriteLine("Total_Terminated_Tasks: {0}", noTerminatedTasks);
SatyamResultsAnalysis.RecordAggregationLog(noResultsNeededForAggregation_new, configString, guids[0]);
string r = ValidatePascalVOCImageSegmentationResult_InstanceLevel(aggEntries, IoUTreshold);
r = noTotalConverged + " " + noTerminatedTasks + " " + r;
File.WriteAllText(DirectoryConstants.defaultTempDirectory + "\\ImageSegmentationResult\\" + configString + ".txt", r);
if (approvalAnalysis)
{
string approvalString = configString + "_PayCover_" + TaskConstants.IMAGE_SEGMENTATION_MTURK_OBJECT_COVERAGE_THRESHOLD_FOR_PAYMENT +
"_PayIoU_" + TaskConstants.IMAGE_SEGMENTATION_MTURK_POLYGON_IOU_THRESHOLD_FOR_PAYMENT;
//for (double ratio = 0; ratio < 1; ratio += 0.2)
//{
// SatyamResultsAnalysis.AnalyzeApprovalRate(aggEntries, entriesBySubmitTime, noResultsNeededForAggregation, noResultsNeededForAggregation_new, guids[0], configString, approvalRatioThreshold: ratio);
//}
SatyamResultsAnalysis.AggregationAnalysis(aggEntries, entriesBySubmitTime, noResultsNeededForAggregation, noResultsNeededForAggregation_new, guids[0], configString);
}
}
public static void ValidateSatyamImageNetClassificationAggregationResultByGUID(string guid, string confusingImageListFilePath = null,
bool prepareDataForTraining = false, string outputDirectory = null)
{
Dictionary <string, ConfusingReason> imageBlackListReason = new Dictionary <string, ConfusingReason>();
if (confusingImageListFilePath != null)
{
imageBlackListReason = getConfusingImageList(confusingImageListFilePath);
}
//get all aggregated results
SatyamAggregatedResultsTableAccess resultsDB = new SatyamAggregatedResultsTableAccess();
List <SatyamAggregatedResultsTableEntry> results = resultsDB.getEntriesByGUID(guid);
resultsDB.close();
int noTotal = 0;
int noCorrect = 0;
SortedDictionary <string, Dictionary <string, int> > confusionMatrix_res_groundtruth = new SortedDictionary <string, Dictionary <string, int> >();
SortedDictionary <string, Dictionary <string, int> > confusionMatrix_groundtruth_res = new SortedDictionary <string, Dictionary <string, int> >();
StringBuilder s = new StringBuilder();
s.Append("<!DOCTYPE html>\n");
s.Append("<html>\n");
s.Append("<body>\n");
String title = String.Format("<h1>Job GUID {0} Incorrect Result Summary</h1>\n", guid);
s.Append(title);
WebClient wc = new WebClient();
for (int i = 0; i < results.Count; i++)
{
SatyamSaveAggregatedDataSatyam data = new SatyamSaveAggregatedDataSatyam(results[i]);
//String uri = data.SatyamURI;
//string[] uri_parts= uri.Split('/');
//string[] name_parts = uri_parts[uri_parts.Length - 1].Split('_');
string fileName = URIUtilities.filenameFromURI(data.SatyamURI);
string imageCategoryName = fileName.Split('_')[0];
String resultString = data.AggregatedResultString;
SingleObjectLabelingAggregatedResult result = JSonUtils.ConvertJSonToObject <SingleObjectLabelingAggregatedResult>(resultString);
// skip all ambulances and black listed
if (IsBlackListed(data.SatyamURI, imageBlackListReason))
{
continue;
}
if (!confusionMatrix_res_groundtruth.ContainsKey(result.Category))
{
confusionMatrix_res_groundtruth.Add(result.Category, new Dictionary <string, int>()
{
{ GroundTruth[imageCategoryName], 0 }
});
}
else
{
if (!confusionMatrix_res_groundtruth[result.Category].ContainsKey(GroundTruth[imageCategoryName]))
{
confusionMatrix_res_groundtruth[result.Category].Add(GroundTruth[imageCategoryName], 0);
}
}
if (!confusionMatrix_groundtruth_res.ContainsKey(GroundTruth[imageCategoryName]))
{
confusionMatrix_groundtruth_res.Add(GroundTruth[imageCategoryName], new Dictionary <string, int>()
{
{ result.Category, 0 }
});
}
else
{
if (!confusionMatrix_groundtruth_res[GroundTruth[imageCategoryName]].ContainsKey(result.Category))
{
confusionMatrix_groundtruth_res[GroundTruth[imageCategoryName]].Add(result.Category, 0);
}
}
if (result.Category.Equals(GroundTruth[imageCategoryName], StringComparison.InvariantCultureIgnoreCase))
{
noCorrect++;
}
else
{
//Console.WriteLine("{0}, Groundtruth: {1}, Aggregated: {2}, Votes: {3}",
// fileName, GroundTruth[imageCategoryName], result.Category,
// JSonUtils.ConvertObjectToJSon(result.metaData));
String record = String.Format("<p>{0}, Groundtruth: {1}, Aggregated: {2}, Votes: {3}</p>\n",
fileName, GroundTruth[imageCategoryName], result.Category,
JSonUtils.ConvertObjectToJSon(result.metaData));
String img = String.Format("<img src=\"{0}\" >\n", data.SatyamURI);
s.Append(record);
s.Append(img);
}
// prepare training dataset
if (prepareDataForTraining)
{
if (GroundTruth[imageCategoryName] != "ambulance" && result.Category != "ambulance")
{
Image im = Image.FromStream(wc.OpenRead(data.SatyamURI));
if (!Directory.Exists(outputDirectory + result.Category))
{
Directory.CreateDirectory(outputDirectory + result.Category);
}
im.Save(outputDirectory + "\\" + result.Category + "\\" + fileName);
}
}
noTotal++;
confusionMatrix_res_groundtruth[result.Category][GroundTruth[imageCategoryName]]++;
confusionMatrix_groundtruth_res[GroundTruth[imageCategoryName]][result.Category]++;
}
Console.WriteLine("Result: {0}/{1}, precision: {2}", noCorrect, noTotal, (double)noCorrect / noTotal);
// write the confusion matrix
s.Append("<p>");
String row = "\t\t";
foreach (string resultCategory in confusionMatrix_res_groundtruth.Keys)
{
row += resultCategory + "\t";
}
row += "<br>\n";
s.Append(row);
Console.WriteLine(row);
foreach (string groundTruthCategory in confusionMatrix_groundtruth_res.Keys)
{
row = groundTruthCategory + "\t";
foreach (string resultCategory in confusionMatrix_res_groundtruth.Keys)
{
if (confusionMatrix_groundtruth_res[groundTruthCategory].ContainsKey(resultCategory))
{
row += confusionMatrix_groundtruth_res[groundTruthCategory][resultCategory].ToString();
}
else
{
row += "0";
}
row += "\t";
}
row += "<br>\n";
s.Append(row);
Console.WriteLine(row);
}
s.Append("</p>\n");
s.Append("</body>\n");
s.Append("</html>\n");
string dataToBeSaved = s.ToString();
SatyamJobStorageAccountAccess storage = new SatyamJobStorageAccountAccess();
string FileName = "AggregatedIncorrectResults-" + guid + ".html";
storage.SaveATextFile("singleobjectlabeling", guid, FileName, dataToBeSaved);
}
public static void ValidatePascalVOCImageSegmentationResult_ClassLevel(
List <SatyamAggregatedResultsTableEntry> resultsPerImage,
List <double> IoUTresholds)
{
SortedDictionary <int, SatyamAggregatedResultsTableEntry> aggResults = new SortedDictionary <int, SatyamAggregatedResultsTableEntry>();
for (int i = 0; i < resultsPerImage.Count; i++)
{
int taskID = resultsPerImage[i].SatyamTaskTableEntryID;
aggResults.Add(taskID, resultsPerImage[i]);
}
SortedDictionary <double, int> tpPerIoU = new SortedDictionary <double, int>();
SortedDictionary <double, int> fpPerIoU = new SortedDictionary <double, int>();
SortedDictionary <double, int> fnPerIoU = new SortedDictionary <double, int>();
foreach (double iou in IoUTresholds)
{
tpPerIoU.Add(iou, 0);
fpPerIoU.Add(iou, 0);
fnPerIoU.Add(iou, 0);
}
foreach (int id in aggResults.Keys)
{
SatyamSaveAggregatedDataSatyam data = new SatyamSaveAggregatedDataSatyam(aggResults[id]);
string fileName = URIUtilities.filenameFromURINoExtension(data.SatyamURI);
PascalVOCSegmentationGroundTruth gt = new PascalVOCSegmentationGroundTruth(fileName);
Console.WriteLine("Results for {0}", fileName);
String resultString = data.AggregatedResultString;
ImageSegmentationAggregatedResult result = JSonUtils.ConvertJSonToObject <ImageSegmentationAggregatedResult>(resultString);
string PNG_URL = result.metaData.PNG_URL;
//List<List<double>> IoUs = gt.getIoUs(result.boxesAndCategories);
SortedDictionary <int, int> noGroundTruthPixels = new SortedDictionary <int, int>();
SortedDictionary <int, int> noDetectionPixels = new SortedDictionary <int, int>();
//List<List<double>> IoUs = gt.getIoUs(PNG_URL, out noDetectionPixels, out noGroundTruthPixels);
List <List <double> > IoUs = gt.getGTOverlaps(PNG_URL, out noDetectionPixels, out noGroundTruthPixels);
List <int> DetectionAreas = noDetectionPixels.Values.ToList();
List <int> GroundtruthAreas = noGroundTruthPixels.Values.ToList();
if (IoUs == null)
{
continue;
}
//double smallAreaToIgnore = 1000;//px
//double smallAreaToIgnore = 625;//px
double smallAreaToIgnore = 1600;//px
List <int> matchedDetections = new List <int>();
for (int i = 0; i < IoUs.Count; i++)
{
if (GroundtruthAreas[i] < smallAreaToIgnore)
{
continue;
}
List <double> ious = IoUs[i];
double maxIoU = 0;
int idx = -1;
for (int j = 0; j < ious.Count; j++)
{
if (ious[j] > maxIoU)
{
maxIoU = ious[j];
idx = j;
}
}
if (maxIoU == 0)
{
Console.WriteLine("No Match");
foreach (double iou in IoUTresholds)
{
fnPerIoU[iou]++;
}
continue;
}
if (DetectionAreas[idx] < smallAreaToIgnore)
{
continue;
}
Console.WriteLine("Match: {0}", maxIoU);
matchedDetections.Add(idx);
foreach (double iou in IoUTresholds)
{
if (maxIoU >= iou)
{
tpPerIoU[iou]++;
}
else
{
fpPerIoU[iou]++;
fnPerIoU[iou]++;
}
}
}
for (int j = 0; j < IoUs[0].Count; j++)
{
if (matchedDetections.Contains(j))
{
continue;
}
if (DetectionAreas[j] < smallAreaToIgnore)
{
continue;
}
double maxIoU = 0;
int idx = -1;
for (int i = 0; i < IoUs.Count; i++)
{
if (IoUs[i][j] > maxIoU)
{
maxIoU = IoUs[i][j];
idx = j;
}
}
bool alreadyCounted = false;
foreach (double iou in IoUTresholds)
{
if (maxIoU >= iou)
{
alreadyCounted = true; break;
}
}
if (alreadyCounted)
{
continue;
}
Console.WriteLine("No Groundtruth");
foreach (double iou in IoUTresholds)
{
fpPerIoU[iou]++;
}
}
}
double AvgPrec = 0;
foreach (double iou in IoUTresholds)
{
int tp = tpPerIoU[iou] + missingGroundTruth.Count;
int fp = fpPerIoU[iou] - missingGroundTruth.Count;
int fn = fnPerIoU[iou] - GroundTruthFilter.Count;
double ap = (double)tp / (double)(tp + fp + fn);
double prec = (double)tp / (double)(tp + fp);
double recl = (double)tp / (double)(tp + fn);
string ret = String.Format("TP: {0}, FP: {1}, FN {2}, AP, {3}, Precision, {4}, Recall, {5}", tp, fp, fn, ap, prec, recl);
Console.WriteLine(ret);
AvgPrec += prec;
}
AvgPrec /= IoUTresholds.Count;
Console.WriteLine("AvgPrec :{0}", AvgPrec);
}
public static void StaticOfflineAggregationWithParameterAndValidation(List <string> guids,
double IoUTreshold,
int MinResults = TaskConstants.IMAGE_SEGMENTATION_MTURK_MIN_RESULTS_TO_AGGREGATE,
int MaxResults = TaskConstants.IMAGE_SEGMENTATION_MTURK_MAX_RESULTS_TO_AGGREGATE,
double CategoryMajorityThreshold = TaskConstants.IMAGE_SEGMENTATION_MTURK_MAJORITY_CATEGORY_THRESHOLD,
double PolygonBoundaryMajorityThreshold = TaskConstants.IMAGE_SEGMENTATION_MTURK_MAJORITY_POLYGON_BOUNDARY_THRESHOLD,
double ObjectsCoverageThreshold = TaskConstants.IMAGE_SEGMENTATION_MTURK_OBJECT_COVERAGE_THRESHOLD_FOR_AGGREGATION_TERMINATION,
double minSimilarityThreshold = TaskConstants.IMAGE_SEGMENTATION_MTURK_POLYGON_IOU_THRESHOLD,
int minResultsForConsensus = TaskConstants.IMAGE_SEGMENTATION_MTURK_MIN_RESULTS_FOR_CONSENSUS)
{
SatyamResultsTableAccess resultsDB = new SatyamResultsTableAccess();
Dictionary <int, List <string> > WorkersPerTask = new Dictionary <int, List <string> >();
List <SatyamResultsTableEntry> entries = new List <SatyamResultsTableEntry>();
foreach (string guid in guids)
{
entries.AddRange(resultsDB.getEntriesByGUIDOrderByID(guid));
}
resultsDB.close();
SortedDictionary <int, List <SatyamResultsTableEntry> > EntriesPerTask = new SortedDictionary <int, List <SatyamResultsTableEntry> >();
SortedDictionary <int, List <ImageSegmentationResult> > ResultsPerTask = new SortedDictionary <int, List <ImageSegmentationResult> >();
List <int> aggregatedTasks = new List <int>();
int noTotalConverged = 0;
//int noCorrect = 0;
int noTerminatedTasks = 0;
List <SatyamAggregatedResultsTableEntry> aggEntries = new List <SatyamAggregatedResultsTableEntry>();
// Organize by taskID
foreach (SatyamResultsTableEntry entry in entries)
{
SatyamResult satyamResult = JSonUtils.ConvertJSonToObject <SatyamResult>(entry.ResultString);
int taskEntryID = entry.SatyamTaskTableEntryID;
if (!EntriesPerTask.ContainsKey(taskEntryID))
{
EntriesPerTask.Add(taskEntryID, new List <SatyamResultsTableEntry>());
ResultsPerTask.Add(taskEntryID, new List <ImageSegmentationResult>());
}
EntriesPerTask[taskEntryID].Add(entry);
ResultsPerTask[taskEntryID].Add(JSonUtils.ConvertJSonToObject <ImageSegmentationResult>(satyamResult.TaskResult));
}
foreach (int taskEntryID in EntriesPerTask.Keys)
{
SatyamResult satyamResult = JSonUtils.ConvertJSonToObject <SatyamResult>(EntriesPerTask[taskEntryID][0].ResultString);
SatyamTask task = JSonUtils.ConvertJSonToObject <SatyamTask>(satyamResult.TaskParametersString);
SatyamJob job = task.jobEntry;
string fileName = URIUtilities.filenameFromURINoExtension(task.SatyamURI);
List <int> taskidfilter = new List <int>()
{
40430,
40432,
40433,
40434,
40440,
40447,
40451,
40460,
};
//if (fileName != "2007_000549") continue;
if (!taskidfilter.Contains(satyamResult.TaskTableEntryID))
{
continue;
}
Console.WriteLine("Aggregating task {0}: {1} results", taskEntryID, EntriesPerTask[taskEntryID].Count);
ImageSegmentationAggregatedResult aggResult = ImageSegmentationAggregator.getAggregatedResult(
ResultsPerTask[taskEntryID], task.SatyamURI, job.JobGUIDString,
MinResults, MaxResults, CategoryMajorityThreshold,
PolygonBoundaryMajorityThreshold, ObjectsCoverageThreshold,
minSimilarityThreshold, minResultsForConsensus);
if (aggResult == null)
{
continue;
}
// aggregation happen
aggregatedTasks.Add(taskEntryID);
noTotalConverged++;
if (ResultsPerTask[taskEntryID].Count >= MaxResults)
{
noTerminatedTasks++;
}
SatyamAggregatedResult SatyamAggResult = new SatyamAggregatedResult();
SatyamAggResult.SatyamTaskTableEntryID = taskEntryID;
SatyamAggResult.AggregatedResultString = JSonUtils.ConvertObjectToJSon <ImageSegmentationAggregatedResult>(aggResult);
SatyamAggResult.TaskParameters = satyamResult.TaskParametersString;
SatyamAggregatedResultsTableEntry aggEntry = new SatyamAggregatedResultsTableEntry();
aggEntry.SatyamTaskTableEntryID = taskEntryID;
aggEntry.JobGUID = job.JobGUIDString;
aggEntry.ResultString = JSonUtils.ConvertObjectToJSon <SatyamAggregatedResult>(SatyamAggResult);
aggEntries.Add(aggEntry);
List <SatyamAggregatedResultsTableEntry> tmpEntries = new List <SatyamAggregatedResultsTableEntry>();
tmpEntries.Add(aggEntry);
ValidatePascalVOCImageSegmentationResult_InstanceLevel(tmpEntries, IoUTreshold);
}
Console.WriteLine("Total_Aggregated_Tasks: {0}", noTotalConverged);
Console.WriteLine("Total_Terminated_Tasks: {0}", noTerminatedTasks);
string r = ValidatePascalVOCImageSegmentationResult_InstanceLevel(aggEntries, IoUTreshold);
}
public static string ValidatePascalVOCImageSegmentationResult_InstanceLevel(
List <SatyamAggregatedResultsTableEntry> resultsPerImage,
double IoUTreshold)
{
SortedDictionary <int, SatyamAggregatedResultsTableEntry> aggResults = new SortedDictionary <int, SatyamAggregatedResultsTableEntry>();
for (int i = 0; i < resultsPerImage.Count; i++)
{
int taskID = resultsPerImage[i].SatyamTaskTableEntryID;
aggResults.Add(taskID, resultsPerImage[i]);
}
int tp = 0;
int fp = 0;
int fn = 0;
foreach (int id in aggResults.Keys)
{
SatyamSaveAggregatedDataSatyam data = new SatyamSaveAggregatedDataSatyam(aggResults[id]);
string fileName = URIUtilities.filenameFromURINoExtension(data.SatyamURI);
PascalVOCSegmentationGroundTruth gt = new PascalVOCSegmentationGroundTruth(fileName);
Console.WriteLine("Results for {0}", fileName);
String resultString = data.AggregatedResultString;
ImageSegmentationAggregatedResult result = JSonUtils.ConvertJSonToObject <ImageSegmentationAggregatedResult>(resultString);
string PNG_URL = result.metaData.PNG_URL;
//List<List<double>> IoUs = gt.getIoUs(result.boxesAndCategories);
SortedDictionary <int, int> noGroundTruthPixels = new SortedDictionary <int, int>();
SortedDictionary <int, int> noDetectionPixels = new SortedDictionary <int, int>();
List <List <double> > IoUs = gt.getIoUs(PNG_URL, out noDetectionPixels, out noGroundTruthPixels);
List <int> DetectionAreas = noDetectionPixels.Values.ToList();
List <int> GroundtruthAreas = noGroundTruthPixels.Values.ToList();
if (IoUs == null)
{
continue;
}
MultipartiteWeightTensor matches = new MultipartiteWeightTensor(2);
matches.setNumPartitionElements(0, IoUs.Count);
matches.setNumPartitionElements(1, IoUs[0].Count);
double[,] array = new double[IoUs.Count, IoUs[0].Count];
for (int j = 0; j < IoUs.Count; j++)
{
for (int k = 0; k < IoUs[0].Count; k++)
{
array[j, k] = IoUs[j][k];
}
}
matches.setWeightMatrix(0, 1, array);
MultipartiteWeightedMatching.GreedyMean matching = new MultipartiteWeightedMatching.GreedyMean();
List <MultipartiteWeightedMatch> polygonAssociation = matching.getMatching(matches);
//double smallAreaToIgnore = 1000;//px
//double smallAreaToIgnore = 625;//px
double smallAreaToIgnore = 1600;//px
foreach (MultipartiteWeightedMatch match in polygonAssociation)
{
if (match.elementList.ContainsKey(0)) // this contains gt
{
if (GroundtruthAreas[match.elementList[0]] < smallAreaToIgnore)
{
continue;
}
if (match.elementList.ContainsKey(1)) // an aggregated result box has been associated
{
if (DetectionAreas[match.elementList[1]] < smallAreaToIgnore)
{
continue;
}
double IoU = IoUs[match.elementList[0]][match.elementList[1]];
Console.WriteLine("Match: {0}", IoU);
if (IoU >= IoUTreshold)
{
tp++;
}
else
{
fp++;
fn++;
}
}
else
{
Console.WriteLine("No Match");
fn++;
}
}
else
{
if (DetectionAreas[match.elementList[1]] < smallAreaToIgnore)
{
continue;
}
Console.WriteLine("No Groundtruth");
fp++;
}
}
}
double ap = (double)tp / (double)(tp + fp + fn);
double prec = (double)tp / (double)(tp + fp);
double recl = (double)tp / (double)(tp + fn);
string ret = String.Format("TP: {0}, FP: {1}, FN {2}, AP, {3}, Precision, {4}, Recall, {5}", tp, fp, fn, ap, prec, recl);
Console.WriteLine(ret);
return(ret);
}
protected void SubmitButton_Click(object sender, EventArgs e)
{
DateTime SubmitTime = DateTime.Now;
DateTime PageLoadTime = Convert.ToDateTime(Hidden_PageLoadTime.Value);
int count;
bool isValidCount = int.TryParse(CountTextBox.Text, out count);
if (!isValidCount)
{
ErrorLabel.Text = "Error : Ivalid Count.";
ErrorLabel.ForeColor = System.Drawing.Color.Red;
ErrorLabel.Font.Bold = true;
}
else if (count < 0)
{
ErrorLabel.Text = "Error : Count cannot be less than zero.";
ErrorLabel.ForeColor = System.Drawing.Color.Red;
ErrorLabel.Font.Bold = true;
}
else
{
ErrorLabel.Text = "";
SatyamTaskTableEntry taskEntry = JSonUtils.ConvertJSonToObject <SatyamTaskTableEntry>(Hidden_TaskEntryString.Value);
SatyamResult result = new SatyamResult();
result.TaskParametersString = taskEntry.TaskParametersString;
result.TaskStartTime = PageLoadTime;
result.TaskEndTime = SubmitTime;
result.TaskTableEntryID = taskEntry.ID;
AmazonTaskResultInfo amazonInfo = new AmazonTaskResultInfo();
amazonInfo.AssignmentID = "";
amazonInfo.WorkerID = "";
amazonInfo.HITID = "";
result.amazonInfo = amazonInfo;
ObjectCountingResult sresult = new ObjectCountingResult();
sresult.Count = count;
string sresultString = JSonUtils.ConvertObjectToJSon <ObjectCountingResult>(sresult);
result.TaskResult = sresultString;
string resultString = JSonUtils.ConvertObjectToJSon <SatyamResult>(result);
SatyamResultsTableAccess resultdb = new SatyamResultsTableAccess();
resultdb.AddEntry(taskEntry.JobTemplateType, taskEntry.UserID, taskEntry.JobGUID, resultString, taskEntry.ID, PageLoadTime, SubmitTime);
resultdb.close();
//SatyamTaskTableAccess taskDB = new SatyamTaskTableAccess();
//taskDB.IncrementDoneScore(taskEntry.ID);
bool NotDone = getNewRandomJob();
if (NotDone == false)
{
Response.Redirect("AllJobsDone.aspx");
}
}
}
private void storeResult()
{
DateTime SubmitTime = DateTime.Now;
DateTime PageLoadTime = Convert.ToDateTime(Hidden_PageLoadTime.Value);
SatyamTaskTableEntry taskEntry = JSonUtils.ConvertJSonToObject <SatyamTaskTableEntry>(Hidden_TaskEntryString.Value);
string tracksString = TracksOutput_Hidden.Value;
Console.WriteLine(tracksString);
string urlList = Hidden_ImageURLList.Value;
string[] fields = urlList.Split(',');
DateTime start = DateTime.MinValue;
List <DateTime> frameTimes = new List <DateTime>();
//double frameTimeSpanInMiliseconds = (Convert.ToDouble(Hidden_ChunkDuration.Value) / (double)fields.Length) * 1000;
double frameTimeSpanInMiliseconds = (double)(1000) / Convert.ToDouble(fps_Hidden.Value);
for (int i = 0; i < fields.Length; i++)
{
DateTime t;
t = start.AddMilliseconds(frameTimeSpanInMiliseconds * i);
frameTimes.Add(t);
}
string s = Raw_VATIC_DVA_Crowdsourced_Track_Collection.Raw_VATIC_DVA_Crowdsourced_Track_Collection_ToTrackStrings(tracksString, frameTimes);
SatyamResult result = new SatyamResult();
result.TaskParametersString = taskEntry.TaskParametersString;
result.TaskStartTime = PageLoadTime;
result.TaskEndTime = SubmitTime;
result.TaskTableEntryID = taskEntry.ID;
AmazonTaskResultInfo amazonInfo = new AmazonTaskResultInfo();
amazonInfo.AssignmentID = "";
amazonInfo.WorkerID = "";
amazonInfo.HITID = "";
result.amazonInfo = amazonInfo;
result.TaskResult = s;
string resultString = JSonUtils.ConvertObjectToJSon <SatyamResult>(result);
SatyamResultsTableAccess resultdb = new SatyamResultsTableAccess();
resultdb.AddEntry(taskEntry.JobTemplateType, taskEntry.UserID, taskEntry.JobGUID, resultString, taskEntry.ID, PageLoadTime, SubmitTime);
resultdb.close();
//SatyamTaskTableAccess taskDB = new SatyamTaskTableAccess();
//taskDB.IncrementDoneScore(taskEntry.ID);
bool NotDone = getNewRandomJob();
if (NotDone == false)
{
Response.Redirect("AllJobsDone.aspx");
}
}
protected void SubmitButton_Click(object sender, EventArgs e)
{
DateTime SubmitTime = DateTime.Now;
DateTime PageLoadTime = Convert.ToDateTime(Hidden_PageLoadTime.Value);
SatyamTaskTableEntry taskEntry = JSonUtils.ConvertJSonToObject <SatyamTaskTableEntry>(Hidden_TaskEntryString.Value);
SatyamResult result = new SatyamResult();
result.TaskParametersString = taskEntry.TaskParametersString;
result.TaskStartTime = PageLoadTime;
result.TaskEndTime = SubmitTime;
result.TaskTableEntryID = taskEntry.ID;
AmazonTaskResultInfo amazonInfo = new AmazonTaskResultInfo();
amazonInfo.AssignmentID = Hidden_AmazonAssignmentID.Value;
amazonInfo.WorkerID = Hidden_AmazonWorkerID.Value;
amazonInfo.HITID = Hidden_HITID.Value;
amazonInfo.PricePerHIT = Convert.ToDouble(Hidden_Price.Value);
result.amazonInfo = amazonInfo;
result.TaskResult = Hidden_Result.Value;
string resultString = JSonUtils.ConvertObjectToJSon <SatyamResult>(result);
SatyamResultsTableAccess resultdb = new SatyamResultsTableAccess();
resultdb.AddEntry(taskEntry.JobTemplateType, taskEntry.UserID, taskEntry.JobGUID, resultString, taskEntry.ID, PageLoadTime, SubmitTime);
resultdb.close();
SatyamTaskTableManagement.UpdateResultNumber(taskEntry.ID);
//SatyamTaskTableAccess taskDB = new SatyamTaskTableAccess();
//taskDB.IncrementDoneScore(taskEntry.ID);
int noDone = Convert.ToInt32(Hidden_NoImagesDone.Value);
noDone++;
Hidden_NoImagesDone.Value = noDone.ToString();
int noTasksPerJob = Convert.ToInt32(Hidden_TasksPerJob.Value);
bool NotDone = false;
if (noDone < noTasksPerJob)
{
NotDone = getNewRandomJob();
}
if (NotDone == false)
{
if (!Testing)
{
SatyamAmazonHITTableAccess HITdb = new SatyamAmazonHITTableAccess();
string HITID = result.amazonInfo.HITID;
HITdb.UpdateStatusByHITID(HITID, HitStatus.submitted);
HITdb.close();
AmazonMTurkNotification.submitAmazonTurkHit(result.amazonInfo.AssignmentID, result.amazonInfo.WorkerID, false);
}
Response.Redirect("MTurkTaskDonePage.aspx");
}
}
public static void AggregateWithParameterAndValidateSatyamImageNetClassificationResultByGUID(string guid,
int MinResults = TaskConstants.SINGLE_OBJECT_LABLING_MTURK_MIN_RESULTS_TO_AGGREGATE,
int MaxResults = TaskConstants.SINGLE_OBJECT_LABLING_MTURK_MAX_RESULTS_TO_AGGREGATE,
double MajorityThreshold = TaskConstants.SINGLE_OBJECT_LABLING_MTURK_MAJORITY_THRESHOLD,
string confusingImageListFilePath = null)
{
string configString = "Min_" + MinResults + "_Max_" + MaxResults + "_Thresh_" + MajorityThreshold;
Console.WriteLine("Aggregating for param set " + configString);
int noTerminatedTasks = 0;
SatyamResultsTableAccess resultsDB = new SatyamResultsTableAccess();
List <SatyamResultsTableEntry> entries = resultsDB.getEntriesByGUIDOrderByID(guid);
resultsDB.close();
//SortedDictionary<DateTime, List<SatyamResultsTableEntry>> entriesBySubmitTime =
// SatyamResultValidation.SortResultsBySubmitTime_OneResultPerTurkerPerTask(entries);
SortedDictionary <DateTime, List <SatyamResultsTableEntry> > entriesBySubmitTime =
SatyamResultValidationToolKit.SortResultsBySubmitTime(entries);
Dictionary <string, ConfusingReason> imageBlackListReason = new Dictionary <string, ConfusingReason>();
if (confusingImageListFilePath != null)
{
imageBlackListReason = getConfusingImageList(confusingImageListFilePath);
}
//Dictionary<int, List<SingleObjectLabelingResult>> ResultsPerTask = new Dictionary<int, List<SingleObjectLabelingResult>>();
Dictionary <int, List <SatyamResultsTableEntry> > ResultsPerTask = new Dictionary <int, List <SatyamResultsTableEntry> >();
List <int> aggregatedTasks = new List <int>();
int noTotalConverged = 0;
int noCorrect = 0;
SortedDictionary <string, Dictionary <string, int> > confusionMatrix_res_groundtruth = new SortedDictionary <string, Dictionary <string, int> >();
SortedDictionary <string, Dictionary <string, int> > confusionMatrix_groundtruth_res = new SortedDictionary <string, Dictionary <string, int> >();
StringBuilder s = new StringBuilder();
s.Append("<!DOCTYPE html>\n");
s.Append("<html>\n");
s.Append("<body>\n");
String title = String.Format("<h1>Job GUID {0} Incorrect Result Summary</h1>\n", guid);
s.Append(title);
Dictionary <int, int> noResultsNeededForAggregation = SatyamResultsAnalysis.getNoResultsNeededForAggregationFromLog(configString, guid);
Dictionary <int, int> noResultsNeededForAggregation_new = new Dictionary <int, int>();
List <SatyamAggregatedResultsTableEntry> aggEntries = new List <SatyamAggregatedResultsTableEntry>();
foreach (DateTime t in entriesBySubmitTime.Keys)
{
foreach (SatyamResultsTableEntry entry in entriesBySubmitTime[t])
{
//SatyamResultsTableEntry entry = entries[i];
SatyamResult satyamResult = JSonUtils.ConvertJSonToObject <SatyamResult>(entry.ResultString);
SatyamTask task = JSonUtils.ConvertJSonToObject <SatyamTask>(satyamResult.TaskParametersString);
SatyamJob job = task.jobEntry;
string fileName = URIUtilities.filenameFromURI(task.SatyamURI);
string imageCategoryName = fileName.Split('_')[0];
if (IsBlackListed(task.SatyamURI, imageBlackListReason))
{
continue;
}
int taskEntryID = entry.SatyamTaskTableEntryID;
if (aggregatedTasks.Contains(taskEntryID))
{
continue;
}
if (!ResultsPerTask.ContainsKey(taskEntryID))
{
ResultsPerTask.Add(taskEntryID, new List <SatyamResultsTableEntry>());
}
//ResultEntriesPerTask[taskEntryID].Add(JSonUtils.ConvertJSonToObject<SingleObjectLabelingResult>(satyamResult.TaskResult));
ResultsPerTask[taskEntryID].Add(entry);
// check log if enough results are collected
if (noResultsNeededForAggregation != null && noResultsNeededForAggregation.ContainsKey(taskEntryID) &&
ResultsPerTask[taskEntryID].Count < noResultsNeededForAggregation[taskEntryID])
{
continue;
}
string aggResultString = SingleObjectLabelingAggregator.GetAggregatedResultString(ResultsPerTask[taskEntryID], MinResults, MaxResults, MajorityThreshold);
//SingleObjectLabelingAggregatedResult aggResult = SingleObjectLabelingAggregator.getAggregatedResult(ResultEntriesPerTask[taskEntryID], MinResults, MaxResults, MajorityThreshold);
if (aggResultString == null)
{
continue;
}
SatyamAggregatedResultsTableEntry aggEntry = new SatyamAggregatedResultsTableEntry();
aggEntry.JobGUID = ResultsPerTask[taskEntryID][0].JobGUID;
aggEntry.JobTemplateType = ResultsPerTask[taskEntryID][0].JobTemplateType;
aggEntry.SatyamTaskTableEntryID = ResultsPerTask[taskEntryID][0].SatyamTaskTableEntryID;
aggEntry.UserID = ResultsPerTask[taskEntryID][0].UserID;
aggEntry.ResultString = aggResultString;
/// aggregation happen
// record logs
if (noResultsNeededForAggregation == null || !noResultsNeededForAggregation.ContainsKey(taskEntryID))
{
noResultsNeededForAggregation_new.Add(taskEntryID, ResultsPerTask[taskEntryID].Count);
}
///
//if (aggResult.Category == "None of the Above")
//{
// continue;
//}
aggEntries.Add(aggEntry);
SatyamSaveAggregatedDataSatyam data = new SatyamSaveAggregatedDataSatyam(aggEntry);
String resultString = data.AggregatedResultString;
SingleObjectLabelingAggregatedResult aggResult = JSonUtils.ConvertJSonToObject <SingleObjectLabelingAggregatedResult>(resultString);
if (!confusionMatrix_res_groundtruth.ContainsKey(aggResult.Category))
{
confusionMatrix_res_groundtruth.Add(aggResult.Category, new Dictionary <string, int>()
{
{ GroundTruth[imageCategoryName], 0 }
});
}
else
{
if (!confusionMatrix_res_groundtruth[aggResult.Category].ContainsKey(GroundTruth[imageCategoryName]))
{
confusionMatrix_res_groundtruth[aggResult.Category].Add(GroundTruth[imageCategoryName], 0);
}
}
if (!confusionMatrix_groundtruth_res.ContainsKey(GroundTruth[imageCategoryName]))
{
confusionMatrix_groundtruth_res.Add(GroundTruth[imageCategoryName], new Dictionary <string, int>()
{
{ aggResult.Category, 0 }
});
}
else
{
if (!confusionMatrix_groundtruth_res[GroundTruth[imageCategoryName]].ContainsKey(aggResult.Category))
{
confusionMatrix_groundtruth_res[GroundTruth[imageCategoryName]].Add(aggResult.Category, 0);
}
}
if (aggResult.Category.Equals(GroundTruth[imageCategoryName], StringComparison.InvariantCultureIgnoreCase))
{
noCorrect++;
}
else
{
//Console.WriteLine("{0}, Groundtruth: {1}, Aggregated: {2}, Votes: {3}",
// fileName, GroundTruth[imageCategoryName], aggResult.Category,
// JSonUtils.ConvertObjectToJSon(aggResult.metaData));
String record = String.Format("<p>{0}, Groundtruth: {1}, Aggregated: {2}, Votes: {3}</p>\n",
fileName, GroundTruth[imageCategoryName], aggResult.Category,
JSonUtils.ConvertObjectToJSon(aggResult.metaData));
String img = String.Format("<img src=\"{0}\" >\n", task.SatyamURI);
s.Append(record);
s.Append(img);
}
noTotalConverged++;
if (ResultsPerTask[taskEntryID].Count >= MaxResults)
{
noTerminatedTasks++;
}
confusionMatrix_res_groundtruth[aggResult.Category][GroundTruth[imageCategoryName]]++;
confusionMatrix_groundtruth_res[GroundTruth[imageCategoryName]][aggResult.Category]++;
aggregatedTasks.Add(taskEntryID);
}
}
SatyamResultsAnalysis.RecordAggregationLog(noResultsNeededForAggregation_new, configString, guid);
s.Append("</body>\n");
s.Append("</html>\n");
string dataToBeSaved = s.ToString();
SatyamJobStorageAccountAccess storage = new SatyamJobStorageAccountAccess();
string FileName = String.Format("AggregatedIncorrectResults-{0}_Min{1}Max{2}Thresh{3}.html", guid, MinResults, MaxResults, MajorityThreshold);
storage.SaveATextFile("singleobjectlabeling", guid, FileName, dataToBeSaved);
s.Clear();
s.Append("<!DOCTYPE html>\n");
s.Append("<html>\n");
s.Append("<body>\n");
s.Append(title);
string resultSummary = String.Format("<p>Result: {0}/{1}, precision: {2}</p>\n", noCorrect, noTotalConverged, (double)noCorrect / noTotalConverged);
resultSummary += String.Format("<p>Terminated: {0}, Not Enough Results: {1}</p>\n", noTerminatedTasks, ResultsPerTask.Count - noTotalConverged);
Console.WriteLine(resultSummary);
s.Append(resultSummary);
// write the confusion matrix
s.Append("<p>");
String row = "\t\t";
foreach (string resultCategory in confusionMatrix_res_groundtruth.Keys)
{
row += resultCategory + "\t";
}
row += "<br>\n";
s.Append(row);
Console.WriteLine(row);
string matString = "";
foreach (string groundTruthCategory in confusionMatrix_groundtruth_res.Keys)
{
row = groundTruthCategory + "\t";
foreach (string resultCategory in confusionMatrix_res_groundtruth.Keys)
{
if (confusionMatrix_groundtruth_res[groundTruthCategory].ContainsKey(resultCategory))
{
row += confusionMatrix_groundtruth_res[groundTruthCategory][resultCategory].ToString();
}
else
{
row += "0";
}
row += "\t";
}
row += "<br>\n";
s.Append(row);
Console.WriteLine(row);
matString += row;
}
s.Append("</p>\n");
s.Append("</body>\n");
s.Append("</html>\n");
string summaryToBeSaved = s.ToString();
FileName = String.Format("Aggregated_Summary-{0}_Min{1}Max{2}Thresh{3}.html", guid, MinResults, MaxResults, MajorityThreshold);
storage.SaveATextFile("singleobjectlabeling", guid, FileName, summaryToBeSaved);
/// local file
string outputString = String.Format("{0} {1} {2} {3} {4} {5}\n", configString, noCorrect, noTotalConverged, (double)noCorrect / noTotalConverged, noTerminatedTasks, ResultsPerTask.Count - noTotalConverged);
string outputfile = DirectoryConstants.defaultTempDirectory + guid + "\\resultSummary.txt";
File.AppendAllText(outputfile, outputString);
string outputmatFile = DirectoryConstants.defaultTempDirectory + guid + "\\" + configString + "_mat.txt";
File.WriteAllText(outputmatFile, matString);
//for (double prob = 0; prob < 1;prob +=0.2)
//{
// SatyamResultsAnalysis.AnalyzeApprovalRate(aggEntries, entries, guid, configString, anotherChanceProbablity: prob);
//}
//for (double ratio = 0; ratio < 1; ratio += 0.2)
//{
// SatyamResultsAnalysis.AnalyzeApprovalRate(aggEntries, entriesBySubmitTime, noResultsNeededForAggregation, noResultsNeededForAggregation_new, guid, configString, approvalRatioThreshold: ratio);
//}
SatyamResultsAnalysis.AggregationAnalysis(aggEntries, entriesBySubmitTime, noResultsNeededForAggregation, noResultsNeededForAggregation_new, guid, configString);
}
public static void AggregateWithParameterAndValidateObjectCountingResultByGUID(string guid,
//bool saveImage = false,
bool prepareTrainingSet = false, string outputDirectory = null,
int MinHeight = TaskConstants.OBJECT_COUNTING_VALIDATION_MIN_HEIGHT,
int MaxOcclusion = TaskConstants.OBJECT_COUNTING_VALIDATION_MAX_OCCLUSION,
double Max_Truncation = TaskConstants.OBJECT_COUNTING_VALIDATION_MIN_TRUNCATION,
int MinResults = TaskConstants.OBJECT_COUNTING_MTURK_MIN_RESULTS_TO_AGGREGATE,
int MaxResults = TaskConstants.OBJECT_COUNTING_MTURK_MAX_RESULTS_TO_AGGREGATE,
double MAX_ABSOLUTE_COUNT_DEVIATION_LOWERBOUND = TaskConstants.OBJECT_COUNTING_MTURK_MAX_ABSOLUTE_COUNT_DEVIATION_LOWERBOUND,
double MAX_DEVIATION_FRACTION = TaskConstants.OBJECT_COUNTING_MTURK_MAX_DEVIATION_FRACTION,
double SUPER_MAJORITY_VALUE = TaskConstants.OBJECT_COUNTING_MTURK_SUPER_MAJORITY_VALUE
)
{
if (!Directory.Exists(DirectoryConstants.defaultTempDirectory + guid))
{
Directory.CreateDirectory(DirectoryConstants.defaultTempDirectory + guid);
}
string configString = "Min_" + MinResults + "_Max_" + MaxResults + "_Dev_" + MAX_DEVIATION_FRACTION + "_Majority_" + SUPER_MAJORITY_VALUE;
Console.WriteLine("Aggregating for param set " + configString);
int noTerminatedTasks = 0;
SatyamResultsTableAccess resultsDB = new SatyamResultsTableAccess();
List <SatyamResultsTableEntry> entries = resultsDB.getEntriesByGUIDOrderByID(guid);
resultsDB.close();
SortedDictionary <DateTime, List <SatyamResultsTableEntry> > entriesBySubmitTime = SatyamResultValidationToolKit.SortResultsBySubmitTime(entries);
int noTotalConverged = 0;
//int noCorrect = 0;
Dictionary <int, List <SatyamResultsTableEntry> > ResultsPerTask = new Dictionary <int, List <SatyamResultsTableEntry> >();
List <int> aggregatedTasks = new List <int>();
List <SatyamAggregatedResultsTableEntry> aggEntries = new List <SatyamAggregatedResultsTableEntry>();
Dictionary <int, int> noResultsNeededForAggregation = SatyamResultsAnalysis.getNoResultsNeededForAggregationFromLog(configString, guid);
Dictionary <int, int> noResultsNeededForAggregation_new = new Dictionary <int, int>();
// play back by time
foreach (DateTime t in entriesBySubmitTime.Keys)
{
//Console.WriteLine("Processing Results of time: {0}", t);
foreach (SatyamResultsTableEntry entry in entriesBySubmitTime[t])
{
SatyamResult satyamResult = JSonUtils.ConvertJSonToObject <SatyamResult>(entry.ResultString);
SatyamTask task = JSonUtils.ConvertJSonToObject <SatyamTask>(satyamResult.TaskParametersString);
SatyamJob job = task.jobEntry;
string fileName = URIUtilities.filenameFromURI(task.SatyamURI);
int taskEntryID = entry.SatyamTaskTableEntryID;
if (aggregatedTasks.Contains(taskEntryID))
{
continue;
}
if (!ResultsPerTask.ContainsKey(taskEntryID))
{
ResultsPerTask.Add(taskEntryID, new List <SatyamResultsTableEntry>());
}
ResultsPerTask[taskEntryID].Add(entry);
// check log if enough results are collected
if (noResultsNeededForAggregation != null && noResultsNeededForAggregation.ContainsKey(taskEntryID) &&
ResultsPerTask[taskEntryID].Count < noResultsNeededForAggregation[taskEntryID])
{
continue;
}
string aggResultString = ObjectCountingAggregator.GetAggregatedResultString(ResultsPerTask[taskEntryID], MinResults, MaxResults, MAX_ABSOLUTE_COUNT_DEVIATION_LOWERBOUND, MAX_DEVIATION_FRACTION, SUPER_MAJORITY_VALUE);
if (aggResultString == null)
{
continue;
}
SatyamAggregatedResultsTableEntry aggEntry = new SatyamAggregatedResultsTableEntry();
aggEntry.JobGUID = ResultsPerTask[taskEntryID][0].JobGUID;
aggEntry.JobTemplateType = ResultsPerTask[taskEntryID][0].JobTemplateType;
aggEntry.SatyamTaskTableEntryID = taskEntryID;
aggEntry.UserID = ResultsPerTask[taskEntryID][0].UserID;
aggEntry.ResultString = aggResultString;
/// aggregation happen
/// // record logs
// record logs
if (noResultsNeededForAggregation == null || !noResultsNeededForAggregation.ContainsKey(taskEntryID))
{
noResultsNeededForAggregation_new.Add(taskEntryID, ResultsPerTask[taskEntryID].Count);
}
aggEntries.Add(aggEntry);
noTotalConverged++;
if (ResultsPerTask[taskEntryID].Count >= MaxResults)
{
noTerminatedTasks++;
}
aggregatedTasks.Add(taskEntryID);
}
}
SatyamResultsAnalysis.RecordAggregationLog(noResultsNeededForAggregation_new, configString, guid);
double totalError, totalGroundtruth;
//ValidateSatyamKITTIObjectCountingAggregationResult(aggEntries, out totalError, out totalGroundtruth,
// MinHeight, MaxOcclusion, Max_Truncation);
ValidateSatyamCARPKObjectCountingAggregationResult(aggEntries, out totalError, out totalGroundtruth);
Console.WriteLine("noTotalConverged {0}", noTotalConverged);
Console.WriteLine("noTerminatedTasks {0}", noTerminatedTasks);
/// local file
string outputString = String.Format("{0} {1} {2} {3} {4} {5} {6}\n", configString, totalError, totalGroundtruth, totalError / totalGroundtruth, noTotalConverged, noTerminatedTasks, ResultsPerTask.Count - noTotalConverged);
string outputfile = DirectoryConstants.defaultTempDirectory + guid + "\\resultSummary.txt";
File.AppendAllText(outputfile, outputString);
string approvalString = configString + "_PayDev_" + TaskConstants.OBJECT_COUNTING_MTURK_MAX_ABSOLUTE_COUNT_DEVIATION_FOR_PAYMENT +
"_PayFrac_" + TaskConstants.OBJECT_COUNTING_MTURK_MAX_DEVIATION_FRACTION_FOR_PAYMENT;
//for (double prob = 0; prob < 1;prob +=0.2)
//{
// SatyamResultsAnalysis.AnalyzeApprovalRate(aggEntries, entries, guid, approvalString, anotherChanceProbablity: prob);
//}
//for (double ratio = 0; ratio < 1; ratio += 0.2)
//{
// SatyamResultsAnalysis.AnalyzeApprovalRate(aggEntries, entriesBySubmitTime, noResultsNeededForAggregation, noResultsNeededForAggregation_new, guid, configString, approvalRatioThreshold: ratio);
//}
SatyamResultsAnalysis.AggregationAnalysis(aggEntries, entriesBySubmitTime, noResultsNeededForAggregation, noResultsNeededForAggregation_new, guid, configString);
}
public static void ValidateSameTaskSameWorkerMultipleResultsEffect(string jobGUID, string confusingImageListFilePath = null)
{
SatyamResultsTableAccess resultsDB = new SatyamResultsTableAccess();
List <SatyamResultsTableEntry> entries = resultsDB.getEntriesByGUIDOrderByID(jobGUID);
resultsDB.close();
Dictionary <int, int> noResultsPerTask = new Dictionary <int, int>();
Dictionary <int, int> noAcceptedResultsPerTask = new Dictionary <int, int>();
Dictionary <int, int> noPaidResultsPerTask = new Dictionary <int, int>();
Dictionary <int, double> moneyPaidPerTask = new Dictionary <int, double>();
SortedDictionary <int, Dictionary <string, List <string> > > ResultsPerWorkerPerTask = new SortedDictionary <int, Dictionary <string, List <string> > >();
SortedDictionary <int, Dictionary <string, int> > noDifferentDuplicateResultsPerWorkerPerTask = new SortedDictionary <int, Dictionary <string, int> >();
Dictionary <int, string> taskSatyamUri = new Dictionary <int, string>();
Dictionary <int, double> timeTakenTillAggregationPerTask = new Dictionary <int, double>();
Dictionary <int, DateTime> finalTaskEndTime = new Dictionary <int, DateTime>();
Dictionary <string, double> finalHITEndTime = new Dictionary <string, double>();
int totalResults = 0;
int noTasksWithDuplicateResultsFromSameWorker = 0;
int noTasksWithMixedResultsFromSameWorker = 0;
int noTaskWithDuplicateResultsAsMajority = 0;
int noTaskWhoseDuplicateResultsChangedAggregation = 0;
int noTaskWhoseDuplicateResultsChangedAggregationIncorrectly = 0;
int noCorrectDecisionsAmongDuplicates = 0;
//noCorrectDuplicateResultsOfSameWorkerSameTask = 0;
int noTasksWithCorrectDuplicateResultsOfSameWorkerSameTask = 0;
int noWorkerSwitchedToCorrect = 0;
int noWorkerSwitchedToCorrectAndMaintained = 0;
int noTasksWithWorkerSwitchedToCorrect = 0;
int noTasksWithWorkerSwitchedToCorrectAndMaintained = 0;
List <double> timeTakenPerResult = new List <double>();
List <double> acceptedTimeTakenPerResult = new List <double>();
List <int> SequenceNumberOfResultPerTask = new List <int>();
List <int> SequenceNumberOfResultPerWorker = new List <int>();
string JobGUID = entries[0].JobGUID;
Dictionary <string, int> noJobsPerWorker = new Dictionary <string, int>();
Dictionary <string, int> noAcceptedJobsPerWorker = new Dictionary <string, int>();
Dictionary <string, int> noPaidJobsPerWorker = new Dictionary <string, int>();
List <double> resultArrivalTimes = new List <double>();
totalResults = entries.Count;
///imagenet only
Dictionary <string, ConfusingReason> imageBlackListReason = new Dictionary <string, ConfusingReason>();
if (confusingImageListFilePath != null)
{
imageBlackListReason = getConfusingImageList(confusingImageListFilePath);
}
for (int i = 0; i < entries.Count; i++)
{
SatyamResultsTableEntry entry = entries[i];
SatyamResult satyamResult = JSonUtils.ConvertJSonToObject <SatyamResult>(entry.ResultString);
SatyamTask task = JSonUtils.ConvertJSonToObject <SatyamTask>(satyamResult.TaskParametersString);
SatyamJob job = task.jobEntry;
if (IsBlackListed(task.SatyamURI, imageBlackListReason))
{
continue;
}
int taskEntryID = entry.SatyamTaskTableEntryID;
if (!noResultsPerTask.ContainsKey(taskEntryID))
{
noResultsPerTask.Add(taskEntryID, 0);
noAcceptedResultsPerTask.Add(taskEntryID, 0);
noPaidResultsPerTask.Add(taskEntryID, 0);
moneyPaidPerTask.Add(taskEntryID, 0);
finalTaskEndTime.Add(taskEntryID, entry.SubmitTime);
ResultsPerWorkerPerTask.Add(taskEntryID, new Dictionary <string, List <string> >());
noDifferentDuplicateResultsPerWorkerPerTask.Add(taskEntryID, new Dictionary <string, int>());
taskSatyamUri.Add(taskEntryID, task.SatyamURI);
}
if (!ResultsPerWorkerPerTask[taskEntryID].ContainsKey(satyamResult.amazonInfo.WorkerID))
{
ResultsPerWorkerPerTask[taskEntryID].Add(satyamResult.amazonInfo.WorkerID, new List <string>());
noDifferentDuplicateResultsPerWorkerPerTask[taskEntryID].Add(satyamResult.amazonInfo.WorkerID, 0);
}
if (!ResultsPerWorkerPerTask[taskEntryID][satyamResult.amazonInfo.WorkerID].Contains(satyamResult.TaskResult))
{
noDifferentDuplicateResultsPerWorkerPerTask[taskEntryID][satyamResult.amazonInfo.WorkerID]++;
}
ResultsPerWorkerPerTask[taskEntryID][satyamResult.amazonInfo.WorkerID].Add(satyamResult.TaskResult);
}
///////////////////////////////////// Per Task Analysis //////////////////////////////////////
SortedDictionary <int, int> resultsPerTaskHistogram = new SortedDictionary <int, int>();
SortedDictionary <int, int> resultsAcceptedPerTaskHistogram = new SortedDictionary <int, int>();
SortedDictionary <int, int> resultsPaidPerTaskHistogram = new SortedDictionary <int, int>();
SortedDictionary <int, int> moneyPaidPerTaskHistogram = new SortedDictionary <int, int>(); //cents
SortedDictionary <int, int> ResultsPerWorkerPerTaskHistogram = new SortedDictionary <int, int>();
List <int> taskIDs = noResultsPerTask.Keys.ToList();
foreach (int taskID in taskIDs)
{
if (ResultsPerWorkerPerTask[taskID].Count != noResultsPerTask[taskID])
{
//has multiple results from same turker
noTasksWithDuplicateResultsFromSameWorker++;
// the aggregation result
List <SingleObjectLabelingResult> allResultsPerTask = new List <SingleObjectLabelingResult>();
SingleObjectLabelingAggregatedResult aggregatedResultPerTask = new SingleObjectLabelingAggregatedResult();
// the aggregation if without duplicate results
List <SingleObjectLabelingResult> OnlyFirstResultOfEachTurkerPerTask = new List <SingleObjectLabelingResult>();
SingleObjectLabelingAggregatedResult aggregatedOnlyFirstResultOfTurkerPerTask = new SingleObjectLabelingAggregatedResult();
foreach (List <string> ResultsStringsPerWorkerPerTask in ResultsPerWorkerPerTask[taskID].Values)
{
foreach (string resultString in ResultsStringsPerWorkerPerTask)
{
allResultsPerTask.Add(JSonUtils.ConvertJSonToObject <SingleObjectLabelingResult>(resultString));
}
OnlyFirstResultOfEachTurkerPerTask.Add(JSonUtils.ConvertJSonToObject <SingleObjectLabelingResult>(ResultsStringsPerWorkerPerTask[0]));
}
aggregatedResultPerTask = SingleObjectLabelingAggregator.getAggregatedResult(allResultsPerTask);
aggregatedOnlyFirstResultOfTurkerPerTask = SingleObjectLabelingAggregator.getAggregatedResult(OnlyFirstResultOfEachTurkerPerTask);
if (aggregatedResultPerTask != null)
{
if (aggregatedOnlyFirstResultOfTurkerPerTask == null ||
aggregatedResultPerTask.Category != aggregatedOnlyFirstResultOfTurkerPerTask.Category)
{
noTaskWhoseDuplicateResultsChangedAggregation++;
if (AggregatedResultEqualsGroundTruth(taskSatyamUri[taskID], JSonUtils.ConvertObjectToJSon(aggregatedResultPerTask)))
{
noTaskWhoseDuplicateResultsChangedAggregationIncorrectly++;
}
}
}
bool hasMixedResults = false;
bool hasCorrectResult = false;
bool atLeastOneShiftedFromIncorrectToCorrect = false;
bool atLeastOneShiftedFromIncorrectToCorrectAndMaintained = false;
foreach (List <string> ResultsStringsPerWorkerPerTask in ResultsPerWorkerPerTask[taskID].Values)
{
if (!ResultsPerWorkerPerTaskHistogram.ContainsKey(ResultsStringsPerWorkerPerTask.Count))
{
ResultsPerWorkerPerTaskHistogram.Add(ResultsStringsPerWorkerPerTask.Count, 0);
}
ResultsPerWorkerPerTaskHistogram[ResultsStringsPerWorkerPerTask.Count]++;
double superMajority = 0.6; // tunable
if (((double)ResultsStringsPerWorkerPerTask.Count + 1) / ((double)noResultsPerTask[taskID] + 2) > superMajority)
{
noTaskWithDuplicateResultsAsMajority++;
}
///////////// imagenet only //////////////////
if (ResultsStringsPerWorkerPerTask.Distinct().Count() > 1)
{
// multiple choices given
hasMixedResults = true;
bool incorrect = false;
bool switchToCorrect = false;
foreach (string resultString in ResultsStringsPerWorkerPerTask)
{
if (SingleObjectLabelingResultEqualsGroundTruth(taskSatyamUri[taskID], resultString))
{
hasCorrectResult = true;
noCorrectDecisionsAmongDuplicates++;
if (incorrect)
{
switchToCorrect = true;
}
incorrect = false;
}
else
{
incorrect = true;
}
}
if (switchToCorrect)
{
// been incorrect and swithed to correct
noWorkerSwitchedToCorrect++;
atLeastOneShiftedFromIncorrectToCorrect = true;
if (!incorrect)
{
//switch to correct and maintain till the end
noWorkerSwitchedToCorrectAndMaintained++;
atLeastOneShiftedFromIncorrectToCorrectAndMaintained = true;
}
}
}
////////////////////////////////////////////////
}
if (hasMixedResults)
{
noTasksWithMixedResultsFromSameWorker++;
}
if (hasCorrectResult)
{
noTasksWithCorrectDuplicateResultsOfSameWorkerSameTask++;
}
if (atLeastOneShiftedFromIncorrectToCorrect)
{
noTasksWithWorkerSwitchedToCorrect++;
}
if (atLeastOneShiftedFromIncorrectToCorrectAndMaintained)
{
noTasksWithWorkerSwitchedToCorrectAndMaintained++;
}
}
}
Console.WriteLine("DuplicateResultsHistogram");
foreach (int no in ResultsPerWorkerPerTaskHistogram.Keys)
{
Console.WriteLine("{0}, {1}", no, ResultsPerWorkerPerTaskHistogram[no]);
}
Console.WriteLine(
//"{0} images (in total {1}({2} of which are correct)) has duplicate results from same turker, \n" +
//"\t{3} images have >=1 workers making more than majority number of results.\n" +
//"\t{11} images aggregation is changed ({12} of which incorrectly) by duplicate results from same turker.\n" +
//"\t{4} images(In total {5} times) with duplicate results include mixed(>= 2) decisions from same worker.\n" +
"\t\t{6} images has correct decision among duplicate decisions.\n" +
"\t\t{7} images({8} times) a worker has switched from incorrect to correct choice\n" +
"\t\t\t{9} images({10} times) of which maintained the correct choice till the last time of their job on that image.",
//noTasksWithDuplicateResultsFromSameWorker, noDuplicateResultsFromSameWorker, noCorrectDecisionsAmongDuplicates,
//noTaskWithDuplicateResultsAsMajority,
//noTasksWithMixedResultsFromSameWorker, noDecisionChangeAmongDuplicateResultsOfSameWorkerSameTask,
//noTasksWithCorrectDuplicateResultsOfSameWorkerSameTask,
noTasksWithWorkerSwitchedToCorrect, noWorkerSwitchedToCorrect,
noTasksWithWorkerSwitchedToCorrectAndMaintained, noWorkerSwitchedToCorrectAndMaintained,
noTaskWhoseDuplicateResultsChangedAggregation, noTaskWhoseDuplicateResultsChangedAggregationIncorrectly
);
}
protected void SubmitButton_Click(object sender, EventArgs e)
{
DateTime SubmitTime = DateTime.Now;
DateTime PageLoadTime = Convert.ToDateTime(Hidden_PageLoadTime.Value);
int count;
bool isValidCount = int.TryParse(CountTextBox.Text, out count);
if (!isValidCount)
{
ErrorLabel.Text = "Error : Ivalid Count.";
ErrorLabel.ForeColor = System.Drawing.Color.Red;
ErrorLabel.Font.Bold = true;
}
else if (count < 0)
{
ErrorLabel.Text = "Error : Count cannot be less than zero.";
ErrorLabel.ForeColor = System.Drawing.Color.Red;
ErrorLabel.Font.Bold = true;
}
else
{
ErrorLabel.Text = "";
CountTextBox.Text = "";
SatyamTaskTableEntry taskEntry = JSonUtils.ConvertJSonToObject <SatyamTaskTableEntry>(Hidden_TaskEntryString.Value);
SatyamResult result = new SatyamResult();
result.TaskParametersString = taskEntry.TaskParametersString;
result.TaskStartTime = PageLoadTime;
result.TaskEndTime = SubmitTime;
result.TaskTableEntryID = taskEntry.ID;
AmazonTaskResultInfo amazonInfo = new AmazonTaskResultInfo();
amazonInfo.AssignmentID = Hidden_AmazonAssignmentID.Value;
amazonInfo.WorkerID = Hidden_AmazonWorkerID.Value;
amazonInfo.HITID = Hidden_HITID.Value;
amazonInfo.PricePerHIT = Convert.ToDouble(Hidden_Price.Value);
result.amazonInfo = amazonInfo;
ObjectCountingResult sresult = new ObjectCountingResult();
sresult.Count = count;
string sresultString = JSonUtils.ConvertObjectToJSon <ObjectCountingResult>(sresult);
result.TaskResult = sresultString;
string resultString = JSonUtils.ConvertObjectToJSon <SatyamResult>(result);
SatyamResultsTableAccess resultdb = new SatyamResultsTableAccess();
resultdb.AddEntry(taskEntry.JobTemplateType, taskEntry.UserID, taskEntry.JobGUID, resultString, taskEntry.ID, PageLoadTime, SubmitTime);
resultdb.close();
SatyamTaskTableManagement.UpdateResultNumber(taskEntry.ID);
//SatyamTaskTableAccess taskDB = new SatyamTaskTableAccess();
//taskDB.IncrementDoneScore(taskEntry.ID);
int noDone = Convert.ToInt32(Hidden_NoImagesDone.Value);
noDone++;
Hidden_NoImagesDone.Value = noDone.ToString();
bool NotDone = false;
int noTasksPerJob = Convert.ToInt32(Hidden_TasksPerJob.Value);
if (noDone < noTasksPerJob)
{
NotDone = getNewRandomJob();
}
if (NotDone == false)
{
if (!Testing)
{
SatyamAmazonHITTableAccess HITdb = new SatyamAmazonHITTableAccess();
string HITID = result.amazonInfo.HITID;
HITdb.UpdateStatusByHITID(HITID, HitStatus.submitted);
HITdb.close();
AmazonMTurkNotification.submitAmazonTurkHit(result.amazonInfo.AssignmentID, result.amazonInfo.WorkerID, false);
}
Response.Redirect("MTurkTaskDonePage.aspx");
}
}
}
public static MultiObjectTrackingResult stitchAllChunksOfOneVideo(List <SatyamAggregatedResultsTableEntry> entries, out List <string> ImageURLs, out int fps)
{
ImageURLs = new List <string>();
fps = 0;
if (entries.Count == 0)
{
return(null);
}
SatyamJobStorageAccountAccess satyamStorage = new SatyamJobStorageAccountAccess();
MultiObjectTrackingResult stitched = new MultiObjectTrackingResult();
int totalFrameCounts = 0;
// ensure the order is correct
SortedDictionary <int, SatyamAggregatedResultsTableEntry> sortedEntries = new SortedDictionary <int, SatyamAggregatedResultsTableEntry>();
List <int> idx = new List <int>();
for (int i = 0; i < entries.Count; i++)
{
SatyamAggregatedResultsTableEntry entry = entries[i];
SatyamAggregatedResult satyamAggResult = JSonUtils.ConvertJSonToObject <SatyamAggregatedResult>(entry.ResultString);
SatyamTask aggTask = JSonUtils.ConvertJSonToObject <SatyamTask>(satyamAggResult.TaskParameters);
string video = URIUtilities.localDirectoryNameFromURI(aggTask.SatyamURI);
string[] fields = video.Split('_');
int startingFrame = Convert.ToInt32(fields[fields.Length - 1]);
if (!sortedEntries.ContainsKey(startingFrame))
{
sortedEntries.Add(startingFrame, entries[i]);
idx.Add(startingFrame);
}
}
idx.Sort();
for (int i = 0; i < idx.Count; i++)
{
//SatyamAggregatedResultsTableEntry entry = entries[i];
SatyamAggregatedResultsTableEntry entry = sortedEntries[idx[i]];
SatyamAggregatedResult satyamAggResult = JSonUtils.ConvertJSonToObject <SatyamAggregatedResult>(entry.ResultString);
SatyamTask aggTask = JSonUtils.ConvertJSonToObject <SatyamTask>(satyamAggResult.TaskParameters);
MultiObjectTrackingSubmittedJob job = JSonUtils.ConvertJSonToObject <MultiObjectTrackingSubmittedJob>(aggTask.jobEntry.JobParameters);
MultiObjectTrackingAggregatedResult aggresult = JSonUtils.ConvertJSonToObject <MultiObjectTrackingAggregatedResult>(satyamAggResult.AggregatedResultString);
int noFramesOverlap = 0;
if (job.ChunkOverlap != 0.0)
{
noFramesOverlap = (int)(job.ChunkOverlap * job.FrameRate);
}
List <string> TraceURLs = satyamStorage.getURLListOfSubDirectoryByURL(aggTask.SatyamURI);
if (i == 0)
{
ImageURLs.AddRange(TraceURLs);
string[] names = aggTask.SatyamURI.Split('/');
fps = job.FrameRate;
stitched = aggresult.tracklets;
totalFrameCounts += TraceURLs.Count;
}
else
{
int noNewFrames = 0;
for (int j = noFramesOverlap; j < TraceURLs.Count; j++)
{
ImageURLs.Add(TraceURLs[j]);
noNewFrames++;
}
//stitched = stitchTwoTracesByOneFrameBoundingBoxes(stitched, aggresult.tracklets, totalFrameCounts, totalFrameCounts+ noNewFrames, noFramesOverlap,fps);
stitched = stitchTwoTracesByTubeletsOfOverlappingVideoChunk(stitched, aggresult.tracklets, totalFrameCounts, totalFrameCounts + noNewFrames, noFramesOverlap, fps);
totalFrameCounts += noNewFrames;
}
//debug
//generateVideoForEvaluation(ImageURLs, stitched, directoryName + "_" + i, videoName, fps);
}
return(stitched);
}
