public static void SaveResultVideosLocally(List <SatyamResultsTableEntry> entries, string directoryName, int fps = 10) { directoryName = directoryName + "\\Raw"; if (!Directory.Exists(directoryName)) { Directory.CreateDirectory(directoryName); } SatyamJobStorageAccountAccess satyamStorage = new SatyamJobStorageAccountAccess(); for (int i = 0; i < entries.Count; i++) { SatyamResultsTableEntry entry = entries[i]; SatyamResult satyamResult = JSonUtils.ConvertJSonToObject <SatyamResult>(entry.ResultString); SatyamTask task = JSonUtils.ConvertJSonToObject <SatyamTask>(satyamResult.TaskParametersString); MultiObjectTrackingSubmittedJob job = JSonUtils.ConvertJSonToObject <MultiObjectTrackingSubmittedJob>(task.jobEntry.JobParameters); string blobDir = URIUtilities.localDirectoryFullPathFromURI(task.SatyamURI); //VATIC_DVA_CrowdsourcedResult taskr = new VATIC_DVA_CrowdsourcedResult(satyamResult.TaskResult, task.SatyamURI, start, end, task.SatyamJobSubmissionsTableEntryID.ToString()); VATIC_DVA_CrowdsourcedResult taskr = MultiObjectTrackingAggregator.createVATIC_DVA_CrowdsourcedResultUsingSatyamBlobImageCount(satyamResult.TaskResult, blobDir, entry.ID.ToString(), job.FrameRate); MultiObjectTrackingResult res = taskr.getCompressedTracksInTimeSegment(); List <string> ImageURLs = satyamStorage.getURLListOfSpecificExtensionUnderSubDirectoryByURI(blobDir, new List <string>() { "jpg", "png" }); string videoName = URIUtilities.localDirectoryNameFromURI(blobDir) + "_" + URIUtilities.filenameFromURINoExtension(task.SatyamURI) + "_" + entry.ID; MultiObjectTrackingAnalyzer.generateVideoForEvaluation(ImageURLs, res, directoryName, videoName, job.FrameRate); } }
public static void SaveKITTIResultVideosLocally(List <SatyamResultsTableEntry> entries, string directoryName, int fps = 10) { directoryName = directoryName + "\\Raw"; if (!Directory.Exists(directoryName)) { Directory.CreateDirectory(directoryName); } SatyamJobStorageAccountAccess satyamStorage = new SatyamJobStorageAccountAccess(); for (int i = 0; i < entries.Count; i++) { SatyamResultsTableEntry entry = entries[i]; SatyamResult satyamResult = JSonUtils.ConvertJSonToObject <SatyamResult>(entry.ResultString); SatyamTask task = JSonUtils.ConvertJSonToObject <SatyamTask>(satyamResult.TaskParametersString); MultiObjectTrackingSubmittedJob job = JSonUtils.ConvertJSonToObject <MultiObjectTrackingSubmittedJob>(task.jobEntry.JobParameters); //VATIC_DVA_CrowdsourcedResult taskr = VATIC_DVA_CrowdsourcedResult.createVATIC_DVA_CrowdsourcedResultUsingSatyamBlobImageCount(satyamResult.TaskResult, task.SatyamURI, task.SatyamJobSubmissionsTableEntryID.ToString(), job.FrameRate); //MultiObjectTrackingResult res = taskr.getCompressedTracksInTimeSegment(); string[] names = task.SatyamURI.Split('/'); string videoName = names[names.Length - 2] + "_" + entry.ID; string[] fields = videoName.Split('_'); string videoSequence = fields[fields.Length - 5]; int startingFrame = Convert.ToInt32(fields[fields.Length - 2]); int maxChunkEndFrame = startingFrame + job.ChunkDuration * job.FrameRate; int noFrameOverlap = (int)(job.ChunkOverlap * job.FrameRate); if (startingFrame != 0) { startingFrame -= noFrameOverlap; } if (entry.JobGUID == "1e43a983-548d-4a2e-8161-5537eb985902") { videoSequence = getCorrectSequenceNo(videoSequence); } string videoFrameDir = DirectoryConstants.KITTITrackingImages + videoSequence; List <string> files = Directory.GetFiles(videoFrameDir).ToList(); List <string> ImageURLs = new List <string>(); for (int j = startingFrame; j < files.Count && j < maxChunkEndFrame; j++) { ImageURLs.Add(files[j]); } if (ImageURLs.Count == 0) { continue; } VATIC_DVA_CrowdsourcedResult taskr = new VATIC_DVA_CrowdsourcedResult(satyamResult.TaskResult, videoName, entry.ID.ToString(), ImageURLs.Count, job.FrameRate); MultiObjectTrackingResult res = taskr.getCompressedTracksInTimeSegment(); generateVideoForEvaluation(ImageURLs, res, directoryName, videoName, job.FrameRate); } }
public static string GetAggregatedResultString(List <SatyamResultsTableEntry> results) { if (results.Count == 0) { return(null); } string resultString = null; List <ImageSegmentationResult> resultList = new List <ImageSegmentationResult>(); List <string> WorkersPerTask = new List <string>(); SatyamResult res0 = JSonUtils.ConvertJSonToObject <SatyamResult>(results[0].ResultString); SatyamTask task = JSonUtils.ConvertJSonToObject <SatyamTask>(res0.TaskParametersString); string SatyamURL = task.SatyamURI; string guid = results[0].JobGUID; foreach (SatyamResultsTableEntry entry in results) { SatyamResult res = JSonUtils.ConvertJSonToObject <SatyamResult>(entry.ResultString); // remove duplicate workers result string workerID = res.amazonInfo.WorkerID; if (WorkersPerTask.Contains(workerID)) { continue; } string assignmentID = res.amazonInfo.AssignmentID; if (assignmentID == "" || assignmentID == "ASSIGNMENT_ID_NOT_AVAILABLE") { continue; } //enclose only non-duplicate results, one per each worker. if (workerID != "" && workerID != TaskConstants.AdminID) { // make a pass for test and admins WorkersPerTask.Add(workerID); } ImageSegmentationResult taskr = JSonUtils.ConvertJSonToObject <ImageSegmentationResult>(res.TaskResult); resultList.Add(taskr); } ImageSegmentationAggregatedResult r = getAggregatedResult(resultList, SatyamURL, guid); if (r != null) { string rString = JSonUtils.ConvertObjectToJSon <ImageSegmentationAggregatedResult>(r); SatyamAggregatedResult aggResult = new SatyamAggregatedResult(); aggResult.SatyamTaskTableEntryID = results[0].SatyamTaskTableEntryID; aggResult.AggregatedResultString = rString; SatyamResult res = JSonUtils.ConvertJSonToObject <SatyamResult>(results[0].ResultString); aggResult.TaskParameters = res.TaskParametersString; resultString = JSonUtils.ConvertObjectToJSon <SatyamAggregatedResult>(aggResult); } return(resultString); }
public static SortedDictionary <string, List <SatyamAggregatedResultsTableEntry> > GroupChunksByVideoName(List <SatyamAggregatedResultsTableEntry> entries) { SortedDictionary <string, List <SatyamAggregatedResultsTableEntry> > entriesPerVideo = new SortedDictionary <string, List <SatyamAggregatedResultsTableEntry> >(); int fps = 0; for (int i = 0; i < entries.Count; i++) { SatyamAggregatedResultsTableEntry entry = entries[i]; SatyamAggregatedResult satyamAggResult = JSonUtils.ConvertJSonToObject <SatyamAggregatedResult>(entry.ResultString); SatyamTask aggTask = JSonUtils.ConvertJSonToObject <SatyamTask>(satyamAggResult.TaskParameters); string video = URIUtilities.localDirectoryNameFromURI(aggTask.SatyamURI); string[] fields = video.Split('_'); string videoName = ""; for (int j = 0; j < fields.Length - 2; j++) { // remove the starting frame videoName += fields[j]; if (j == fields.Length - 3) { break; } videoName += '_'; MultiObjectTrackingSubmittedJob job = JSonUtils.ConvertJSonToObject <MultiObjectTrackingSubmittedJob>(aggTask.jobEntry.JobParameters); fps = job.FrameRate; } if (!entriesPerVideo.ContainsKey(videoName)) { entriesPerVideo.Add(videoName, new List <SatyamAggregatedResultsTableEntry>()); } entriesPerVideo[videoName].Add(entries[i]); } return(entriesPerVideo); }
public static void SaveAggregatedResultVideosLocally(List <SatyamAggregatedResultsTableEntry> entries, string directoryName) { directoryName = directoryName + "\\Aggregated"; if (!Directory.Exists(directoryName)) { Directory.CreateDirectory(directoryName); } SatyamJobStorageAccountAccess satyamStorage = new SatyamJobStorageAccountAccess(); for (int i = 0; i < entries.Count; i++) { SatyamAggregatedResultsTableEntry entry = entries[i]; SatyamAggregatedResult satyamAggResult = JSonUtils.ConvertJSonToObject <SatyamAggregatedResult>(entry.ResultString); SatyamTask aggTask = JSonUtils.ConvertJSonToObject <SatyamTask>(satyamAggResult.TaskParameters); MultiObjectTrackingSubmittedJob job = JSonUtils.ConvertJSonToObject <MultiObjectTrackingSubmittedJob>(aggTask.jobEntry.JobParameters); MultiObjectTrackingAggregatedResult aggresult = JSonUtils.ConvertJSonToObject <MultiObjectTrackingAggregatedResult>(satyamAggResult.AggregatedResultString); List <string> ImageURLs = satyamStorage.getURLListOfSubDirectoryByURL(aggTask.SatyamURI); string videoName = URIUtilities.localDirectoryNameFromURI(aggTask.SatyamURI) + "_" + entry.ID; generateVideoForEvaluation(ImageURLs, aggresult.tracklets, directoryName, videoName, job.FrameRate); } }
public static void SaveResultVideosLocally(List <SatyamResultsTableEntry> entries, string directoryName, int fps = 10) { directoryName = directoryName + "\\Raw"; if (!Directory.Exists(directoryName)) { Directory.CreateDirectory(directoryName); } SatyamJobStorageAccountAccess satyamStorage = new SatyamJobStorageAccountAccess(); for (int i = 0; i < entries.Count; i++) { SatyamResultsTableEntry entry = entries[i]; SatyamResult satyamResult = JSonUtils.ConvertJSonToObject <SatyamResult>(entry.ResultString); SatyamTask task = JSonUtils.ConvertJSonToObject <SatyamTask>(satyamResult.TaskParametersString); MultiObjectTrackingSubmittedJob job = JSonUtils.ConvertJSonToObject <MultiObjectTrackingSubmittedJob>(task.jobEntry.JobParameters); //VATIC_DVA_CrowdsourcedResult taskr = new VATIC_DVA_CrowdsourcedResult(satyamResult.TaskResult, task.SatyamURI, start, end, task.SatyamJobSubmissionsTableEntryID.ToString()); VATIC_DVA_CrowdsourcedResult taskr = MultiObjectTrackingAggregator.createVATIC_DVA_CrowdsourcedResultUsingSatyamBlobImageCount(satyamResult.TaskResult, task.SatyamURI, entry.ID.ToString(), job.FrameRate); MultiObjectTrackingResult res = taskr.getCompressedTracksInTimeSegment(); List <string> ImageURLs = satyamStorage.getURLListOfSubDirectoryByURL(task.SatyamURI); string[] names = task.SatyamURI.Split('/'); string videoName = names[names.Length - 2] + "_" + entry.ID; generateVideoForEvaluation(ImageURLs, res, directoryName, videoName, job.FrameRate); } }
public static void SaveResultOnlyText(string jobGUID) { string directoryName = DirectoryConstants.defaultTempDirectory + "\\" + jobGUID + "\\"; SatyamResultsTableAccess resultsDB = new SatyamResultsTableAccess(); List <SatyamResultsTableEntry> entries = resultsDB.getEntriesByGUID(jobGUID); resultsDB.close(); if (!Directory.Exists(directoryName)) { Directory.CreateDirectory(directoryName); } directoryName = directoryName + "\\Raw\\"; if (!Directory.Exists(directoryName)) { Directory.CreateDirectory(directoryName); } for (int i = 0; i < entries.Count; i++) { SatyamResultsTableEntry entry = entries[i]; SatyamResult satyamResult = JSonUtils.ConvertJSonToObject <SatyamResult>(entry.ResultString); SatyamTask task = JSonUtils.ConvertJSonToObject <SatyamTask>(satyamResult.TaskParametersString); SatyamJob job = task.jobEntry; string result = satyamResult.TaskResult; string ofilename = URIUtilities.filenameFromURI(task.SatyamURI); string[] fields = ofilename.Split('.'); string fileName = ""; for (int j = 0; j < fields.Count(); j++) { fileName += fields[j]; if (j == fields.Count() - 1) { break; } fileName += "."; } //string fileName = ofilename; fileName = fileName + "-Result"; fileName = fileName + "-" + entry.ID; Console.WriteLine("Saving " + fileName); string resultFile = directoryName + fileName + ".txt"; StreamWriter f = new System.IO.StreamWriter(resultFile); f.WriteLine(result); f.Close(); } }
private bool getNewRandomJob() { double price = 0; bool success = Double.TryParse(Hidden_Price.Value, out price); if (!success) { price = 0; } SatyamTaskTableAccess taskTableDB = new SatyamTaskTableAccess(); SatyamTaskTableEntry entry = null; if (SubmitButton.Enabled == true) { //entry = taskTableDB.getMinimumTriedEntryByTemplateAndPrice(TaskConstants.Counting_Image_MTurk, price); entry = taskTableDB.getMinimumTriedNewEntryForWorkerIDByTemplateAndPrice(Hidden_AmazonWorkerID.Value, TaskConstants.Counting_Image_MTurk, price); } else { entry = taskTableDB.getMinimumTriedEntryByTemplate(TaskConstants.Counting_Image_MTurk); } if (entry != null) { taskTableDB.IncrementDoneScore(entry.ID); SatyamTask task = JSonUtils.ConvertJSonToObject <SatyamTask>(entry.TaskParametersString); string uri = task.SatyamURI; DisplayImage.ImageUrl = uri; SatyamJob jobDefinitionEntry = task.jobEntry; ObjectCountingSubmittedJob job = JSonUtils.ConvertJSonToObject <ObjectCountingSubmittedJob>(jobDefinitionEntry.JobParameters); ObjectNameLabel.Text = job.ObjectName; ObjectNameLabel1.Text = job.ObjectName; Hidden_TasksPerJob.Value = jobDefinitionEntry.TasksPerJob.ToString(); if (job.Description != "") { DescriptionPanel.Visible = true; DescriptionTextPanel.Controls.Add(new LiteralControl(job.Description)); } Hidden_TaskEntryString.Value = JSonUtils.ConvertObjectToJSon <SatyamTaskTableEntry>(entry); Hidden_PageLoadTime.Value = DateTime.Now.ToString(); NoLabeled.Text = Hidden_NoImagesDone.Value; taskTableDB.close(); return(true); } else { taskTableDB.close(); return(false); } }
public static void SaveResultImagesLocally(List <SatyamResultsTableEntry> entries, string directoryName) { if (!Directory.Exists(directoryName)) { Directory.CreateDirectory(directoryName); } directoryName = directoryName + "\\Raw\\"; if (!Directory.Exists(directoryName)) { Directory.CreateDirectory(directoryName); } for (int i = 0; i < entries.Count; i++) { SatyamResultsTableEntry entry = entries[i]; SatyamResult satyamResult = JSonUtils.ConvertJSonToObject <SatyamResult>(entry.ResultString); SatyamTask task = JSonUtils.ConvertJSonToObject <SatyamTask>(satyamResult.TaskParametersString); SatyamJob job = task.jobEntry; string result = satyamResult.TaskResult; Image originalImage = ImageUtilities.getImageFromURI(task.SatyamURI); Image ResultImage = DrawResultStringOnImage(result, originalImage); string ofilename = URIUtilities.filenameFromURI(task.SatyamURI); string[] fields = ofilename.Split('.'); string fileName = ""; for (int j = 0; j < fields.Count(); j++) { fileName += fields[j]; if (j == fields.Count() - 1) { break; } fileName += "."; } //string fileName = ofilename; fileName = fileName + "-Result"; fileName = fileName + "-" + entry.ID; Console.WriteLine("Saving " + fileName); ImageUtilities.saveImage(ResultImage, directoryName, fileName); string resultFile = directoryName + fileName + ".txt"; StreamWriter f = new System.IO.StreamWriter(resultFile); f.WriteLine(result); f.Close(); } }
private MultiObjectLocalizationAndLabelingResult LoadTFServingResult(SatyamTaskTableEntry entry) { SatyamTask task = JSonUtils.ConvertJSonToObject <SatyamTask>(entry.TaskParametersString); string image_url = task.SatyamURI; SatyamJob jobDefinitionEntry = task.jobEntry; MultiObjectLocalizationAndLabelingSubmittedJob job = JSonUtils.ConvertJSonToObject <MultiObjectLocalizationAndLabelingSubmittedJob>(jobDefinitionEntry.JobParameters); List <string> categories = job.Categories; MultiObjectLocalizationAndLabelingResult res = TensorflowServingClient.GetImageDetectionResult(image_url, categories); return(res); }
public SatyamSaveAggregatedDataRequester(SatyamAggregatedResultsTableEntry entry) { SatyamAggregatedResult result = JSonUtils.ConvertJSonToObject <SatyamAggregatedResult>(entry.ResultString); SatyamTask task = JSonUtils.ConvertJSonToObject <SatyamTask>(result.TaskParameters); SatyamJob job = task.jobEntry; //OrignialURI = task.OriginalURI; JobGUID = entry.JobGUID; JobTemplateType = entry.JobTemplateType; UserID = entry.UserID; JobSubmitTime = job.JobSubmitTime; JobParameters = job.JobParameters; JobCompletionTime = DateTime.Now; AggregatedResultString = result.AggregatedResultString; }
private bool getNewRandomJob() { SatyamTaskTableAccess taskTableDB = new SatyamTaskTableAccess(); SatyamTaskTableEntry entry = taskTableDB.getMinimumTriedEntryByTemplate(TaskConstants.Segmentation_Image); taskTableDB.close(); if (entry == null) { return(false); } taskTableDB = new SatyamTaskTableAccess(); taskTableDB.IncrementDoneScore(entry.ID); taskTableDB.close(); SatyamTask task = JSonUtils.ConvertJSonToObject <SatyamTask>(entry.TaskParametersString); string uri = task.SatyamURI; TheImage.ImageUrl = uri; SatyamJob jobDefinitionEntry = task.jobEntry; ImageSegmentationSubmittedJob job = JSonUtils.ConvertJSonToObject <ImageSegmentationSubmittedJob>(jobDefinitionEntry.JobParameters); List <string> categories = job.Categories; CategorySelection_RadioButtonList.Items.Clear(); for (int i = 0; i < categories.Count; i++) { ListItem l = new ListItem(categories[i]); CategorySelection_RadioButtonList.Items.Add(l); } if (job.Description != "") { DescriptionPanel.Visible = true; DescriptionTextPanel.Controls.Add(new LiteralControl(job.Description)); } Hidden_BoundaryLines.Value = JSonUtils.ConvertObjectToJSon(job.BoundaryLines); Hidden_TaskEntryString.Value = JSonUtils.ConvertObjectToJSon <SatyamTaskTableEntry>(entry); Hidden_PageLoadTime.Value = DateTime.Now.ToString(); return(true); }
public static void SaveKITTIAggregatedResultVideosLocally(List <SatyamAggregatedResultsTableEntry> entries, string directoryName) { directoryName = directoryName + "\\Aggregated"; if (!Directory.Exists(directoryName)) { Directory.CreateDirectory(directoryName); } SatyamJobStorageAccountAccess satyamStorage = new SatyamJobStorageAccountAccess(); for (int i = 0; i < entries.Count; i++) { SatyamAggregatedResultsTableEntry entry = entries[i]; SatyamAggregatedResult satyamAggResult = JSonUtils.ConvertJSonToObject <SatyamAggregatedResult>(entry.ResultString); SatyamTask aggTask = JSonUtils.ConvertJSonToObject <SatyamTask>(satyamAggResult.TaskParameters); MultiObjectTrackingSubmittedJob job = JSonUtils.ConvertJSonToObject <MultiObjectTrackingSubmittedJob>(aggTask.jobEntry.JobParameters); MultiObjectTrackingAggregatedResult aggresult = JSonUtils.ConvertJSonToObject <MultiObjectTrackingAggregatedResult>(satyamAggResult.AggregatedResultString); string[] names = aggTask.SatyamURI.Split('/'); string videoName = names[names.Length - 2] + "_" + entry.ID; string[] fields = videoName.Split('_'); string videoSequence = fields[fields.Length - 5]; int startingFrame = Convert.ToInt32(fields[fields.Length - 2]); int maxChunkEndFrame = startingFrame + job.ChunkDuration * job.FrameRate; int noFrameOverlap = (int)(job.ChunkOverlap * job.FrameRate); if (startingFrame != 0) { startingFrame -= noFrameOverlap; } if (entry.JobGUID == "1e43a983-548d-4a2e-8161-5537eb985902") { videoSequence = getCorrectSequenceNo(videoSequence); } string videoFrameDir = DirectoryConstants.KITTITrackingImages + videoSequence; List <string> files = Directory.GetFiles(videoFrameDir).ToList(); List <string> ImageURLs = new List <string>(); for (int j = startingFrame; j < files.Count && j < maxChunkEndFrame; j++) { ImageURLs.Add(files[j]); } generateVideoForEvaluation(ImageURLs, aggresult.tracklets, directoryName, videoName, job.FrameRate); } }
public static MultiObjectLabelingAndLocalizationAnalysisPerJob Analyse(List <SatyamResultsTableEntry> entries) { MultiObjectLabelingAndLocalizationAnalysisPerJob ana = new MultiObjectLabelingAndLocalizationAnalysisPerJob(); ana.jobTemplateType = entries[0].JobTemplateType; for (int i = 0; i < entries.Count; i++) { SatyamResultsTableEntry entry = entries[i]; SatyamResult satyamResult = JSonUtils.ConvertJSonToObject <SatyamResult>(entry.ResultString); SatyamTask task = JSonUtils.ConvertJSonToObject <SatyamTask>(satyamResult.TaskParametersString); SatyamJob job = task.jobEntry; MultiObjectLocalizationAndLabelingResult res = JSonUtils.ConvertJSonToObject <MultiObjectLocalizationAndLabelingResult>(satyamResult.TaskResult); } return(ana); }
public static void AdjustTasksByGUID(string guid, double price) { SatyamTaskTableAccess taskDB = new SatyamTaskTableAccess(); List <SatyamTaskTableEntry> entries = taskDB.getAllEntriesByGUID(guid); taskDB.ClearByJobGUID(guid); /// update the price and the taskparam... foreach (SatyamTaskTableEntry entry in entries) { int taskID = entry.ID; SatyamTask task = JSonUtils.ConvertJSonToObject <SatyamTask>(entry.TaskParametersString); task.jobEntry.amazonHITInformation.Price = price; string newTaskParams = JSonUtils.ConvertObjectToJSon(task); taskDB.AddEntryWithSpecificID(taskID, entry.JobTemplateType, entry.UserID, entry.JobGUID, newTaskParams, entry.JobSubmitTime, price); } taskDB.close(); }
public static void SaveByGUIDRequester(string guid) { //get all the results of a GUID SatyamResultsTableAccess resultsDB = new SatyamResultsTableAccess(); List <SatyamResultsTableEntry> results = resultsDB.getEntriesByGUID(guid); if (results.Count == 0) { resultsDB.close(); return; } //get the azure info SatyamResult result = JSonUtils.ConvertJSonToObject <SatyamResult>(results[0].ResultString); SatyamTask task = JSonUtils.ConvertJSonToObject <SatyamTask>(result.TaskParametersString); SatyamJob job = task.jobEntry; string ConnectionString = job.azureInformation.AzureBlobStorageConnectionString; string ContainerName = job.azureInformation.AzureBlobStorageContainerName; string DirectoryName = job.azureInformation.AzureBlobStorageContainerDirectoryName; //create the data to be saved List <SatyamResultSaveDataRequester> savingDataList = new List <SatyamResultSaveDataRequester>(); StringBuilder s = new StringBuilder(); for (int i = 0; i < results.Count; i++) { SatyamResultSaveDataRequester data = new SatyamResultSaveDataRequester(results[i]); String jsonString = JSonUtils.ConvertObjectToJSon(data); s.Append(jsonString); if (i == results.Count - 1) { s.Append("\n"); } } string dataToBeSaved = s.ToString(); BlobContainerManager bcm = new BlobContainerManager(); bcm.Connect(ConnectionString); string FileName = "Results-" + results[0].JobGUID + ".txt"; bcm.SaveATextFile(ContainerName, DirectoryName, FileName, dataToBeSaved); resultsDB.close(); }
public static void TestGetGlobalStatistics() { Dictionary <string, int> workerSubmissions = new Dictionary <string, int>(); Dictionary <string, int> HitSubmissions = new Dictionary <string, int>(); Dictionary <int, int> TaskSubmissions = new Dictionary <int, int>(); SatyamResultsTableAccess resultsDB = new SatyamResultsTableAccess(); List <SatyamResultsTableEntry> entries = new List <SatyamResultsTableEntry>(); entries = (resultsDB.getAllEntries()); for (int i = 0; i < entries.Count; i++) { SatyamResultsTableEntry entry = entries[i]; SatyamResult satyamResult = JSonUtils.ConvertJSonToObject <SatyamResult>(entry.ResultString); SatyamTask task = JSonUtils.ConvertJSonToObject <SatyamTask>(satyamResult.TaskParametersString); SatyamJob job = task.jobEntry; string worker = satyamResult.amazonInfo.WorkerID; string hitID = satyamResult.amazonInfo.HITID; int taskID = entry.SatyamTaskTableEntryID; if (!workerSubmissions.ContainsKey(worker)) { workerSubmissions.Add(worker, 0); } workerSubmissions[worker]++; if (!HitSubmissions.ContainsKey(hitID)) { HitSubmissions.Add(hitID, 0); } HitSubmissions[hitID]++; if (!TaskSubmissions.ContainsKey(taskID)) { TaskSubmissions.Add(taskID, 0); } TaskSubmissions[taskID]++; } Console.WriteLine(workerSubmissions.Count); Console.WriteLine(HitSubmissions.Count); Console.WriteLine(TaskSubmissions.Count); }
public SatyamResultSaveDataRequester(SatyamResultsTableEntry entry) { string status = entry.Status; if (status == ResultStatus.accepted || status == ResultStatus.accepted_Paid || status == ResultStatus.accepted_NotPaid) { resultAccepted = true; } else { resultAccepted = false; } if (status == ResultStatus.rejected_Paid || status == ResultStatus.accepted_Paid) { resultPaid = true; } else { resultPaid = false; } SatyamResult result = JSonUtils.ConvertJSonToObject <SatyamResult>(entry.ResultString); TaskStartTime = result.TaskStartTime; TaskEndTime = result.TaskEndTime; JobGUID = entry.JobGUID; JobTemplateType = entry.JobTemplateType; SatyamTask task = JSonUtils.ConvertJSonToObject <SatyamTask>(result.TaskParametersString); //OrignialURI = task.OriginalURI; SatyamJob job = task.jobEntry; UserID = job.UserID; JobSubmitTime = job.JobSubmitTime; JobParameters = job.JobParameters; amazonInfo = new AmazonSavingInfo(); amazonInfo.AssignmentID = result.amazonInfo.AssignmentID; amazonInfo.pricePerHIT = result.amazonInfo.PricePerHIT; amazonInfo.TasksPerHIT = job.TasksPerJob; amazonInfo.WorkerID = result.amazonInfo.WorkerID; ResultString = result.TaskResult; }
public static void SaveAggregatedResultImagesLocally(List <SatyamAggregatedResultsTableEntry> entries, string directoryName) { if (!Directory.Exists(directoryName)) { Directory.CreateDirectory(directoryName); } directoryName = directoryName + "\\Aggregated"; if (!Directory.Exists(directoryName)) { Directory.CreateDirectory(directoryName); } for (int i = 0; i < entries.Count; i++) { SatyamAggregatedResultsTableEntry entry = entries[i]; SatyamAggregatedResult satyamResult = JSonUtils.ConvertJSonToObject <SatyamAggregatedResult>(entry.ResultString); SatyamTask task = JSonUtils.ConvertJSonToObject <SatyamTask>(satyamResult.TaskParameters); SatyamJob job = task.jobEntry; ImageSegmentationAggregatedResult res = JSonUtils.ConvertJSonToObject <ImageSegmentationAggregatedResult>(satyamResult.AggregatedResultString); //string fileName = URIUtilities.filenameFromURINoExtension(task.SatyamURI); //Image originalImage = ImageUtilities.getImageFromURI(task.SatyamURI); //Image ResultImage = DrawResultsOnImage(res.boxesAndCategories, originalImage); //fileName = fileName + "-AggregatedResult"; //ImageUtilities.saveImage(ResultImage, directoryName, fileName); WebClient wb = new WebClient(); Image im = Image.FromStream(wb.OpenRead(res.metaData.PNG_URL)); string fileName = URIUtilities.filenameFromURI(res.metaData.PNG_URL); //wb.DownloadFile(directoryName + "\\" + fileName, res.metaData.PNG_URL); im.Save(directoryName + "\\" + fileName); } }
private bool getNewRandomJob() { SatyamTaskTableAccess taskTableDB = new SatyamTaskTableAccess(); SatyamTaskTableEntry entry = taskTableDB.getMinimumTriedEntryByTemplate(TaskConstants.Counting_Video); if (entry != null) { taskTableDB.IncrementDoneScore(entry.ID); SatyamTask task = JSonUtils.ConvertJSonToObject <SatyamTask>(entry.TaskParametersString); string uri = task.SatyamURI; Hidden_VideoURL.Value = uri; SatyamJob jobDefinitionEntry = task.jobEntry; ObjectCountingSubmittedJob job = JSonUtils.ConvertJSonToObject <ObjectCountingSubmittedJob>(jobDefinitionEntry.JobParameters); ObjectNameLabel.Text = job.ObjectName; ObjectNameLabel1.Text = job.ObjectName; if (job.Description != "") { DescriptionPanel.Visible = true; DescriptionTextPanel.Controls.Add(new LiteralControl(job.Description)); } CountTextBox.Text = ""; Hidden_TaskEntryString.Value = JSonUtils.ConvertObjectToJSon <SatyamTaskTableEntry>(entry); Hidden_PageLoadTime.Value = DateTime.Now.ToString(); taskTableDB.close(); return(true); } else { taskTableDB.close(); return(false); } }
public static void SaveAggregatedResultVideosLocally(List <SatyamAggregatedResultsTableEntry> entries, string directoryName) { directoryName = directoryName + "\\Aggregated"; if (!Directory.Exists(directoryName)) { Directory.CreateDirectory(directoryName); } SatyamJobStorageAccountAccess satyamStorage = new SatyamJobStorageAccountAccess(); for (int i = 0; i < entries.Count; i++) { SatyamAggregatedResultsTableEntry entry = entries[i]; SatyamAggregatedResult satyamAggResult = JSonUtils.ConvertJSonToObject <SatyamAggregatedResult>(entry.ResultString); SatyamTask aggTask = JSonUtils.ConvertJSonToObject <SatyamTask>(satyamAggResult.TaskParameters); MultiObjectTrackingSubmittedJob job = JSonUtils.ConvertJSonToObject <MultiObjectTrackingSubmittedJob>(aggTask.jobEntry.JobParameters); TrackletLabelingAggregatedResult aggresult = JSonUtils.ConvertJSonToObject <TrackletLabelingAggregatedResult>(satyamAggResult.AggregatedResultString); WebClient wb = new WebClient(); Stream aggTrackStream = wb.OpenRead(aggresult.AggregatedTrackletsString_URL); StreamReader reader = new StreamReader(aggTrackStream); String aggTrackString = reader.ReadToEnd(); MultiObjectTrackingResult aggTracks = JSonUtils.ConvertJSonToObject <MultiObjectTrackingResult>(aggTrackString); string blobDir = URIUtilities.localDirectoryFullPathFromURI(aggTask.SatyamURI); List <string> ImageURLs = satyamStorage.getURLListOfSpecificExtensionUnderSubDirectoryByURI(blobDir, new List <string>() { "jpg", "png" }); string videoName = URIUtilities.localDirectoryNameFromURI(blobDir) + "_" + URIUtilities.filenameFromURINoExtension(aggTask.SatyamURI) + "_" + entry.ID; MultiObjectTrackingAnalyzer.generateVideoForEvaluation(ImageURLs, aggTracks, directoryName, videoName, job.FrameRate); } }
// If the results were not good enough, or aggregation method changed, retargeted, users might want to reopen the job // All we need to do: (IN STRICT ORDER) // clear all results back to inconclusive, // remove all aggregated results // restore the task table as it was for the guid, // change the job status back to launched, and // WARNING: // for safety, this has to be run atomically synchronously, without any parallel process. for now. public static void reopenJobForMoreResults(string guid) { SatyamJobSubmissionsTableAccess jobDB = new SatyamJobSubmissionsTableAccess(); SatyamJobSubmissionsTableAccessEntry jobEntry = jobDB.getEntryByJobGIUD(guid); jobDB.close(); if (jobEntry.JobStatus != JobStatus.completed) { Console.WriteLine("Job not completed yet!"); return; } SatyamAggregatedResultsTableAccess aggResultDB = new SatyamAggregatedResultsTableAccess(); bool delSuccess = aggResultDB.DeleteEntriesByGUID(guid); aggResultDB.close(); if (!delSuccess) { Console.WriteLine("Delete Agg Result DB Failed"); return; } SatyamResultsTableAccess resultDB = new SatyamResultsTableAccess(); if (!resultDB.UpdateStatusByGUID(guid, ResultStatus.inconclusive)) { Console.WriteLine("Update Result DB Failed"); //resultDB.close(); //return; } List <SatyamResultsTableEntry> results = resultDB.getEntriesByGUID(guid); resultDB.close(); Dictionary <int, SatyamTask> taskParamsByTaskID = new Dictionary <int, SatyamTask>(); foreach (SatyamResultsTableEntry result in results) { if (taskParamsByTaskID.ContainsKey(result.SatyamTaskTableEntryID)) { continue; } SatyamResult satyamRes = JSonUtils.ConvertJSonToObject <SatyamResult>(result.ResultString); SatyamTask task = JSonUtils.ConvertJSonToObject <SatyamTask>(satyamRes.TaskParametersString); taskParamsByTaskID.Add(result.SatyamTaskTableEntryID, task); } SatyamTaskTableAccess taskDB = new SatyamTaskTableAccess(); foreach (int taskID in taskParamsByTaskID.Keys) { SatyamTask task = taskParamsByTaskID[taskID]; SatyamJob job = task.jobEntry; bool suc = taskDB.AddEntryWithSpecificID(taskID, job.JobTemplateType, job.UserID, job.JobGUIDString, JSonUtils.ConvertObjectToJSon(task), job.JobSubmitTime, job.amazonHITInformation.Price); if (!suc) { Console.WriteLine("Update Task Table Failed"); taskDB.close(); return; } } taskDB.close(); jobDB = new SatyamJobSubmissionsTableAccess(); bool success = jobDB.UpdateEntryStatus(guid, JobStatus.launched); jobDB.close(); if (!success) { Console.WriteLine("Update Job Entry Failed"); return; } }
public static void AggregateWithParameterAndValidateSatyamVideoClassificationResultByGUID(string guid, int MinResults = TaskConstants.SINGLE_OBJECT_LABLING_MTURK_MIN_RESULTS_TO_AGGREGATE, int MaxResults = TaskConstants.SINGLE_OBJECT_LABLING_MTURK_MAX_RESULTS_TO_AGGREGATE, double MajorityThreshold = TaskConstants.SINGLE_OBJECT_LABLING_MTURK_MAJORITY_THRESHOLD, bool prepareDataForTraining = false, string outputDirectory = null, string confusingVideoListFilePath = null) { string configString = "Min_" + MinResults + "_Max_" + MaxResults + "_Thresh_" + MajorityThreshold; Console.WriteLine("Aggregating for param set " + configString); int noTerminatedTasks = 0; SatyamResultsTableAccess resultsDB = new SatyamResultsTableAccess(); List <SatyamResultsTableEntry> entries = resultsDB.getEntriesByGUIDOrderByID(guid); resultsDB.close(); SortedDictionary <DateTime, List <SatyamResultsTableEntry> > entriesBySubmitTime = SatyamResultValidationToolKit.SortResultsBySubmitTime(entries); int noTotalConverged = 0; int noCorrect = 0; Dictionary <int, List <SatyamResultsTableEntry> > ResultsPerTask = new Dictionary <int, List <SatyamResultsTableEntry> >(); List <int> aggregatedTasks = new List <int>(); SortedDictionary <string, Dictionary <string, int> > confusionMatrix_res_groundtruth = new SortedDictionary <string, Dictionary <string, int> >(); SortedDictionary <string, Dictionary <string, int> > confusionMatrix_groundtruth_res = new SortedDictionary <string, Dictionary <string, int> >(); List <SatyamAggregatedResultsTableEntry> aggEntries = new List <SatyamAggregatedResultsTableEntry>(); Dictionary <int, int> noResultsNeededForAggregation = SatyamResultsAnalysis.getNoResultsNeededForAggregationFromLog(configString, guid); Dictionary <int, int> noResultsNeededForAggregation_new = new Dictionary <int, int>(); foreach (DateTime t in entriesBySubmitTime.Keys) { //Console.WriteLine("Processing Results of time: {0}", t); foreach (SatyamResultsTableEntry entry in entriesBySubmitTime[t]) { SatyamResult satyamResult = JSonUtils.ConvertJSonToObject <SatyamResult>(entry.ResultString); SatyamTask task = JSonUtils.ConvertJSonToObject <SatyamTask>(satyamResult.TaskParametersString); SatyamJob job = task.jobEntry; string fileName = URIUtilities.filenameFromURI(task.SatyamURI); string VideoCategoryGroundTruth = getVideoCategoryFromFileName(fileName); int taskEntryID = entry.SatyamTaskTableEntryID; if (aggregatedTasks.Contains(taskEntryID)) { continue; } if (!ResultsPerTask.ContainsKey(taskEntryID)) { ResultsPerTask.Add(taskEntryID, new List <SatyamResultsTableEntry>()); } ResultsPerTask[taskEntryID].Add(entry); // check log if enough results are collected if (noResultsNeededForAggregation != null && noResultsNeededForAggregation.ContainsKey(taskEntryID) && ResultsPerTask[taskEntryID].Count < noResultsNeededForAggregation[taskEntryID]) { continue; } //SingleObjectLabelingAggregatedResult aggResult = SingleObjectLabelingAggregator.getAggregatedResult(ResultsPerTask[taskEntryID], MinResults, MaxResult, MajorityThreshold); string aggResultString = SingleObjectLabelingAggregator.GetAggregatedResultString(ResultsPerTask[taskEntryID], MinResults, MaxResults, MajorityThreshold); if (aggResultString == null) { continue; } SatyamAggregatedResultsTableEntry aggEntry = new SatyamAggregatedResultsTableEntry(); aggEntry.JobGUID = ResultsPerTask[taskEntryID][0].JobGUID; aggEntry.JobTemplateType = ResultsPerTask[taskEntryID][0].JobTemplateType; aggEntry.SatyamTaskTableEntryID = taskEntryID; aggEntry.UserID = ResultsPerTask[taskEntryID][0].UserID; aggEntry.ResultString = aggResultString; /// aggregation happen // record logs if (noResultsNeededForAggregation == null || !noResultsNeededForAggregation.ContainsKey(taskEntryID)) { noResultsNeededForAggregation_new.Add(taskEntryID, ResultsPerTask[taskEntryID].Count); } aggEntries.Add(aggEntry); noTotalConverged++; if (ResultsPerTask[taskEntryID].Count >= MaxResults) { noTerminatedTasks++; } aggregatedTasks.Add(taskEntryID); } } SatyamResultsAnalysis.RecordAggregationLog(noResultsNeededForAggregation_new, configString, guid); string outputmatFile = DirectoryConstants.defaultTempDirectory + guid + "\\" + configString + "_mat.txt"; EvaluateAndPrintConfusionMatrixOfAggregatedResultEntries(aggEntries, outputmatFile, out noCorrect, prepareDataForTraining, outputDirectory); Console.WriteLine("noTotalConverged {0}", noTotalConverged); Console.WriteLine("noTerminatedTasks {0}", noTerminatedTasks); Console.WriteLine("Result: {0}/{1}, precision: {2}", noCorrect, noTotalConverged, (double)noCorrect / noTotalConverged); /// local file string outputString = String.Format("{0} {1} {2} {3} {4} {5}\n", configString, noCorrect, noTotalConverged, (double)noCorrect / noTotalConverged, noTerminatedTasks, ResultsPerTask.Count - noTotalConverged); string outputfile = DirectoryConstants.defaultTempDirectory + guid + "\\resultSummary.txt"; File.AppendAllText(outputfile, outputString); //for (double prob = 0; prob < 1;prob +=0.2) //{ // SatyamResultsAnalysis.AnalyzeApprovalRate(aggEntries, entries, guid, configString, anotherChanceProbablity: prob); //} //for (double ratio = 0; ratio < 1; ratio += 0.2) //{ // SatyamResultsAnalysis.AnalyzeApprovalRate(aggEntries, entriesBySubmitTime, noResultsNeededForAggregation, noResultsNeededForAggregation_new, guid, configString, approvalRatioThreshold: ratio); //} SatyamResultsAnalysis.AggregationAnalysis(aggEntries, entriesBySubmitTime, noResultsNeededForAggregation, noResultsNeededForAggregation_new, guid, configString); }
private bool getNewRandomJob() { SatyamTaskTableAccess taskTableDB = new SatyamTaskTableAccess(); SatyamTaskTableEntry entry = taskTableDB.getMinimumTriedEntryByTemplate(TaskConstants.Tracking); if (entry != null) { taskTableDB.IncrementDoneScore(entry.ID); SatyamTask task = JSonUtils.ConvertJSonToObject <SatyamTask>(entry.TaskParametersString); SatyamJobStorageAccountAccess satyamStorage = new SatyamJobStorageAccountAccess(); List <string> ImageURLs = satyamStorage.getURLListOfSubDirectoryByURL(task.SatyamURI); //string urls = ""; //for (int i=0;i<ImageURLs.Count;i++) //{ // urls += ImageURLs[i]; // if (i == ImageURLs.Count - 1) break; // urls += ','; //} Hidden_ImageURLList.Value = ObjectsToStrings.ListString(ImageURLs, ','); SatyamJob jobDefinitionEntry = task.jobEntry; MultiObjectTrackingSubmittedJob job = JSonUtils.ConvertJSonToObject <MultiObjectTrackingSubmittedJob>(jobDefinitionEntry.JobParameters); Dictionary <string, List <string> > subcategories = job.Categories; List <string> categories = subcategories.Keys.ToList(); //CategorySelection_RakdioButtonList.Items.Clear(); for (int i = 0; i < categories.Count; i++) { ListItem l = new ListItem(categories[i]); //CategorySelection_RadioButtonList.Items.Add(l); } if (job.Description != "") { //DescriptionPanel.Visible = true; //DescriptionTextPanel.Controls.Add(new LiteralControl(job.Description)); } //Hidden_BoundaryLines.Value = JSonUtils.ConvertObjectToJSon(job.BoundaryLines); Hidden_TaskEntryString.Value = JSonUtils.ConvertObjectToJSon <SatyamTaskTableEntry>(entry); Hidden_PageLoadTime.Value = DateTime.Now.ToString(); // pass parameters from old template Slug_Hidden.Value = "null"; Start_Hidden.Value = "0"; Stop_Hidden.Value = (ImageURLs.Count - 1).ToString(); Skip_Hidden.Value = "0"; PerObject_Hidden.Value = "0.1"; Completion_Hidden.Value = "0.5"; BlowRadius_Hidden.Value = "0"; JobId_Hidden.Value = "1"; LabelString_Hidden.Value = ObjectsToStrings.ListString(categories.ToList(), ','); Attributes_Hidden.Value = ObjectsToStrings.DictionaryStringListString(subcategories, ',', ':', '_'); Training_Hidden.Value = "0"; fps_Hidden.Value = job.FrameRate.ToString(); Hidden_ChunkDuration.Value = job.ChunkDuration.ToString(); var web = new WebClient(); System.Drawing.Image x = System.Drawing.Image.FromStream(web.OpenRead(ImageURLs[0])); ImageWidth_Hidden.Value = x.Width.ToString(); ImageHeight_Hidden.Value = x.Height.ToString(); // image boundary for now //string[] region = new string[] { "0-0-1242-0-1242-375-0-375-0-0" }; string[] region = new string[] { "0-0-" + x.Width + "-0-" + x.Width + "-" + x.Height + "-0-" + x.Height + "-0-0" }; RegionString_Hidden.Value = ObjectsToStrings.ListString(region, ','); taskTableDB.close(); return(true); } else { taskTableDB.close(); return(false); } }
public static void AggregateWithParameter(string guid, int MinResults = TaskConstants.TRACKLET_LABELING_MTURK_MIN_RESULTS_TO_AGGREGATE, int MaxResults = TaskConstants.TRACKLET_LABELING_MTURK_MAX_RESULTS_TO_AGGREGATE, double boxToleranceThreshold = TaskConstants.TRACKLET_LABELING_BOX_DEVIATION_THRESHOLD, double ObjectCoverageApprovalThresholdPerVideo = TaskConstants.TRACKLET_LABELING_APPROVALRATIO_PER_VIDEO, double BoxCoverageApprovalThresholdPerTrack = TaskConstants.TRACKLET_LABELING_APPROVALRATIO_PER_TRACK, int consensusNumber = TaskConstants.TRACKLET_LABELING_MIN_RESULTS_FOR_CONSENSUS, double minTubeletIoUSimilarityThreshold = TaskConstants.TRACKLET_LABELING_MIN_TUBELET_SIMILARITY_THRESHOLD, double attributeMajority = TaskConstants.TRACKLET_LABELING_MTURK_ATTRIBUTE_MAJORITY_THRESHOLD, bool allFalseAttributeInvalid = false ) { string configString = "Min_" + MinResults + "_Max_" + MaxResults + "_IoU_" + minTubeletIoUSimilarityThreshold + "_Ratio_" + ObjectCoverageApprovalThresholdPerVideo; Console.WriteLine("Aggregating for " + guid + " with param set " + configString); SatyamResultsTableAccess resultsDB = new SatyamResultsTableAccess(); List <SatyamResultsTableEntry> entries = resultsDB.getEntriesByGUID(guid); resultsDB.close(); SortedDictionary <DateTime, List <SatyamResultsTableEntry> > entriesBySubmitTime = SatyamResultValidationToolKit.SortResultsBySubmitTime_OneResultPerTurkerPerTask(entries); Dictionary <int, List <MultiObjectTrackingResult> > ResultsPerTask = new Dictionary <int, List <MultiObjectTrackingResult> >(); List <int> aggregatedTasks = new List <int>(); int noTotalConverged = 0; //int noCorrect = 0; int noTerminatedTasks = 0; List <SatyamAggregatedResultsTableEntry> aggEntries = new List <SatyamAggregatedResultsTableEntry>(); Dictionary <int, int> noResultsNeededForAggregation = SatyamResultsAnalysis.getNoResultsNeededForAggregationFromLog(configString, guid); Dictionary <int, int> noResultsNeededForAggregation_new = new Dictionary <int, int>(); // play back by time Dictionary <int, List <string> > WorkersPerTask = new Dictionary <int, List <string> >(); foreach (DateTime t in entriesBySubmitTime.Keys) { //Console.WriteLine("Processing Results of time: {0}", t); List <SatyamResultsTableEntry> ResultEntries = entriesBySubmitTime[t]; foreach (SatyamResultsTableEntry entry in ResultEntries) { SatyamResult satyamResult = JSonUtils.ConvertJSonToObject <SatyamResult>(entry.ResultString); SatyamTask task = JSonUtils.ConvertJSonToObject <SatyamTask>(satyamResult.TaskParametersString); MultiObjectTrackingSubmittedJob job = JSonUtils.ConvertJSonToObject <MultiObjectTrackingSubmittedJob>(task.jobEntry.JobParameters); string fileName = URIUtilities.filenameFromURINoExtension(task.SatyamURI); int taskEntryID = entry.SatyamTaskTableEntryID; if (aggregatedTasks.Contains(taskEntryID)) { continue; } if (!ResultsPerTask.ContainsKey(taskEntryID)) { ResultsPerTask.Add(taskEntryID, new List <MultiObjectTrackingResult>()); WorkersPerTask.Add(taskEntryID, new List <string>()); } // remove duplicate workers result string workerID = satyamResult.amazonInfo.WorkerID; if (WorkersPerTask[taskEntryID].Contains(workerID)) { continue; } //enclose only non-duplicate results, one per each worker. WorkersPerTask[taskEntryID].Add(workerID); string videoName = URIUtilities.localDirectoryNameFromURI(task.SatyamURI); string[] fields = videoName.Split('_'); int startingFrame = Convert.ToInt32(fields[fields.Length - 1]); int maxChunkEndFrame = startingFrame + job.ChunkDuration * job.FrameRate; int noFrameOverlap = (int)(job.ChunkOverlap * job.FrameRate); if (startingFrame != 0) { startingFrame -= noFrameOverlap; } string blobDir = URIUtilities.localDirectoryFullPathFromURI(task.SatyamURI); VATIC_DVA_CrowdsourcedResult taskr = TrackletLabelingAggregator.createVATIC_DVA_CrowdsourcedResultUsingSatyamBlobImageCount(satyamResult.TaskResult, blobDir, entry.SatyamTaskTableEntryID.ToString(), job.FrameRate); //MultiObjectTrackingResult result_tmp = vatic_tmp.getCompressedTracksInTimeSegment(); //VATIC_DVA_CrowdsourcedResult taskr = new VATIC_DVA_CrowdsourcedResult(satyamResult.TaskResult, videoName, entry.ID.ToString(), ImageURLs.Count, job.FrameRate); MultiObjectTrackingResult res = taskr.getCompressedTracksInTimeSegment(); if (allFalseAttributeInvalid) { if (TrackletLabelingAggregator.AllAttributeAllFalse(res)) { continue; } } ResultsPerTask[taskEntryID].Add(res); // check log if enough results are collected if (noResultsNeededForAggregation != null && noResultsNeededForAggregation.ContainsKey(taskEntryID) && ResultsPerTask[taskEntryID].Count < noResultsNeededForAggregation[taskEntryID]) { continue; } // hack for masters int tempMin = MinResults; if (TaskConstants.masterGUIDs.Contains(entry.JobGUID)) { tempMin = 1; } TrackletLabelingAggregatedResult aggResult = TrackletLabelingAggregator.getAggregatedResult(ResultsPerTask[taskEntryID], guid, task.SatyamURI, job.FrameRate, job.BoundaryLines, MinResults, MaxResults, boxToleranceThreshold, ObjectCoverageApprovalThresholdPerVideo, BoxCoverageApprovalThresholdPerTrack, consensusNumber, minTubeletIoUSimilarityThreshold, attributeMajority); if (aggResult == null) { continue; } //////////////// aggregation happen // record logs if (noResultsNeededForAggregation == null || !noResultsNeededForAggregation.ContainsKey(taskEntryID)) { noResultsNeededForAggregation_new.Add(taskEntryID, ResultsPerTask[taskEntryID].Count); } aggregatedTasks.Add(taskEntryID); noTotalConverged++; if (ResultsPerTask[taskEntryID].Count >= MaxResults) { noTerminatedTasks++; } SatyamAggregatedResult SatyamAggResult = new SatyamAggregatedResult(); SatyamAggResult.SatyamTaskTableEntryID = taskEntryID; SatyamAggResult.AggregatedResultString = JSonUtils.ConvertObjectToJSon <TrackletLabelingAggregatedResult>(aggResult); SatyamAggResult.TaskParameters = JSonUtils.ConvertJSonToObject <SatyamResult>(entry.ResultString).TaskParametersString; SatyamAggregatedResultsTableEntry aggEntry = new SatyamAggregatedResultsTableEntry(); aggEntry.SatyamTaskTableEntryID = taskEntryID; aggEntry.JobGUID = entry.JobGUID; aggEntry.ResultString = JSonUtils.ConvertObjectToJSon <SatyamAggregatedResult>(SatyamAggResult); aggEntries.Add(aggEntry); } } Console.WriteLine("Total_Aggregated_Tasks: {0}", noTotalConverged); Console.WriteLine("Total_Terminated_Tasks: {0}", noTerminatedTasks); SatyamResultsAnalysis.RecordAggregationLog(noResultsNeededForAggregation_new, configString, guid); TrackletLabelingAnalyzer.GroupEntriesByVideoNameAndStitchAndSaveAggregatedResultVideosLocally(aggEntries, DirectoryConstants.defaultTempDirectory + guid); }
private bool getNewRandomJob() { double price = 0; bool success = Double.TryParse(Hidden_Price.Value, out price); if (!success) { price = 0; } SatyamTaskTableAccess taskTableDB = new SatyamTaskTableAccess(); SatyamTaskTableEntry entry = null; if (Submit_Button.Enabled == true) { //entry = taskTableDB.getMinimumTriedNewEntryForWorkerIDByTemplateAndPrice(Hidden_AmazonWorkerID.Value, // TaskConstants.TrackletLabeling_MTurk, price); entry = taskTableDB.getTopKNewEntryForWorkerIDByTemplateAndPrice(50, Hidden_AmazonWorkerID.Value, TaskConstants.TrackletLabeling_MTurk, price); } else { entry = taskTableDB.getMinimumTriedEntryByTemplate(TaskConstants.TrackletLabeling_MTurk); } taskTableDB.close(); if (entry == null) { return(false); } taskTableDB = new SatyamTaskTableAccess(); taskTableDB.IncrementDoneScore(entry.ID); taskTableDB.close(); SatyamTask task = JSonUtils.ConvertJSonToObject <SatyamTask>(entry.TaskParametersString); SatyamJobStorageAccountAccess satyamStorage = new SatyamJobStorageAccountAccess(); string videoDir = URIUtilities.localDirectoryFullPathFromURI(task.SatyamURI); List <string> ImageURLs = satyamStorage.getURLListOfSpecificExtensionUnderSubDirectoryByURI(videoDir, new List <string>() { "jpg" }); string annotationFilePath = task.SatyamURI; //string urls = ""; //for (int i=0;i<ImageURLs.Count;i++) //{ // urls += ImageURLs[i]; // if (i == ImageURLs.Count - 1) break; // urls += ','; //} Hidden_ImageURLList.Value = ObjectsToStrings.ListString(ImageURLs, ','); SatyamJob jobDefinitionEntry = task.jobEntry; MultiObjectTrackingSubmittedJob job = JSonUtils.ConvertJSonToObject <MultiObjectTrackingSubmittedJob>(jobDefinitionEntry.JobParameters); Dictionary <string, List <string> > subcategories = job.Categories; List <string> categories = subcategories.Keys.ToList(); //CategorySelection_RakdioButtonList.Items.Clear(); for (int i = 0; i < categories.Count; i++) { ListItem l = new ListItem(categories[i]); //CategorySelection_RadioButtonList.Items.Add(l); } if (job.Description != "") { //DescriptionPanel.Visible = true; //DescriptionTextPanel.Controls.Add(new LiteralControl(job.Description)); } //Hidden_BoundaryLines.Value = JSonUtils.ConvertObjectToJSon(job.BoundaryLines); Hidden_TaskEntryString.Value = JSonUtils.ConvertObjectToJSon <SatyamTaskTableEntry>(entry); Hidden_PageLoadTime.Value = DateTime.Now.ToString(); // pass parameters from old template Slug_Hidden.Value = "null"; Start_Hidden.Value = "0"; Stop_Hidden.Value = (ImageURLs.Count - 1).ToString(); Skip_Hidden.Value = "0"; PerObject_Hidden.Value = "0.1"; Completion_Hidden.Value = "0.5"; BlowRadius_Hidden.Value = "0"; JobId_Hidden.Value = "1"; LabelString_Hidden.Value = ObjectsToStrings.ListString(categories.ToList(), ','); Attributes_Hidden.Value = ObjectsToStrings.DictionaryStringListString(subcategories, ',', ':', '_'); Training_Hidden.Value = "0"; fps_Hidden.Value = job.FrameRate.ToString(); Hidden_ChunkDuration.Value = job.ChunkDuration.ToString(); var web = new WebClient(); System.Drawing.Image x = System.Drawing.Image.FromStream(web.OpenRead(ImageURLs[0])); ImageWidth_Hidden.Value = x.Width.ToString(); ImageHeight_Hidden.Value = x.Height.ToString(); // image boundary for now //string[] region = new string[] { "0-0-1242-0-1242-375-0-375-0-0" }; string[] region = new string[] { "0-0-" + x.Width + "-0-" + x.Width + "-" + x.Height + "-0-" + x.Height + "-0-0" }; RegionString_Hidden.Value = ObjectsToStrings.ListString(region, ','); // temp test List <VATIC_Tracklet> prevTracesTemp = new List <VATIC_Tracklet>(); WebClient client = new WebClient(); Stream stream = client.OpenRead(annotationFilePath); StreamReader reader = new StreamReader(stream); List <string> trace = new List <string>(); while (reader.Peek() >= 0) { string content = reader.ReadLine(); trace.Add(content); } Dictionary <string, VATIC_Tracklet> tracklets = VATIC_Tracklet.ReadTrackletsFromVIRAT(trace); foreach (string id in tracklets.Keys) { //string output = JSonUtils.ConvertObjectToJSon(tracklets[id]); prevTracesTemp.Add(tracklets[id]); } string output = JSonUtils.ConvertObjectToJSon(prevTracesTemp); PreviousTrackString_Hidden.Value = output; return(true); }
private bool getNewRandomJob() { double price = 0; bool success = Double.TryParse(Hidden_Price.Value, out price); if (!success) { price = 0; } SatyamTaskTableAccess taskTableDB = new SatyamTaskTableAccess(); SatyamTaskTableEntry entry = null; if (SubmitButton.Enabled == true) { //entry = taskTableDB.getMinimumTriedEntryByTemplateAndPrice("SINGLE_OBJECT_LABLING_MTURK", price); entry = taskTableDB.getMinimumTriedNewEntryForWorkerIDByTemplateAndPrice(Hidden_AmazonWorkerID.Value, TaskConstants.Classification_Video_MTurk, price); } else { entry = taskTableDB.getMinimumTriedEntryByTemplate(TaskConstants.Classification_Video_MTurk); } if (entry != null) { taskTableDB.IncrementDoneScore(entry.ID); SatyamTask task = JSonUtils.ConvertJSonToObject <SatyamTask>(entry.TaskParametersString); string uri = task.SatyamURI; Hidden_VideoURL.Value = uri; SatyamJob jobDefinitionEntry = task.jobEntry; SingleObjectLabelingSubmittedJob job = JSonUtils.ConvertJSonToObject <SingleObjectLabelingSubmittedJob>(jobDefinitionEntry.JobParameters); List <string> categories = job.Categories; CategorySelection_RadioButtonList.Items.Clear(); for (int i = 0; i < categories.Count; i++) { ListItem l = new ListItem(categories[i]); CategorySelection_RadioButtonList.Items.Add(l); } if (job.Description != "") { DescriptionPanel.Visible = true; DescriptionTextPanel.Controls.Add(new LiteralControl(job.Description)); //DescriptionLabel.Text = job.Description; } Hidden_TasksPerJob.Value = jobDefinitionEntry.TasksPerJob.ToString(); Hidden_TaskEntryString.Value = JSonUtils.ConvertObjectToJSon <SatyamTaskTableEntry>(entry); Hidden_PageLoadTime.Value = DateTime.Now.ToString(); NoLabeled.Text = Hidden_NoImagesDone.Value; taskTableDB.close(); return(true); } else { taskTableDB.close(); return(false); } }
//public static Dictionary<string, int> getRemainingHITsFinishedTasksFromResultDB(string guid, int taskPerHIT) //{ // SatyamResultsTableAccess resultDB = new SatyamResultsTableAccess(); // List<SatyamResultsTableEntry> entries = resultDB.getEntriesByGUID(guid); // Dictionary<string, int> noResultPerHITID = new Dictionary<string, int>(); // //int noFinishedHIT = 0; // foreach (SatyamResultsTableEntry entry in entries) // { // string HITId = JSonUtils.ConvertJSonToObject<SatyamResult>(entry.ResultString).amazonInfo.HITID; // if (!noResultPerHITID.ContainsKey(HITId)) // { // noResultPerHITID.Add(HITId,1); // } // else // { // noResultPerHITID[HITId]++; // //if (noResultPerHITID[HITId] == taskPerHIT) // //{ // // noFinishedHIT++; // //} // } // } // return noResultPerHITID; //} //makes sure that there are enough HITS in Amazon to ensure that jobs will be done public static void LaunchAmazonHITsFromTaskTable(int max = int.MaxValue) { //Group them by jobguids since each job can have many tasks Dictionary <string, List <SatyamTaskTableEntry> > TasksByGUID = OrganizeMTurkTasksByGUID(); if (TasksByGUID == null) //there is nothing to do!! { return; } //after running the aggregator, the number of tasks in each guid <= no of HITS pending //so if there are less HITS pending we launch more and if there are more we leave them alone until the job is done SatyamAmazonHITTableAccess hitDB = new SatyamAmazonHITTableAccess(); SatyamJobSubmissionsTableAccess jobDB = new SatyamJobSubmissionsTableAccess(); //now check how manyy hits are pending and launch to ensure that there are enough hits in amazon foreach (KeyValuePair <string, List <SatyamTaskTableEntry> > entry in TasksByGUID) { String jobGUID = entry.Key; SatyamTask task = JSonUtils.ConvertJSonToObject <SatyamTask>(entry.Value[0].TaskParametersString); SatyamJob job = task.jobEntry; //// filter task templates -- temporary filter //if (job.JobTemplateType == TaskConstants.Segmentation_Image_MTurk) continue; string AmazonAccessKeyID = job.amazonHITInformation.AmazonAccessKeyID; string AmazonSecretAccessKeyID = job.amazonHITInformation.AmazonSecretAccessKeyID; string JobTemplateType = entry.Value[0].JobTemplateType; Dictionary <string, Object> hitParams = new Dictionary <string, object>(); hitParams.Add("Title", job.amazonHITInformation.AmazonMTurkTaskTitle); hitParams.Add("TaskDescription", job.amazonHITInformation.AmazonMTurkTaskDescription); hitParams.Add("KeyWords", job.amazonHITInformation.AmazonMTurkTaskKeywords); hitParams.Add("Reward", job.amazonHITInformation.Price); //convert cents to dollars if (TaskConstants.masterGUIDs.Contains(job.JobGUIDString)) { hitParams.Add("Masters", true); } int tasksPerHIT = job.TasksPerJob; List <SatyamAmazonHITTableAccessEntry> hitEntries = hitDB.getAllEntriesByJobGUIDAndStatus(jobGUID, HitStatus.pending); // the task will be marked as submitted when the webpage exit int noHITsRemaining = hitEntries.Count; ////////////////////////// check and deal with pending stragglers, the actual hit might have expired. //int noStragglers = 0; //foreach (SatyamAmazonHITTableAccessEntry pendingEntry in hitEntries) //{ // if ((DateTime.Now - pendingEntry.CreationTime).TotalHours > 1) // { // // the hit will be finished in 1hr if taken by a user, // // there is a higher chance that this hit have already been expired // // launch more for stragglers // noStragglers++; // } //} //noHITsRemaining -= noStragglers; /////////////////////////////// //////////////////////////////////// the query amazon way //AmazonMTurkHIT hit = new AmazonMTurkHIT(); //hit.setAccount(AmazonAccessKeyID, AmazonSecretAccessKeyID, false); ////an alternative without querying amazon ////Dictionary<string, int> noHITsFinishedTasks = getRemainingHITsFinishedTasksFromResultDB(jobGUID, tasksPerHIT); ////int noHITsRemaining = 0; //noHITsRemaining = 0; //int noPendingHITDone = 0; //foreach (SatyamAmazonHITTableAccessEntry hitEntry in hitEntries) //{ // int noTasksPending = hit.getNumTasksRemaining(hitEntry.HITID); // noHITsRemaining += noTasksPending; // //an alternative without querying amazon // //if (noHITsFinishedTasks.ContainsKey(hitEntry.HITID) && noHITsFinishedTasks[hitEntry.HITID] == tasksPerHIT) // //{ // // //HIT done // // noPendingHITDone++; // //} //} ////int noHITsRemainingFromResultDB = hitEntries.Count - noPendingHITDone; ////Console.WriteLine("Job {0}, HITS Remaining/fromDB/Pending/noTasks: {1}/{2}/{3}", //// jobGUID, noHITsRemaining, noHITsRemainingFromResultDB, hitEntries.Count, TasksByGUID[jobGUID].Count); ////if (noHITsRemainingFromResultDB!= noHITsRemaining) ////{ //// // not gonna work, what is the hit has less tasks than tasksPerHIT?? //// Console.WriteLine("Database method is wrong"); ////} ///////////////////////////////////////////////////////// int NoParallelTurker = 3; // too large will make the task stale on AMT, too small will hurt latency int difference = (int)Math.Ceiling((double)(TasksByGUID[jobGUID].Count * NoParallelTurker - noHITsRemaining * tasksPerHIT) / (double)tasksPerHIT); //int difference = 50; //difference *= 2; // boost for parallelism //difference = 10; //difference = Math.Min(difference, max) * NoParallelTurker; if (difference > 0) { AmazonMTurkHITBatchLauncher hitLauncher = new AmazonMTurkHITBatchLauncher(AmazonAccessKeyID, AmazonSecretAccessKeyID); string url = TaskConstants.JobTemplateTypeToTaskURI[JobTemplateType]; double reward = job.amazonHITInformation.Price; url = url + "?reward=" + reward; List <string> hitIDs = null; string hitParametersString = JSonUtils.ConvertObjectToJSon <Dictionary <string, object> >(hitParams); hitLauncher.LaunchBatches(url, hitParams, difference, out hitIDs); if (hitIDs != null) { foreach (string hitID in hitIDs) { hitDB.AddEntry(JobTemplateType, job.UserID, hitID, job.JobGUIDString, DateTime.Now, hitParametersString, AmazonAccessKeyID, AmazonSecretAccessKeyID); } jobDB.UpdateEntryStatus(entry.Key, JobStatus.launched); } } } jobDB.close(); hitDB.close(); }
private bool getNewRandomJob() { double price = 0; bool success = Double.TryParse(Hidden_Price.Value, out price); if (!success) { price = 0; } SatyamTaskTableAccess taskTableDB = new SatyamTaskTableAccess(); SatyamTaskTableEntry entry = null; //Thread.Sleep(5000); int maxDoneScore = TaskConstants.MULTI_OBJECT_LOCALIZATION_AND_LABLING_MAX_DONE_SCORE; if (SubmitButton.Enabled == true) { //entry = taskTableDB.getMinimumTriedNewEntryForWorkerIDByTemplateAndPrice(Hidden_AmazonWorkerID.Value, // TaskConstants.Detection_Image_MTurk, price); entry = taskTableDB.getMinimumTriedNewEntryForWorkerIDByTemplateAndPriceAndMaxDoneScore(Hidden_AmazonWorkerID.Value, TaskConstants.Detection_Image_MTurk, price, MaxDoneScore: maxDoneScore); } else { //entry = taskTableDB.getMinimumTriedEntryByTemplate(TaskConstants.Detection_Image_MTurk); entry = taskTableDB.getMinimumTriedEntryByTemplateAndMaxDoneScore(TaskConstants.Detection_Image_MTurk, MaxDoneScore: maxDoneScore); } if (entry == null) { taskTableDB.close(); return(false); } //doneScore = entry.DoneScore; //taskTableDB.IncrementDoneScore(entry.ID); taskTableDB.close(); SatyamTask task = JSonUtils.ConvertJSonToObject <SatyamTask>(entry.TaskParametersString); string uri = task.SatyamURI; DisplayImage.ImageUrl = uri; SatyamJob jobDefinitionEntry = task.jobEntry; MultiObjectLocalizationAndLabelingSubmittedJob job = JSonUtils.ConvertJSonToObject <MultiObjectLocalizationAndLabelingSubmittedJob>(jobDefinitionEntry.JobParameters); List <string> categories = job.Categories; CategorySelection_RadioButtonList.Items.Clear(); for (int i = 0; i < categories.Count; i++) { ListItem l = new ListItem(categories[i]); CategorySelection_RadioButtonList.Items.Add(l); } if (job.Description != "") { DescriptionPanel.Visible = true; DescriptionTextPanel.Controls.Add(new LiteralControl(job.Description)); } Hidden_BoundaryLines.Value = JSonUtils.ConvertObjectToJSon(job.BoundaryLines); Hidden_TaskEntryString.Value = JSonUtils.ConvertObjectToJSon <SatyamTaskTableEntry>(entry); Hidden_PageLoadTime.Value = DateTime.Now.ToString(); NoLabeled.Text = Hidden_NoImagesDone.Value; Hidden_TasksPerJob.Value = jobDefinitionEntry.TasksPerJob.ToString(); /////////////////////////////////Load Previous Turker Results //MultiObjectLocalizationAndLabelingResult res = LoadLatestTurkerResult(entry); ///////////////////////////////Load Previous Aggregation Results MultiObjectLocalizationAndLabelingResult res = LoadLatestProgressiveAggregationResult(entry); //TFServing Backend if (res == null && TFServingBackend) { res = LoadTFServingResult(entry); Hidden_PrevResultID.Value = "-1"; // means TF } if (res == null) { Hidden_PrevResults.Value = "[]"; Hidden_PrevResultID.Value = 0.ToString(); } else { string prevBoxes = JSonUtils.ConvertObjectToJSon(res.objects); Hidden_ImageHeight.Value = res.imageHeight.ToString(); Hidden_ImageWidth.Value = res.imageWidth.ToString(); Hidden_PrevResults.Value = prevBoxes; } return(true); }
public static void AdaptTaskParameters() { double TargetPricePerHour = -1; double Budget = -1; //double TargetPricePerHour = 13.56; //double MinPricePerTask = 0.01; //double MaxPricePerTask = 0.5; double MinPricePerTaskDifferenceToAdjust = 0.01;//USD int MinNoResultsToAdjust = 10; Dictionary <string, List <SatyamTaskTableEntry> > MTurkTasksByGUID = OrganizeMTurkTasksByGUID(); if (MTurkTasksByGUID == null) { return; } // get statistics foreach (string guid in MTurkTasksByGUID.Keys) { // testing for counting only List <string> ACGuids = new List <string>() { "e5fdff10-f018-4779-9d18-70c25fa3259f", //carpk 164 "a388df30-8742-4b6e-911d-32093502c2e8", //kitti 100 }; if (!ACGuids.Contains(guid)) { continue; } SatyamTaskTableEntry task0 = MTurkTasksByGUID[guid][0]; double CurrentPricePerTask = task0.PricePerTask; SatyamTask t = JSonUtils.ConvertJSonToObject <SatyamTask>(task0.TaskParametersString); TargetPricePerHour = t.jobEntry.TargetPricePerTask; //TargetPricePerHour = 13.56; //temporary Budget = t.jobEntry.TotalBudget; if (TargetPricePerHour <= 0) { continue; //invalid } SatyamResultPerJobDataAnalysis ana = SatyamResultsAnalysis.AnalyzeFromResultsTable(new List <string>() { guid }); if (ana == null) { continue; } int noAcceptedResults = ana.totalAcceptedResults; if (noAcceptedResults < MinNoResultsToAdjust) { continue; } double medianTimePerTaskInSeconds = ana.medianAcceptedTimeTakenPerResultInSeconds; double TargetPricePerTask = TargetPricePerHour / 3600 * medianTimePerTaskInSeconds; //TargetPricePerTask = Math.Max(TargetPricePerTask, MinPricePerTask); //TargetPricePerTask = Math.Min(TargetPricePerTask, MaxPricePerTask); TargetPricePerTask = Math.Round(TargetPricePerTask, 2);// round to cents. // is the price within the budget? double MinBudgetNeeded = MTurkTasksByGUID[guid].Count * TargetPricePerTask * 3;// at least 3 results per task. if (Budget < MinBudgetNeeded) { Console.WriteLine("Budget too small to adapt price"); continue; } if (Math.Abs(CurrentPricePerTask - TargetPricePerTask) > MinPricePerTaskDifferenceToAdjust) { // need to adjust Console.WriteLine("Adapting the price to {0}", TargetPricePerTask); ExpireHitByGUID(guid); AdjustTasksByGUID(guid, TargetPricePerTask); } } }