public static bool IsBlackListed(string satyamURI, Dictionary <string, ConfusingReason> imageBlackListReason) { string fileName = URIUtilities.filenameFromURI(satyamURI); string imageCategoryName = fileName.Split('_')[0]; if (GroundTruth[imageCategoryName] == "ambulance" || imageCategoryName == "ambulance") { return(true); } if (imageBlackListReason.ContainsKey(fileName)) { if (imageBlackListReason[fileName] == ConfusingReason.NotAvailable) { return(true); } if (imageBlackListReason[fileName] == ConfusingReason.MultiObjMultiClass) { return(true); } //if (imageBlackListReason[fileName] == ConfusingReason.SingleObjConfusingClass) //{ // return true; //} } return(false); }
public static void SaveResultOnlyText(string jobGUID) { string directoryName = DirectoryConstants.defaultTempDirectory + "\\" + jobGUID + "\\"; SatyamResultsTableAccess resultsDB = new SatyamResultsTableAccess(); List <SatyamResultsTableEntry> entries = resultsDB.getEntriesByGUID(jobGUID); resultsDB.close(); if (!Directory.Exists(directoryName)) { Directory.CreateDirectory(directoryName); } directoryName = directoryName + "\\Raw\\"; if (!Directory.Exists(directoryName)) { Directory.CreateDirectory(directoryName); } for (int i = 0; i < entries.Count; i++) { SatyamResultsTableEntry entry = entries[i]; SatyamResult satyamResult = JSonUtils.ConvertJSonToObject <SatyamResult>(entry.ResultString); SatyamTask task = JSonUtils.ConvertJSonToObject <SatyamTask>(satyamResult.TaskParametersString); SatyamJob job = task.jobEntry; string result = satyamResult.TaskResult; string ofilename = URIUtilities.filenameFromURI(task.SatyamURI); string[] fields = ofilename.Split('.'); string fileName = ""; for (int j = 0; j < fields.Count(); j++) { fileName += fields[j]; if (j == fields.Count() - 1) { break; } fileName += "."; } //string fileName = ofilename; fileName = fileName + "-Result"; fileName = fileName + "-" + entry.ID; Console.WriteLine("Saving " + fileName); string resultFile = directoryName + fileName + ".txt"; StreamWriter f = new System.IO.StreamWriter(resultFile); f.WriteLine(result); f.Close(); } }
public static void UploadFileFromRemoteFolder(BlobContainerManager source, string sourceContainerName, string sourceDirectoryName, BlobContainerManager destination, string destinationContainerName, string destinationDirectoryName) { List <String> filesToBeCopied = source.getURLList(sourceContainerName, sourceDirectoryName); foreach (string uri in filesToBeCopied) { string fileName = URIUtilities.filenameFromURI(uri); destination.UploadFromURI(uri, destinationContainerName, destinationDirectoryName, fileName); //destination.UploadFromURI(uri, destinationContainerName, destinationDirectoryName, fileName); } }
public static void SaveResultImagesLocally(List <SatyamResultsTableEntry> entries, string directoryName) { if (!Directory.Exists(directoryName)) { Directory.CreateDirectory(directoryName); } directoryName = directoryName + "\\Raw\\"; if (!Directory.Exists(directoryName)) { Directory.CreateDirectory(directoryName); } for (int i = 0; i < entries.Count; i++) { SatyamResultsTableEntry entry = entries[i]; SatyamResult satyamResult = JSonUtils.ConvertJSonToObject <SatyamResult>(entry.ResultString); SatyamTask task = JSonUtils.ConvertJSonToObject <SatyamTask>(satyamResult.TaskParametersString); SatyamJob job = task.jobEntry; string result = satyamResult.TaskResult; Image originalImage = ImageUtilities.getImageFromURI(task.SatyamURI); Image ResultImage = DrawResultStringOnImage(result, originalImage); string ofilename = URIUtilities.filenameFromURI(task.SatyamURI); string[] fields = ofilename.Split('.'); string fileName = ""; for (int j = 0; j < fields.Count(); j++) { fileName += fields[j]; if (j == fields.Count() - 1) { break; } fileName += "."; } //string fileName = ofilename; fileName = fileName + "-Result"; fileName = fileName + "-" + entry.ID; Console.WriteLine("Saving " + fileName); ImageUtilities.saveImage(ResultImage, directoryName, fileName); string resultFile = directoryName + fileName + ".txt"; StreamWriter f = new System.IO.StreamWriter(resultFile); f.WriteLine(result); f.Close(); } }
public static bool AggregatedResultEqualsGroundTruth(string satyamUri, string resultString) { //string[] uri_parts = satyamUri.Split('/'); //string[] name_parts = uri_parts[uri_parts.Length - 1].Split('_'); //string imageName = name_parts[0]; string fileName = URIUtilities.filenameFromURI(satyamUri); string imageCategoryName = fileName.Split('_')[0]; SingleObjectLabelingAggregatedResult result = JSonUtils.ConvertJSonToObject <SingleObjectLabelingAggregatedResult>(resultString); return(result.Category.Equals(GroundTruth[imageCategoryName], StringComparison.InvariantCultureIgnoreCase)); }
public static void UploadFileFromRemoteFolder(BlobContainerManager source, string sourceContainerName, string sourceDirectoryName, BlobContainerManager destination, string destinationContainerName, string destinationDirectoryName, List <string> fileTypes) { List <String> filesToBeCopied = source.getURLList(sourceContainerName, sourceDirectoryName); foreach (string uri in filesToBeCopied) { string fileName = URIUtilities.filenameFromURI(uri); if (CheckFileExtensions.IsAllowedType(fileName, fileTypes)) { destination.UploadFromURI(uri, destinationContainerName, destinationDirectoryName, fileName); } } }
public static void SaveAggregatedResultImagesLocally(List <SatyamAggregatedResultsTableEntry> entries, string directoryName) { if (!Directory.Exists(directoryName)) { Directory.CreateDirectory(directoryName); } directoryName = directoryName + "\\Aggregated"; if (!Directory.Exists(directoryName)) { Directory.CreateDirectory(directoryName); } for (int i = 0; i < entries.Count; i++) { SatyamAggregatedResultsTableEntry entry = entries[i]; SatyamAggregatedResult satyamResult = JSonUtils.ConvertJSonToObject <SatyamAggregatedResult>(entry.ResultString); SatyamTask task = JSonUtils.ConvertJSonToObject <SatyamTask>(satyamResult.TaskParameters); SatyamJob job = task.jobEntry; ImageSegmentationAggregatedResult res = JSonUtils.ConvertJSonToObject <ImageSegmentationAggregatedResult>(satyamResult.AggregatedResultString); //string fileName = URIUtilities.filenameFromURINoExtension(task.SatyamURI); //Image originalImage = ImageUtilities.getImageFromURI(task.SatyamURI); //Image ResultImage = DrawResultsOnImage(res.boxesAndCategories, originalImage); //fileName = fileName + "-AggregatedResult"; //ImageUtilities.saveImage(ResultImage, directoryName, fileName); WebClient wb = new WebClient(); Image im = Image.FromStream(wb.OpenRead(res.metaData.PNG_URL)); string fileName = URIUtilities.filenameFromURI(res.metaData.PNG_URL); //wb.DownloadFile(directoryName + "\\" + fileName, res.metaData.PNG_URL); im.Save(directoryName + "\\" + fileName); } }
public static void EvaluateAndPrintConfusionMatrixOfAggregatedResultEntries(List <SatyamAggregatedResultsTableEntry> aggResultEntries, string outputFile, out int noCorrect, //out SortedDictionary<string, Dictionary<string, int>> confusionMatrix_res_groundtruth, //out SortedDictionary<string, Dictionary<string, int>> confusionMatrix_groundtruth_res, bool prepareDataForTraining = false, string outputDirectory = null) { noCorrect = 0; //SortedDictionary<string, Dictionary<string, int>> confusionMatrix_res_groundtruth = new SortedDictionary<string, Dictionary<string, int>>(); //SortedDictionary<string, Dictionary<string, int>> confusionMatrix_groundtruth_res = new SortedDictionary<string, Dictionary<string, int>>(); List <KeyValuePair <string, string> > detections_gts = new List <KeyValuePair <string, string> >(); WebClient wc = new WebClient(); foreach (SatyamAggregatedResultsTableEntry aggResultEntry in aggResultEntries) { SatyamSaveAggregatedDataSatyam data = new SatyamSaveAggregatedDataSatyam(aggResultEntry); SingleObjectLabelingAggregatedResult aggResult = JSonUtils.ConvertJSonToObject <SingleObjectLabelingAggregatedResult>(data.AggregatedResultString); string fileName = URIUtilities.filenameFromURI(data.SatyamURI); string VideoCategoryGroundTruth = getVideoCategoryFromFileName(fileName); detections_gts.Add(new KeyValuePair <string, string>(aggResult.Category, VideoCategoryGroundTruth)); //if (!confusionMatrix_res_groundtruth.ContainsKey(aggResult.Category)) //{ // confusionMatrix_res_groundtruth.Add(aggResult.Category, new Dictionary<string, int>()); //} //if (!confusionMatrix_res_groundtruth[aggResult.Category].ContainsKey(VideoCategoryGroundTruth)) //{ // confusionMatrix_res_groundtruth[aggResult.Category].Add(VideoCategoryGroundTruth, 0); //} //if (!confusionMatrix_groundtruth_res.ContainsKey(VideoCategoryGroundTruth)) //{ // confusionMatrix_groundtruth_res.Add(VideoCategoryGroundTruth, new Dictionary<string, int>()); //} //if (!confusionMatrix_groundtruth_res[VideoCategoryGroundTruth].ContainsKey(aggResult.Category)) //{ // confusionMatrix_groundtruth_res[VideoCategoryGroundTruth].Add(aggResult.Category, 0); //} //confusionMatrix_res_groundtruth[aggResult.Category][VideoCategoryGroundTruth]++; //confusionMatrix_groundtruth_res[VideoCategoryGroundTruth][aggResult.Category]++; if (aggResult.Category.Equals(VideoCategoryGroundTruth, StringComparison.InvariantCultureIgnoreCase)) { noCorrect++; } else { Console.WriteLine("{0}, Groundtruth: {1}, Aggregated: {2}, Votes: {3}", fileName, VideoCategoryGroundTruth, aggResult.Category, JSonUtils.ConvertObjectToJSon(aggResult.metaData)); } // prepare training dataset if (prepareDataForTraining) { if (!Directory.Exists(outputDirectory + "\\" + aggResult.Category)) { Directory.CreateDirectory(outputDirectory + "\\" + aggResult.Category); } wc.DownloadFile(data.SatyamURI, outputDirectory + "\\" + aggResult.Category + "\\" + fileName); } } SortedDictionary <string, Dictionary <string, int> > confusionMatrix_res_groundtruth; SortedDictionary <string, Dictionary <string, int> > confusionMatrix_groundtruth_res; SatyamResultValidationToolKit.getConfusionMat(detections_gts, out confusionMatrix_res_groundtruth, out confusionMatrix_groundtruth_res); SatyamResultValidationToolKit.printConfusionMatrix(confusionMatrix_groundtruth_res, confusionMatrix_res_groundtruth, outputFile); }
public static void AggregateWithParameterAndValidateSatyamVideoClassificationResultByGUID(string guid, int MinResults = TaskConstants.SINGLE_OBJECT_LABLING_MTURK_MIN_RESULTS_TO_AGGREGATE, int MaxResults = TaskConstants.SINGLE_OBJECT_LABLING_MTURK_MAX_RESULTS_TO_AGGREGATE, double MajorityThreshold = TaskConstants.SINGLE_OBJECT_LABLING_MTURK_MAJORITY_THRESHOLD, bool prepareDataForTraining = false, string outputDirectory = null, string confusingVideoListFilePath = null) { string configString = "Min_" + MinResults + "_Max_" + MaxResults + "_Thresh_" + MajorityThreshold; Console.WriteLine("Aggregating for param set " + configString); int noTerminatedTasks = 0; SatyamResultsTableAccess resultsDB = new SatyamResultsTableAccess(); List <SatyamResultsTableEntry> entries = resultsDB.getEntriesByGUIDOrderByID(guid); resultsDB.close(); SortedDictionary <DateTime, List <SatyamResultsTableEntry> > entriesBySubmitTime = SatyamResultValidationToolKit.SortResultsBySubmitTime(entries); int noTotalConverged = 0; int noCorrect = 0; Dictionary <int, List <SatyamResultsTableEntry> > ResultsPerTask = new Dictionary <int, List <SatyamResultsTableEntry> >(); List <int> aggregatedTasks = new List <int>(); SortedDictionary <string, Dictionary <string, int> > confusionMatrix_res_groundtruth = new SortedDictionary <string, Dictionary <string, int> >(); SortedDictionary <string, Dictionary <string, int> > confusionMatrix_groundtruth_res = new SortedDictionary <string, Dictionary <string, int> >(); List <SatyamAggregatedResultsTableEntry> aggEntries = new List <SatyamAggregatedResultsTableEntry>(); Dictionary <int, int> noResultsNeededForAggregation = SatyamResultsAnalysis.getNoResultsNeededForAggregationFromLog(configString, guid); Dictionary <int, int> noResultsNeededForAggregation_new = new Dictionary <int, int>(); foreach (DateTime t in entriesBySubmitTime.Keys) { //Console.WriteLine("Processing Results of time: {0}", t); foreach (SatyamResultsTableEntry entry in entriesBySubmitTime[t]) { SatyamResult satyamResult = JSonUtils.ConvertJSonToObject <SatyamResult>(entry.ResultString); SatyamTask task = JSonUtils.ConvertJSonToObject <SatyamTask>(satyamResult.TaskParametersString); SatyamJob job = task.jobEntry; string fileName = URIUtilities.filenameFromURI(task.SatyamURI); string VideoCategoryGroundTruth = getVideoCategoryFromFileName(fileName); int taskEntryID = entry.SatyamTaskTableEntryID; if (aggregatedTasks.Contains(taskEntryID)) { continue; } if (!ResultsPerTask.ContainsKey(taskEntryID)) { ResultsPerTask.Add(taskEntryID, new List <SatyamResultsTableEntry>()); } ResultsPerTask[taskEntryID].Add(entry); // check log if enough results are collected if (noResultsNeededForAggregation != null && noResultsNeededForAggregation.ContainsKey(taskEntryID) && ResultsPerTask[taskEntryID].Count < noResultsNeededForAggregation[taskEntryID]) { continue; } //SingleObjectLabelingAggregatedResult aggResult = SingleObjectLabelingAggregator.getAggregatedResult(ResultsPerTask[taskEntryID], MinResults, MaxResult, MajorityThreshold); string aggResultString = SingleObjectLabelingAggregator.GetAggregatedResultString(ResultsPerTask[taskEntryID], MinResults, MaxResults, MajorityThreshold); if (aggResultString == null) { continue; } SatyamAggregatedResultsTableEntry aggEntry = new SatyamAggregatedResultsTableEntry(); aggEntry.JobGUID = ResultsPerTask[taskEntryID][0].JobGUID; aggEntry.JobTemplateType = ResultsPerTask[taskEntryID][0].JobTemplateType; aggEntry.SatyamTaskTableEntryID = taskEntryID; aggEntry.UserID = ResultsPerTask[taskEntryID][0].UserID; aggEntry.ResultString = aggResultString; /// aggregation happen // record logs if (noResultsNeededForAggregation == null || !noResultsNeededForAggregation.ContainsKey(taskEntryID)) { noResultsNeededForAggregation_new.Add(taskEntryID, ResultsPerTask[taskEntryID].Count); } aggEntries.Add(aggEntry); noTotalConverged++; if (ResultsPerTask[taskEntryID].Count >= MaxResults) { noTerminatedTasks++; } aggregatedTasks.Add(taskEntryID); } } SatyamResultsAnalysis.RecordAggregationLog(noResultsNeededForAggregation_new, configString, guid); string outputmatFile = DirectoryConstants.defaultTempDirectory + guid + "\\" + configString + "_mat.txt"; EvaluateAndPrintConfusionMatrixOfAggregatedResultEntries(aggEntries, outputmatFile, out noCorrect, prepareDataForTraining, outputDirectory); Console.WriteLine("noTotalConverged {0}", noTotalConverged); Console.WriteLine("noTerminatedTasks {0}", noTerminatedTasks); Console.WriteLine("Result: {0}/{1}, precision: {2}", noCorrect, noTotalConverged, (double)noCorrect / noTotalConverged); /// local file string outputString = String.Format("{0} {1} {2} {3} {4} {5}\n", configString, noCorrect, noTotalConverged, (double)noCorrect / noTotalConverged, noTerminatedTasks, ResultsPerTask.Count - noTotalConverged); string outputfile = DirectoryConstants.defaultTempDirectory + guid + "\\resultSummary.txt"; File.AppendAllText(outputfile, outputString); //for (double prob = 0; prob < 1;prob +=0.2) //{ // SatyamResultsAnalysis.AnalyzeApprovalRate(aggEntries, entries, guid, configString, anotherChanceProbablity: prob); //} //for (double ratio = 0; ratio < 1; ratio += 0.2) //{ // SatyamResultsAnalysis.AnalyzeApprovalRate(aggEntries, entriesBySubmitTime, noResultsNeededForAggregation, noResultsNeededForAggregation_new, guid, configString, approvalRatioThreshold: ratio); //} SatyamResultsAnalysis.AggregationAnalysis(aggEntries, entriesBySubmitTime, noResultsNeededForAggregation, noResultsNeededForAggregation_new, guid, configString); }
/// <summary> /// Input should be a folder of videos and corresponding annotation file with the same name /// </summary> /// <param name="jobEntry"></param> public static bool ProcessAndUploadToAzureBlob(SatyamJobSubmissionsTableAccessEntry jobEntry) { // if the input is a folder of folders of frames, then copy directly SatyamJobStorageAccountAccess satyamStorage = new SatyamJobStorageAccountAccess(); string satyamContainerName = SatyamTaskGenerator.JobTemplateToSatyamContainerNameMap[jobEntry.JobTemplateType]; string GUID = jobEntry.JobGUID; string satyamDirectoryName = GUID; SatyamJob job = JSonUtils.ConvertJSonToObject <SatyamJob>(jobEntry.JobParametersString); MultiObjectTrackingSubmittedJob jobParams = JSonUtils.ConvertJSonToObject <MultiObjectTrackingSubmittedJob>(job.JobParameters); BlobContainerManager bcm = new BlobContainerManager(); string status = bcm.Connect(job.azureInformation.AzureBlobStorageConnectionString); List <string> FileTypes = SatyamTaskGenerator.ValidFileTypesByTemplate[job.JobTemplateType]; if (status != "SUCCESS") { return(false); } string guidFolder = DirectoryConstants.defaultTempDirectory + "\\" + GUID; Directory.CreateDirectory(guidFolder); int chunkLength = jobParams.ChunkDuration; // sec int outputFPS = jobParams.FrameRate; double chunkOverlap = jobParams.ChunkOverlap; // sec var client = new WebClient(); // sample to frames int noFramePerChunk = (int)(chunkLength * outputFPS); int noFrameOverlap = (int)(chunkOverlap * outputFPS); List <string> videoUrls = bcm.getURLList(job.azureInformation.AzureBlobStorageContainerName, job.azureInformation.AzureBlobStorageContainerDirectoryName); Dictionary <string, List <string> > videos = new Dictionary <string, List <string> >(); foreach (string videolink in videoUrls) { string videoName = URIUtilities.filenameFromURINoExtension(videolink); if (!videos.ContainsKey(videoName)) { videos.Add(videoName, new List <string>()); } videos[videoName].Add(videolink); } foreach (string videoName in videos.Keys) { // filter out those that doesn't provide a annotation file with it.... if (videos[videoName].Count != 2) { Console.WriteLine("Warning: Not 2 files provided for {0}.", videoName); //Directory.Delete(guidFolder, true); //return false; continue; } string videoURL = ""; string annotationURL = ""; foreach (string fileLink in videos[videoName]) { string extension = URIUtilities.fileExtensionFromURI(fileLink); if (extension != "txt") { videoURL = fileLink; } else { annotationURL = fileLink; } } string outputDirectory = guidFolder + "\\" + videoName; Directory.CreateDirectory(outputDirectory); string videoNameWithExtension = URIUtilities.filenameFromURI(videoURL); client.DownloadFile(videoURL, outputDirectory + "\\" + videoNameWithExtension); FFMpegWrappers.ExtractFrames(DirectoryConstants.ffmpeg, outputDirectory + "\\" + videoNameWithExtension, outputDirectory, videoName, DateTime.Now, outputFPS); File.Delete(outputDirectory + "\\" + videoNameWithExtension); List <string> chunkFolders = TrackingDataPreprocessor.GroupFramesIntoChunks(outputDirectory, noFramePerChunk); //parse VIRAT annotation file string annotationNameWithExtension = URIUtilities.filenameFromURI(annotationURL); client.DownloadFile(annotationURL, outputDirectory + "\\" + annotationNameWithExtension); parseAnnotationFileIntoChunkFolders(chunkFolders, outputDirectory + "\\" + annotationNameWithExtension, noFramePerChunk, outputFPS); //upload for (int i = 0; i < chunkFolders.Count; i++) { string subDir = chunkFolders[i]; satyamStorage.uploadALocalFolder(subDir, satyamContainerName, satyamDirectoryName + "/Video_" + videoName + "_startingFrame_" + noFramePerChunk * i); } Directory.Delete(outputDirectory, true); } return(true); }
public static void AggregateWithParameterAndValidateObjectCountingResultByGUID(string guid, //bool saveImage = false, bool prepareTrainingSet = false, string outputDirectory = null, int MinHeight = TaskConstants.OBJECT_COUNTING_VALIDATION_MIN_HEIGHT, int MaxOcclusion = TaskConstants.OBJECT_COUNTING_VALIDATION_MAX_OCCLUSION, double Max_Truncation = TaskConstants.OBJECT_COUNTING_VALIDATION_MIN_TRUNCATION, int MinResults = TaskConstants.OBJECT_COUNTING_MTURK_MIN_RESULTS_TO_AGGREGATE, int MaxResults = TaskConstants.OBJECT_COUNTING_MTURK_MAX_RESULTS_TO_AGGREGATE, double MAX_ABSOLUTE_COUNT_DEVIATION_LOWERBOUND = TaskConstants.OBJECT_COUNTING_MTURK_MAX_ABSOLUTE_COUNT_DEVIATION_LOWERBOUND, double MAX_DEVIATION_FRACTION = TaskConstants.OBJECT_COUNTING_MTURK_MAX_DEVIATION_FRACTION, double SUPER_MAJORITY_VALUE = TaskConstants.OBJECT_COUNTING_MTURK_SUPER_MAJORITY_VALUE ) { if (!Directory.Exists(DirectoryConstants.defaultTempDirectory + guid)) { Directory.CreateDirectory(DirectoryConstants.defaultTempDirectory + guid); } string configString = "Min_" + MinResults + "_Max_" + MaxResults + "_Dev_" + MAX_DEVIATION_FRACTION + "_Majority_" + SUPER_MAJORITY_VALUE; Console.WriteLine("Aggregating for param set " + configString); int noTerminatedTasks = 0; SatyamResultsTableAccess resultsDB = new SatyamResultsTableAccess(); List <SatyamResultsTableEntry> entries = resultsDB.getEntriesByGUIDOrderByID(guid); resultsDB.close(); SortedDictionary <DateTime, List <SatyamResultsTableEntry> > entriesBySubmitTime = SatyamResultValidationToolKit.SortResultsBySubmitTime(entries); int noTotalConverged = 0; //int noCorrect = 0; Dictionary <int, List <SatyamResultsTableEntry> > ResultsPerTask = new Dictionary <int, List <SatyamResultsTableEntry> >(); List <int> aggregatedTasks = new List <int>(); List <SatyamAggregatedResultsTableEntry> aggEntries = new List <SatyamAggregatedResultsTableEntry>(); Dictionary <int, int> noResultsNeededForAggregation = SatyamResultsAnalysis.getNoResultsNeededForAggregationFromLog(configString, guid); Dictionary <int, int> noResultsNeededForAggregation_new = new Dictionary <int, int>(); // play back by time foreach (DateTime t in entriesBySubmitTime.Keys) { //Console.WriteLine("Processing Results of time: {0}", t); foreach (SatyamResultsTableEntry entry in entriesBySubmitTime[t]) { SatyamResult satyamResult = JSonUtils.ConvertJSonToObject <SatyamResult>(entry.ResultString); SatyamTask task = JSonUtils.ConvertJSonToObject <SatyamTask>(satyamResult.TaskParametersString); SatyamJob job = task.jobEntry; string fileName = URIUtilities.filenameFromURI(task.SatyamURI); int taskEntryID = entry.SatyamTaskTableEntryID; if (aggregatedTasks.Contains(taskEntryID)) { continue; } if (!ResultsPerTask.ContainsKey(taskEntryID)) { ResultsPerTask.Add(taskEntryID, new List <SatyamResultsTableEntry>()); } ResultsPerTask[taskEntryID].Add(entry); // check log if enough results are collected if (noResultsNeededForAggregation != null && noResultsNeededForAggregation.ContainsKey(taskEntryID) && ResultsPerTask[taskEntryID].Count < noResultsNeededForAggregation[taskEntryID]) { continue; } string aggResultString = ObjectCountingAggregator.GetAggregatedResultString(ResultsPerTask[taskEntryID], MinResults, MaxResults, MAX_ABSOLUTE_COUNT_DEVIATION_LOWERBOUND, MAX_DEVIATION_FRACTION, SUPER_MAJORITY_VALUE); if (aggResultString == null) { continue; } SatyamAggregatedResultsTableEntry aggEntry = new SatyamAggregatedResultsTableEntry(); aggEntry.JobGUID = ResultsPerTask[taskEntryID][0].JobGUID; aggEntry.JobTemplateType = ResultsPerTask[taskEntryID][0].JobTemplateType; aggEntry.SatyamTaskTableEntryID = taskEntryID; aggEntry.UserID = ResultsPerTask[taskEntryID][0].UserID; aggEntry.ResultString = aggResultString; /// aggregation happen /// // record logs // record logs if (noResultsNeededForAggregation == null || !noResultsNeededForAggregation.ContainsKey(taskEntryID)) { noResultsNeededForAggregation_new.Add(taskEntryID, ResultsPerTask[taskEntryID].Count); } aggEntries.Add(aggEntry); noTotalConverged++; if (ResultsPerTask[taskEntryID].Count >= MaxResults) { noTerminatedTasks++; } aggregatedTasks.Add(taskEntryID); } } SatyamResultsAnalysis.RecordAggregationLog(noResultsNeededForAggregation_new, configString, guid); double totalError, totalGroundtruth; //ValidateSatyamKITTIObjectCountingAggregationResult(aggEntries, out totalError, out totalGroundtruth, // MinHeight, MaxOcclusion, Max_Truncation); ValidateSatyamCARPKObjectCountingAggregationResult(aggEntries, out totalError, out totalGroundtruth); Console.WriteLine("noTotalConverged {0}", noTotalConverged); Console.WriteLine("noTerminatedTasks {0}", noTerminatedTasks); /// local file string outputString = String.Format("{0} {1} {2} {3} {4} {5} {6}\n", configString, totalError, totalGroundtruth, totalError / totalGroundtruth, noTotalConverged, noTerminatedTasks, ResultsPerTask.Count - noTotalConverged); string outputfile = DirectoryConstants.defaultTempDirectory + guid + "\\resultSummary.txt"; File.AppendAllText(outputfile, outputString); string approvalString = configString + "_PayDev_" + TaskConstants.OBJECT_COUNTING_MTURK_MAX_ABSOLUTE_COUNT_DEVIATION_FOR_PAYMENT + "_PayFrac_" + TaskConstants.OBJECT_COUNTING_MTURK_MAX_DEVIATION_FRACTION_FOR_PAYMENT; //for (double prob = 0; prob < 1;prob +=0.2) //{ // SatyamResultsAnalysis.AnalyzeApprovalRate(aggEntries, entries, guid, approvalString, anotherChanceProbablity: prob); //} //for (double ratio = 0; ratio < 1; ratio += 0.2) //{ // SatyamResultsAnalysis.AnalyzeApprovalRate(aggEntries, entriesBySubmitTime, noResultsNeededForAggregation, noResultsNeededForAggregation_new, guid, configString, approvalRatioThreshold: ratio); //} SatyamResultsAnalysis.AggregationAnalysis(aggEntries, entriesBySubmitTime, noResultsNeededForAggregation, noResultsNeededForAggregation_new, guid, configString); }
public static bool ProcessAndUploadToAzureBlob(SatyamJobSubmissionsTableAccessEntry jobEntry) { // if the input is a folder of folders of frames, then copy directly SatyamJobStorageAccountAccess satyamStorage = new SatyamJobStorageAccountAccess(); string satyamContainerName = SatyamTaskGenerator.JobTemplateToSatyamContainerNameMap[jobEntry.JobTemplateType]; string GUID = jobEntry.JobGUID; string satyamDirectoryName = GUID; SatyamJob job = JSonUtils.ConvertJSonToObject <SatyamJob>(jobEntry.JobParametersString); MultiObjectTrackingSubmittedJob jobParams = JSonUtils.ConvertJSonToObject <MultiObjectTrackingSubmittedJob>(job.JobParameters); BlobContainerManager bcm = new BlobContainerManager(); string status = bcm.Connect(job.azureInformation.AzureBlobStorageConnectionString); List <string> FileTypes = SatyamTaskGenerator.ValidFileTypesByTemplate[job.JobTemplateType]; if (status != "SUCCESS") { return(false); } string guidFolder = DirectoryConstants.defaultTempDirectory + "\\" + GUID; Directory.CreateDirectory(guidFolder); int chunkLength = jobParams.ChunkDuration; // sec int outputFPS = jobParams.FrameRate; double chunkOverlap = jobParams.ChunkOverlap; // sec var client = new WebClient(); if (jobParams.DataSrcFormat == DataFormat.Video) { // sample to frames int noFramePerChunk = (int)(chunkLength * outputFPS); int noFrameOverlap = (int)(chunkOverlap * outputFPS); List <string> videoUrls = bcm.getURLList(job.azureInformation.AzureBlobStorageContainerName, job.azureInformation.AzureBlobStorageContainerDirectoryName); foreach (string video in videoUrls) { string videoName = URIUtilities.filenameFromURINoExtension(video); string videonameExtension = URIUtilities.filenameFromURI(video); string outputDirectory = guidFolder + "\\" + videoName; Directory.CreateDirectory(outputDirectory); client.DownloadFile(video, outputDirectory + "\\" + videonameExtension); FFMpegWrappers.ExtractFrames(DirectoryConstants.ffmpeg, outputDirectory + "\\" + videonameExtension, outputDirectory, videoName, DateTime.Now, outputFPS); Console.WriteLine("deleting downloaded file..."); File.Delete(outputDirectory + "\\" + videonameExtension); GroupFramesIntoChunksAndUploadChunks(videoName, outputDirectory, noFramePerChunk, noFrameOverlap, satyamContainerName, satyamDirectoryName); Directory.Delete(outputDirectory, true); } } if (jobParams.DataSrcFormat == DataFormat.VideoFrame) { int noFramePerChunk = (int)(chunkLength * outputFPS);//just use one fps for now, assume input should've already downsampled int noFrameOverlap = (int)(chunkOverlap * outputFPS); // chunk according to parameters List <string> frameUrls = bcm.getURLList(job.azureInformation.AzureBlobStorageContainerName, job.azureInformation.AzureBlobStorageContainerDirectoryName); Dictionary <string, List <string> > framesPerVideo = new Dictionary <string, List <string> >(); foreach (string url in frameUrls) { // assumed hierarchy: blob/directory.../videoname/frameid.jpg //string frameName = URIUtilities.filenameFromURINoExtension(url); string[] urlparts = url.Split('/'); string videoName = urlparts[urlparts.Length - 2]; if (!framesPerVideo.ContainsKey(videoName)) { framesPerVideo.Add(videoName, new List <string>()); } framesPerVideo[videoName].Add(url); } foreach (string video in framesPerVideo.Keys) { string outputDirectory = guidFolder + "\\" + video; Directory.CreateDirectory(outputDirectory); foreach (string frameURL in framesPerVideo[video]) { string frameName = URIUtilities.filenameFromURI(frameURL); client.DownloadFile(frameURL, outputDirectory + "\\" + frameName); } GroupFramesIntoChunksAndUploadChunks(video, outputDirectory, noFramePerChunk, noFrameOverlap, satyamContainerName, satyamDirectoryName); Directory.Delete(outputDirectory, true); } } return(true); }
public static void SaveResultImagesLocally(List <SatyamResultsTableEntry> entries, string directoryName) { if (!Directory.Exists(directoryName)) { Directory.CreateDirectory(directoryName); } directoryName = directoryName + "\\Raw"; if (!Directory.Exists(directoryName)) { Directory.CreateDirectory(directoryName); } for (int i = 0; i < entries.Count; i++) { SatyamResultsTableEntry entry = entries[i]; SatyamResult satyamResult = JSonUtils.ConvertJSonToObject <SatyamResult>(entry.ResultString); SatyamTask task = JSonUtils.ConvertJSonToObject <SatyamTask>(satyamResult.TaskParametersString); SatyamJob job = task.jobEntry; MultiObjectLocalizationAndLabelingResult res = JSonUtils.ConvertJSonToObject <MultiObjectLocalizationAndLabelingResult>(satyamResult.TaskResult); string ofilename = URIUtilities.filenameFromURI(task.SatyamURI); string[] fields = ofilename.Split('.'); string fileName = fields[0]; Image originalImage = ImageUtilities.getImageFromURI(task.SatyamURI); MultiObjectLocalizationAndLabelingSubmittedJob jobDefinition = JSonUtils.ConvertJSonToObject <MultiObjectLocalizationAndLabelingSubmittedJob>(job.JobParameters); Image imageWithBoundary = DrawingBoxesAndLinesOnImages.addLinesToImage(originalImage, jobDefinition.BoundaryLines, Color.Red, true); List <Rectangle> rectangles = new List <Rectangle>(); List <Color> colors = new List <Color>(); List <string> ids = new List <string>(); List <bool> dashed = new List <bool>(); for (int j = 0; j < res.objects.Count; j++) { MultiObjectLocalizationAndLabelingResultSingleEntry box = res.objects[j]; int x = box.boundingBox.tlx; int y = box.boundingBox.tly; int width = box.boundingBox.brx - box.boundingBox.tlx; int height = box.boundingBox.bry - box.boundingBox.tly; Rectangle r = new Rectangle(x, y, width, height); rectangles.Add(r); string category = box.Category; int colorIndex = jobDefinition.Categories.IndexOf(category); colors.Add(DrawingBoxesAndLinesOnImages.Colors[colorIndex]); string id = j + "-" + category; ids.Add(id); dashed.Add(false); } Image imageWithBoxesAndBoundary = DrawingBoxesAndLinesOnImages.addRectanglesToImage(imageWithBoundary, rectangles, colors, ids, dashed); fileName = fileName + "-Result"; if (satyamResult.amazonInfo.AssignmentID != "") { //fileName = fileName + "-" + satyamResult.amazonInfo.AssignmentID + "_" + entry.ID; fileName = fileName + "-" + entry.ID + "_" + satyamResult.PrevResultID; } ImageUtilities.saveImage(imageWithBoxesAndBoundary, directoryName, fileName); } }
public static void AggregateWithParameterAndValidateSatyamImageNetClassificationResultByGUID(string guid, int MinResults = TaskConstants.SINGLE_OBJECT_LABLING_MTURK_MIN_RESULTS_TO_AGGREGATE, int MaxResults = TaskConstants.SINGLE_OBJECT_LABLING_MTURK_MAX_RESULTS_TO_AGGREGATE, double MajorityThreshold = TaskConstants.SINGLE_OBJECT_LABLING_MTURK_MAJORITY_THRESHOLD, string confusingImageListFilePath = null) { string configString = "Min_" + MinResults + "_Max_" + MaxResults + "_Thresh_" + MajorityThreshold; Console.WriteLine("Aggregating for param set " + configString); int noTerminatedTasks = 0; SatyamResultsTableAccess resultsDB = new SatyamResultsTableAccess(); List <SatyamResultsTableEntry> entries = resultsDB.getEntriesByGUIDOrderByID(guid); resultsDB.close(); //SortedDictionary<DateTime, List<SatyamResultsTableEntry>> entriesBySubmitTime = // SatyamResultValidation.SortResultsBySubmitTime_OneResultPerTurkerPerTask(entries); SortedDictionary <DateTime, List <SatyamResultsTableEntry> > entriesBySubmitTime = SatyamResultValidationToolKit.SortResultsBySubmitTime(entries); Dictionary <string, ConfusingReason> imageBlackListReason = new Dictionary <string, ConfusingReason>(); if (confusingImageListFilePath != null) { imageBlackListReason = getConfusingImageList(confusingImageListFilePath); } //Dictionary<int, List<SingleObjectLabelingResult>> ResultsPerTask = new Dictionary<int, List<SingleObjectLabelingResult>>(); Dictionary <int, List <SatyamResultsTableEntry> > ResultsPerTask = new Dictionary <int, List <SatyamResultsTableEntry> >(); List <int> aggregatedTasks = new List <int>(); int noTotalConverged = 0; int noCorrect = 0; SortedDictionary <string, Dictionary <string, int> > confusionMatrix_res_groundtruth = new SortedDictionary <string, Dictionary <string, int> >(); SortedDictionary <string, Dictionary <string, int> > confusionMatrix_groundtruth_res = new SortedDictionary <string, Dictionary <string, int> >(); StringBuilder s = new StringBuilder(); s.Append("<!DOCTYPE html>\n"); s.Append("<html>\n"); s.Append("<body>\n"); String title = String.Format("<h1>Job GUID {0} Incorrect Result Summary</h1>\n", guid); s.Append(title); Dictionary <int, int> noResultsNeededForAggregation = SatyamResultsAnalysis.getNoResultsNeededForAggregationFromLog(configString, guid); Dictionary <int, int> noResultsNeededForAggregation_new = new Dictionary <int, int>(); List <SatyamAggregatedResultsTableEntry> aggEntries = new List <SatyamAggregatedResultsTableEntry>(); foreach (DateTime t in entriesBySubmitTime.Keys) { foreach (SatyamResultsTableEntry entry in entriesBySubmitTime[t]) { //SatyamResultsTableEntry entry = entries[i]; SatyamResult satyamResult = JSonUtils.ConvertJSonToObject <SatyamResult>(entry.ResultString); SatyamTask task = JSonUtils.ConvertJSonToObject <SatyamTask>(satyamResult.TaskParametersString); SatyamJob job = task.jobEntry; string fileName = URIUtilities.filenameFromURI(task.SatyamURI); string imageCategoryName = fileName.Split('_')[0]; if (IsBlackListed(task.SatyamURI, imageBlackListReason)) { continue; } int taskEntryID = entry.SatyamTaskTableEntryID; if (aggregatedTasks.Contains(taskEntryID)) { continue; } if (!ResultsPerTask.ContainsKey(taskEntryID)) { ResultsPerTask.Add(taskEntryID, new List <SatyamResultsTableEntry>()); } //ResultEntriesPerTask[taskEntryID].Add(JSonUtils.ConvertJSonToObject<SingleObjectLabelingResult>(satyamResult.TaskResult)); ResultsPerTask[taskEntryID].Add(entry); // check log if enough results are collected if (noResultsNeededForAggregation != null && noResultsNeededForAggregation.ContainsKey(taskEntryID) && ResultsPerTask[taskEntryID].Count < noResultsNeededForAggregation[taskEntryID]) { continue; } string aggResultString = SingleObjectLabelingAggregator.GetAggregatedResultString(ResultsPerTask[taskEntryID], MinResults, MaxResults, MajorityThreshold); //SingleObjectLabelingAggregatedResult aggResult = SingleObjectLabelingAggregator.getAggregatedResult(ResultEntriesPerTask[taskEntryID], MinResults, MaxResults, MajorityThreshold); if (aggResultString == null) { continue; } SatyamAggregatedResultsTableEntry aggEntry = new SatyamAggregatedResultsTableEntry(); aggEntry.JobGUID = ResultsPerTask[taskEntryID][0].JobGUID; aggEntry.JobTemplateType = ResultsPerTask[taskEntryID][0].JobTemplateType; aggEntry.SatyamTaskTableEntryID = ResultsPerTask[taskEntryID][0].SatyamTaskTableEntryID; aggEntry.UserID = ResultsPerTask[taskEntryID][0].UserID; aggEntry.ResultString = aggResultString; /// aggregation happen // record logs if (noResultsNeededForAggregation == null || !noResultsNeededForAggregation.ContainsKey(taskEntryID)) { noResultsNeededForAggregation_new.Add(taskEntryID, ResultsPerTask[taskEntryID].Count); } /// //if (aggResult.Category == "None of the Above") //{ // continue; //} aggEntries.Add(aggEntry); SatyamSaveAggregatedDataSatyam data = new SatyamSaveAggregatedDataSatyam(aggEntry); String resultString = data.AggregatedResultString; SingleObjectLabelingAggregatedResult aggResult = JSonUtils.ConvertJSonToObject <SingleObjectLabelingAggregatedResult>(resultString); if (!confusionMatrix_res_groundtruth.ContainsKey(aggResult.Category)) { confusionMatrix_res_groundtruth.Add(aggResult.Category, new Dictionary <string, int>() { { GroundTruth[imageCategoryName], 0 } }); } else { if (!confusionMatrix_res_groundtruth[aggResult.Category].ContainsKey(GroundTruth[imageCategoryName])) { confusionMatrix_res_groundtruth[aggResult.Category].Add(GroundTruth[imageCategoryName], 0); } } if (!confusionMatrix_groundtruth_res.ContainsKey(GroundTruth[imageCategoryName])) { confusionMatrix_groundtruth_res.Add(GroundTruth[imageCategoryName], new Dictionary <string, int>() { { aggResult.Category, 0 } }); } else { if (!confusionMatrix_groundtruth_res[GroundTruth[imageCategoryName]].ContainsKey(aggResult.Category)) { confusionMatrix_groundtruth_res[GroundTruth[imageCategoryName]].Add(aggResult.Category, 0); } } if (aggResult.Category.Equals(GroundTruth[imageCategoryName], StringComparison.InvariantCultureIgnoreCase)) { noCorrect++; } else { //Console.WriteLine("{0}, Groundtruth: {1}, Aggregated: {2}, Votes: {3}", // fileName, GroundTruth[imageCategoryName], aggResult.Category, // JSonUtils.ConvertObjectToJSon(aggResult.metaData)); String record = String.Format("<p>{0}, Groundtruth: {1}, Aggregated: {2}, Votes: {3}</p>\n", fileName, GroundTruth[imageCategoryName], aggResult.Category, JSonUtils.ConvertObjectToJSon(aggResult.metaData)); String img = String.Format("<img src=\"{0}\" >\n", task.SatyamURI); s.Append(record); s.Append(img); } noTotalConverged++; if (ResultsPerTask[taskEntryID].Count >= MaxResults) { noTerminatedTasks++; } confusionMatrix_res_groundtruth[aggResult.Category][GroundTruth[imageCategoryName]]++; confusionMatrix_groundtruth_res[GroundTruth[imageCategoryName]][aggResult.Category]++; aggregatedTasks.Add(taskEntryID); } } SatyamResultsAnalysis.RecordAggregationLog(noResultsNeededForAggregation_new, configString, guid); s.Append("</body>\n"); s.Append("</html>\n"); string dataToBeSaved = s.ToString(); SatyamJobStorageAccountAccess storage = new SatyamJobStorageAccountAccess(); string FileName = String.Format("AggregatedIncorrectResults-{0}_Min{1}Max{2}Thresh{3}.html", guid, MinResults, MaxResults, MajorityThreshold); storage.SaveATextFile("singleobjectlabeling", guid, FileName, dataToBeSaved); s.Clear(); s.Append("<!DOCTYPE html>\n"); s.Append("<html>\n"); s.Append("<body>\n"); s.Append(title); string resultSummary = String.Format("<p>Result: {0}/{1}, precision: {2}</p>\n", noCorrect, noTotalConverged, (double)noCorrect / noTotalConverged); resultSummary += String.Format("<p>Terminated: {0}, Not Enough Results: {1}</p>\n", noTerminatedTasks, ResultsPerTask.Count - noTotalConverged); Console.WriteLine(resultSummary); s.Append(resultSummary); // write the confusion matrix s.Append("<p>"); String row = "\t\t"; foreach (string resultCategory in confusionMatrix_res_groundtruth.Keys) { row += resultCategory + "\t"; } row += "<br>\n"; s.Append(row); Console.WriteLine(row); string matString = ""; foreach (string groundTruthCategory in confusionMatrix_groundtruth_res.Keys) { row = groundTruthCategory + "\t"; foreach (string resultCategory in confusionMatrix_res_groundtruth.Keys) { if (confusionMatrix_groundtruth_res[groundTruthCategory].ContainsKey(resultCategory)) { row += confusionMatrix_groundtruth_res[groundTruthCategory][resultCategory].ToString(); } else { row += "0"; } row += "\t"; } row += "<br>\n"; s.Append(row); Console.WriteLine(row); matString += row; } s.Append("</p>\n"); s.Append("</body>\n"); s.Append("</html>\n"); string summaryToBeSaved = s.ToString(); FileName = String.Format("Aggregated_Summary-{0}_Min{1}Max{2}Thresh{3}.html", guid, MinResults, MaxResults, MajorityThreshold); storage.SaveATextFile("singleobjectlabeling", guid, FileName, summaryToBeSaved); /// local file string outputString = String.Format("{0} {1} {2} {3} {4} {5}\n", configString, noCorrect, noTotalConverged, (double)noCorrect / noTotalConverged, noTerminatedTasks, ResultsPerTask.Count - noTotalConverged); string outputfile = DirectoryConstants.defaultTempDirectory + guid + "\\resultSummary.txt"; File.AppendAllText(outputfile, outputString); string outputmatFile = DirectoryConstants.defaultTempDirectory + guid + "\\" + configString + "_mat.txt"; File.WriteAllText(outputmatFile, matString); //for (double prob = 0; prob < 1;prob +=0.2) //{ // SatyamResultsAnalysis.AnalyzeApprovalRate(aggEntries, entries, guid, configString, anotherChanceProbablity: prob); //} //for (double ratio = 0; ratio < 1; ratio += 0.2) //{ // SatyamResultsAnalysis.AnalyzeApprovalRate(aggEntries, entriesBySubmitTime, noResultsNeededForAggregation, noResultsNeededForAggregation_new, guid, configString, approvalRatioThreshold: ratio); //} SatyamResultsAnalysis.AggregationAnalysis(aggEntries, entriesBySubmitTime, noResultsNeededForAggregation, noResultsNeededForAggregation_new, guid, configString); }
public static void ValidateSatyamImageNetClassificationAggregationResultByGUID(string guid, string confusingImageListFilePath = null, bool prepareDataForTraining = false, string outputDirectory = null) { Dictionary <string, ConfusingReason> imageBlackListReason = new Dictionary <string, ConfusingReason>(); if (confusingImageListFilePath != null) { imageBlackListReason = getConfusingImageList(confusingImageListFilePath); } //get all aggregated results SatyamAggregatedResultsTableAccess resultsDB = new SatyamAggregatedResultsTableAccess(); List <SatyamAggregatedResultsTableEntry> results = resultsDB.getEntriesByGUID(guid); resultsDB.close(); int noTotal = 0; int noCorrect = 0; SortedDictionary <string, Dictionary <string, int> > confusionMatrix_res_groundtruth = new SortedDictionary <string, Dictionary <string, int> >(); SortedDictionary <string, Dictionary <string, int> > confusionMatrix_groundtruth_res = new SortedDictionary <string, Dictionary <string, int> >(); StringBuilder s = new StringBuilder(); s.Append("<!DOCTYPE html>\n"); s.Append("<html>\n"); s.Append("<body>\n"); String title = String.Format("<h1>Job GUID {0} Incorrect Result Summary</h1>\n", guid); s.Append(title); WebClient wc = new WebClient(); for (int i = 0; i < results.Count; i++) { SatyamSaveAggregatedDataSatyam data = new SatyamSaveAggregatedDataSatyam(results[i]); //String uri = data.SatyamURI; //string[] uri_parts= uri.Split('/'); //string[] name_parts = uri_parts[uri_parts.Length - 1].Split('_'); string fileName = URIUtilities.filenameFromURI(data.SatyamURI); string imageCategoryName = fileName.Split('_')[0]; String resultString = data.AggregatedResultString; SingleObjectLabelingAggregatedResult result = JSonUtils.ConvertJSonToObject <SingleObjectLabelingAggregatedResult>(resultString); // skip all ambulances and black listed if (IsBlackListed(data.SatyamURI, imageBlackListReason)) { continue; } if (!confusionMatrix_res_groundtruth.ContainsKey(result.Category)) { confusionMatrix_res_groundtruth.Add(result.Category, new Dictionary <string, int>() { { GroundTruth[imageCategoryName], 0 } }); } else { if (!confusionMatrix_res_groundtruth[result.Category].ContainsKey(GroundTruth[imageCategoryName])) { confusionMatrix_res_groundtruth[result.Category].Add(GroundTruth[imageCategoryName], 0); } } if (!confusionMatrix_groundtruth_res.ContainsKey(GroundTruth[imageCategoryName])) { confusionMatrix_groundtruth_res.Add(GroundTruth[imageCategoryName], new Dictionary <string, int>() { { result.Category, 0 } }); } else { if (!confusionMatrix_groundtruth_res[GroundTruth[imageCategoryName]].ContainsKey(result.Category)) { confusionMatrix_groundtruth_res[GroundTruth[imageCategoryName]].Add(result.Category, 0); } } if (result.Category.Equals(GroundTruth[imageCategoryName], StringComparison.InvariantCultureIgnoreCase)) { noCorrect++; } else { //Console.WriteLine("{0}, Groundtruth: {1}, Aggregated: {2}, Votes: {3}", // fileName, GroundTruth[imageCategoryName], result.Category, // JSonUtils.ConvertObjectToJSon(result.metaData)); String record = String.Format("<p>{0}, Groundtruth: {1}, Aggregated: {2}, Votes: {3}</p>\n", fileName, GroundTruth[imageCategoryName], result.Category, JSonUtils.ConvertObjectToJSon(result.metaData)); String img = String.Format("<img src=\"{0}\" >\n", data.SatyamURI); s.Append(record); s.Append(img); } // prepare training dataset if (prepareDataForTraining) { if (GroundTruth[imageCategoryName] != "ambulance" && result.Category != "ambulance") { Image im = Image.FromStream(wc.OpenRead(data.SatyamURI)); if (!Directory.Exists(outputDirectory + result.Category)) { Directory.CreateDirectory(outputDirectory + result.Category); } im.Save(outputDirectory + "\\" + result.Category + "\\" + fileName); } } noTotal++; confusionMatrix_res_groundtruth[result.Category][GroundTruth[imageCategoryName]]++; confusionMatrix_groundtruth_res[GroundTruth[imageCategoryName]][result.Category]++; } Console.WriteLine("Result: {0}/{1}, precision: {2}", noCorrect, noTotal, (double)noCorrect / noTotal); // write the confusion matrix s.Append("<p>"); String row = "\t\t"; foreach (string resultCategory in confusionMatrix_res_groundtruth.Keys) { row += resultCategory + "\t"; } row += "<br>\n"; s.Append(row); Console.WriteLine(row); foreach (string groundTruthCategory in confusionMatrix_groundtruth_res.Keys) { row = groundTruthCategory + "\t"; foreach (string resultCategory in confusionMatrix_res_groundtruth.Keys) { if (confusionMatrix_groundtruth_res[groundTruthCategory].ContainsKey(resultCategory)) { row += confusionMatrix_groundtruth_res[groundTruthCategory][resultCategory].ToString(); } else { row += "0"; } row += "\t"; } row += "<br>\n"; s.Append(row); Console.WriteLine(row); } s.Append("</p>\n"); s.Append("</body>\n"); s.Append("</html>\n"); string dataToBeSaved = s.ToString(); SatyamJobStorageAccountAccess storage = new SatyamJobStorageAccountAccess(); string FileName = "AggregatedIncorrectResults-" + guid + ".html"; storage.SaveATextFile("singleobjectlabeling", guid, FileName, dataToBeSaved); }