Exemplo n.º 1
0
        private static void ValidateSatyamCARPKObjectCountingAggregationResult(List <SatyamAggregatedResultsTableEntry> aggResultEntries,
                                                                               out double totalError, out double totalGroundTruth)
        {
            List <double> aggResultCounts   = new List <double>();
            List <int>    GroundTruthCounts = new List <int>();
            List <double> abs_errors        = new List <double>();
            List <double> errors            = new List <double>();

            foreach (SatyamAggregatedResultsTableEntry aggResultEntry in aggResultEntries)
            {
                SatyamSaveAggregatedDataSatyam data = new SatyamSaveAggregatedDataSatyam(aggResultEntry);
                string fileName      = URIUtilities.filenameFromURINoExtension(data.SatyamURI);
                string labelFilePath = DirectoryConstants.CARPKCountingLabels + fileName + ".txt";

                string[] labels           = File.ReadAllLines(labelFilePath);
                int      GroundtruthCount = labels.Length;

                string jobGUID      = aggResultEntry.JobGUID;
                int    taskID       = aggResultEntry.SatyamTaskTableEntryID;
                String resultString = data.AggregatedResultString;
                ObjectCountingAggregatedResult result = JSonUtils.ConvertJSonToObject <ObjectCountingAggregatedResult>(resultString);


                aggResultCounts.Add(result.Count);
                GroundTruthCounts.Add(GroundtruthCount);
                abs_errors.Add(Math.Abs(GroundtruthCount - result.Count));
                errors.Add(GroundtruthCount - result.Count);
            }
            totalError       = abs_errors.Sum();
            totalGroundTruth = GroundTruthCounts.Sum();
            double totalErrorRatio = totalError / totalGroundTruth;

            Console.WriteLine("Error: {0} / {1} = {2}", totalError, totalGroundTruth, totalErrorRatio);
            Console.WriteLine("Total Aggregated {0}", aggResultEntries.Count);
        }
        public static void SaveAggregatedResultVideosLocally(List <SatyamAggregatedResultsTableEntry> entries, string directoryName)
        {
            directoryName = directoryName + "\\Aggregated";

            if (!Directory.Exists(directoryName))
            {
                Directory.CreateDirectory(directoryName);
            }
            SatyamJobStorageAccountAccess satyamStorage = new SatyamJobStorageAccountAccess();

            for (int i = 0; i < entries.Count; i++)
            {
                SatyamAggregatedResultsTableEntry entry           = entries[i];
                SatyamAggregatedResult            satyamAggResult = JSonUtils.ConvertJSonToObject <SatyamAggregatedResult>(entry.ResultString);
                SatyamTask aggTask = JSonUtils.ConvertJSonToObject <SatyamTask>(satyamAggResult.TaskParameters);
                MultiObjectTrackingSubmittedJob     job       = JSonUtils.ConvertJSonToObject <MultiObjectTrackingSubmittedJob>(aggTask.jobEntry.JobParameters);
                MultiObjectTrackingAggregatedResult aggresult = JSonUtils.ConvertJSonToObject <MultiObjectTrackingAggregatedResult>(satyamAggResult.AggregatedResultString);

                List <string> ImageURLs = satyamStorage.getURLListOfSubDirectoryByURL(aggTask.SatyamURI);


                string videoName = URIUtilities.localDirectoryNameFromURI(aggTask.SatyamURI) + "_" + entry.ID;

                generateVideoForEvaluation(ImageURLs, aggresult.tracklets, directoryName, videoName, job.FrameRate);
            }
        }
        public static SortedDictionary <string, List <SatyamAggregatedResultsTableEntry> > GroupChunksByVideoName(List <SatyamAggregatedResultsTableEntry> entries)
        {
            SortedDictionary <string, List <SatyamAggregatedResultsTableEntry> > entriesPerVideo = new SortedDictionary <string, List <SatyamAggregatedResultsTableEntry> >();
            int fps = 0;

            for (int i = 0; i < entries.Count; i++)
            {
                SatyamAggregatedResultsTableEntry entry           = entries[i];
                SatyamAggregatedResult            satyamAggResult = JSonUtils.ConvertJSonToObject <SatyamAggregatedResult>(entry.ResultString);
                SatyamTask aggTask   = JSonUtils.ConvertJSonToObject <SatyamTask>(satyamAggResult.TaskParameters);
                string     video     = URIUtilities.localDirectoryNameFromURI(aggTask.SatyamURI);
                string[]   fields    = video.Split('_');
                string     videoName = "";
                for (int j = 0; j < fields.Length - 2; j++)
                {
                    // remove the starting frame

                    videoName += fields[j];
                    if (j == fields.Length - 3)
                    {
                        break;
                    }
                    videoName += '_';
                    MultiObjectTrackingSubmittedJob job = JSonUtils.ConvertJSonToObject <MultiObjectTrackingSubmittedJob>(aggTask.jobEntry.JobParameters);
                    fps = job.FrameRate;
                }

                if (!entriesPerVideo.ContainsKey(videoName))
                {
                    entriesPerVideo.Add(videoName, new List <SatyamAggregatedResultsTableEntry>());
                }
                entriesPerVideo[videoName].Add(entries[i]);
            }
            return(entriesPerVideo);
        }
Exemplo n.º 4
0
        public static bool IsBlackListed(string satyamURI, Dictionary <string, ConfusingReason> imageBlackListReason)
        {
            string fileName          = URIUtilities.filenameFromURI(satyamURI);
            string imageCategoryName = fileName.Split('_')[0];

            if (GroundTruth[imageCategoryName] == "ambulance" || imageCategoryName == "ambulance")
            {
                return(true);
            }
            if (imageBlackListReason.ContainsKey(fileName))
            {
                if (imageBlackListReason[fileName] == ConfusingReason.NotAvailable)
                {
                    return(true);
                }
                if (imageBlackListReason[fileName] == ConfusingReason.MultiObjMultiClass)
                {
                    return(true);
                }

                //if (imageBlackListReason[fileName] == ConfusingReason.SingleObjConfusingClass)
                //{
                //    return true;
                //}
            }
            return(false);
        }
        public static void SaveResultVideosLocally(List <SatyamResultsTableEntry> entries, string directoryName, int fps = 10)
        {
            directoryName = directoryName + "\\Raw";

            if (!Directory.Exists(directoryName))
            {
                Directory.CreateDirectory(directoryName);
            }

            SatyamJobStorageAccountAccess satyamStorage = new SatyamJobStorageAccountAccess();



            for (int i = 0; i < entries.Count; i++)
            {
                SatyamResultsTableEntry entry        = entries[i];
                SatyamResult            satyamResult = JSonUtils.ConvertJSonToObject <SatyamResult>(entry.ResultString);
                SatyamTask task = JSonUtils.ConvertJSonToObject <SatyamTask>(satyamResult.TaskParametersString);
                MultiObjectTrackingSubmittedJob job = JSonUtils.ConvertJSonToObject <MultiObjectTrackingSubmittedJob>(task.jobEntry.JobParameters);

                string blobDir = URIUtilities.localDirectoryFullPathFromURI(task.SatyamURI);

                //VATIC_DVA_CrowdsourcedResult taskr = new VATIC_DVA_CrowdsourcedResult(satyamResult.TaskResult, task.SatyamURI, start, end, task.SatyamJobSubmissionsTableEntryID.ToString());
                VATIC_DVA_CrowdsourcedResult taskr = MultiObjectTrackingAggregator.createVATIC_DVA_CrowdsourcedResultUsingSatyamBlobImageCount(satyamResult.TaskResult, blobDir, entry.ID.ToString(), job.FrameRate);
                MultiObjectTrackingResult    res   = taskr.getCompressedTracksInTimeSegment();

                List <string> ImageURLs = satyamStorage.getURLListOfSpecificExtensionUnderSubDirectoryByURI(blobDir, new List <string>()
                {
                    "jpg", "png"
                });
                string videoName = URIUtilities.localDirectoryNameFromURI(blobDir) + "_" + URIUtilities.filenameFromURINoExtension(task.SatyamURI) + "_" + entry.ID;

                MultiObjectTrackingAnalyzer.generateVideoForEvaluation(ImageURLs, res, directoryName, videoName, job.FrameRate);
            }
        }
Exemplo n.º 6
0
        public static void SaveResultOnlyText(string jobGUID)
        {
            string directoryName = DirectoryConstants.defaultTempDirectory + "\\" + jobGUID + "\\";
            SatyamResultsTableAccess       resultsDB = new SatyamResultsTableAccess();
            List <SatyamResultsTableEntry> entries   = resultsDB.getEntriesByGUID(jobGUID);

            resultsDB.close();

            if (!Directory.Exists(directoryName))
            {
                Directory.CreateDirectory(directoryName);
            }

            directoryName = directoryName + "\\Raw\\";

            if (!Directory.Exists(directoryName))
            {
                Directory.CreateDirectory(directoryName);
            }

            for (int i = 0; i < entries.Count; i++)
            {
                SatyamResultsTableEntry entry        = entries[i];
                SatyamResult            satyamResult = JSonUtils.ConvertJSonToObject <SatyamResult>(entry.ResultString);
                SatyamTask task = JSonUtils.ConvertJSonToObject <SatyamTask>(satyamResult.TaskParametersString);
                SatyamJob  job  = task.jobEntry;

                string result = satyamResult.TaskResult;


                string   ofilename = URIUtilities.filenameFromURI(task.SatyamURI);
                string[] fields    = ofilename.Split('.');
                string   fileName  = "";
                for (int j = 0; j < fields.Count(); j++)
                {
                    fileName += fields[j];
                    if (j == fields.Count() - 1)
                    {
                        break;
                    }
                    fileName += ".";
                }


                //string fileName = ofilename;
                fileName = fileName + "-Result";

                fileName = fileName + "-" + entry.ID;

                Console.WriteLine("Saving " + fileName);

                string       resultFile = directoryName + fileName + ".txt";
                StreamWriter f          = new System.IO.StreamWriter(resultFile);
                f.WriteLine(result);
                f.Close();
            }
        }
Exemplo n.º 7
0
        public static void summarize(string guid)
        {
            string resultDir = DirectoryConstants.defaultTempDirectory + guid + "\\";
            string outputFile = resultDir + "resultSummary.txt";
            if (File.Exists(outputFile))
            {
                File.Delete(outputFile);
            }

            List<string> files = Directory.GetFiles(resultDir).ToList();
            SortedDictionary<double, SortedDictionary<double, SortedDictionary<double, SortedDictionary<double, string>>>> results = 
                new SortedDictionary<double, SortedDictionary<double, SortedDictionary<double, SortedDictionary<double, string>>>>();
            foreach(string file in files)
            {
                string filename = URIUtilities.filenameFromDirectoryNoExtension(file);
                string[] fields = filename.Split('_');
                double min = Convert.ToDouble(fields[1]);
                double max = Convert.ToDouble(fields[3]);
                double dev = Convert.ToDouble(fields[5]);
                double ratio = Convert.ToDouble(fields[7]);
                string content = File.ReadAllText(file).Replace("\n", String.Empty);
                if (!results.ContainsKey(min))
                {
                    results.Add(min, new SortedDictionary<double, SortedDictionary<double, SortedDictionary<double, string>>>());
                }
                if (!results[min].ContainsKey(max))
                {
                    results[min].Add(max, new SortedDictionary<double, SortedDictionary<double, string>>());
                }
                if (!results[min][max].ContainsKey(dev))
                {
                    results[min][max].Add(dev, new SortedDictionary<double, string>());
                }
                if (!results[min][max][dev].ContainsKey(ratio))
                {
                    results[min][max][dev].Add(ratio, filename + " " + content);
                }
            }

            List<string> contents = new List<string>();
            foreach( double m in results.Keys)
            {
                foreach (double n in results[m].Keys)
                {
                    foreach (double d in results[m][n].Keys)
                    {
                        foreach (double r in results[m][n][d].Keys)
                        {
                            contents.Add(results[m][n][d][r]);
                        }
                    }
                }
            }

            File.WriteAllLines(outputFile, contents.ToArray());
        }
Exemplo n.º 8
0
        public static bool AggregatedResultEqualsGroundTruth(string satyamUri, string resultString)
        {
            //string[] uri_parts = satyamUri.Split('/');
            //string[] name_parts = uri_parts[uri_parts.Length - 1].Split('_');
            //string imageName = name_parts[0];
            string fileName          = URIUtilities.filenameFromURI(satyamUri);
            string imageCategoryName = fileName.Split('_')[0];
            SingleObjectLabelingAggregatedResult result = JSonUtils.ConvertJSonToObject <SingleObjectLabelingAggregatedResult>(resultString);

            return(result.Category.Equals(GroundTruth[imageCategoryName], StringComparison.InvariantCultureIgnoreCase));
        }
Exemplo n.º 9
0
        public static void SaveResultImagesLocally(List <SatyamResultsTableEntry> entries, string directoryName)
        {
            if (!Directory.Exists(directoryName))
            {
                Directory.CreateDirectory(directoryName);
            }

            directoryName = directoryName + "\\Raw\\";

            if (!Directory.Exists(directoryName))
            {
                Directory.CreateDirectory(directoryName);
            }

            for (int i = 0; i < entries.Count; i++)
            {
                SatyamResultsTableEntry entry        = entries[i];
                SatyamResult            satyamResult = JSonUtils.ConvertJSonToObject <SatyamResult>(entry.ResultString);
                SatyamTask task = JSonUtils.ConvertJSonToObject <SatyamTask>(satyamResult.TaskParametersString);
                SatyamJob  job  = task.jobEntry;

                string result        = satyamResult.TaskResult;
                Image  originalImage = ImageUtilities.getImageFromURI(task.SatyamURI);

                Image ResultImage = DrawResultStringOnImage(result, originalImage);

                string   ofilename = URIUtilities.filenameFromURI(task.SatyamURI);
                string[] fields    = ofilename.Split('.');
                string   fileName  = "";
                for (int j = 0; j < fields.Count(); j++)
                {
                    fileName += fields[j];
                    if (j == fields.Count() - 1)
                    {
                        break;
                    }
                    fileName += ".";
                }


                //string fileName = ofilename;
                fileName = fileName + "-Result";

                fileName = fileName + "-" + entry.ID;

                Console.WriteLine("Saving " + fileName);

                ImageUtilities.saveImage(ResultImage, directoryName, fileName);
                string       resultFile = directoryName + fileName + ".txt";
                StreamWriter f          = new System.IO.StreamWriter(resultFile);
                f.WriteLine(result);
                f.Close();
            }
        }
Exemplo n.º 10
0
        public static void UploadFileFromRemoteFolder(BlobContainerManager source, string sourceContainerName, string sourceDirectoryName, BlobContainerManager destination, string destinationContainerName, string destinationDirectoryName)
        {
            List <String> filesToBeCopied = source.getURLList(sourceContainerName, sourceDirectoryName);

            foreach (string uri in filesToBeCopied)
            {
                string fileName = URIUtilities.filenameFromURI(uri);
                destination.UploadFromURI(uri, destinationContainerName, destinationDirectoryName, fileName);
                //destination.UploadFromURI(uri, destinationContainerName, destinationDirectoryName, fileName);
            }
        }
Exemplo n.º 11
0
        public static void TestFilterGroundTruthFile()
        {
            string outputDir = DirectoryConstants.defaultTempDirectory + "filteredKITTIGroundTruth\\";
            string outputDir_Seattle = outputDir + "SeattleLive-5-WestLake-NS\\";
            string outputDir_Stewart = outputDir + "1-Stewart-NS\\";
            string outputDir_BellevueNE = outputDir + "Bellevue-148th-NE29th\\";
            string outputDir_BellevueMain = outputDir + "Bellevue-148th-Main\\";

            string inputDir = @"C:\research\dataset\LongDuration\";
            string inputDir_Seattle = inputDir + "SeattleLive-5-WestLake-NS\\";
            string inputDir_Stewart = inputDir + "1-Stewart-NS\\";
            string inputDir_BellevueNE = inputDir + "Bellevue-148th-NE29th\\";
            string inputDir_BellevueMain = inputDir + "Bellevue-148th-Main\\";

            List<string> inputs = new List<string>()
            {
                inputDir_BellevueNE,
                inputDir_BellevueMain,
                inputDir_Seattle,
                inputDir_Stewart,
            };
            List<string> outputs = new List<string>()
            {
                outputDir_BellevueNE,
                outputDir_BellevueMain,
                outputDir_Seattle,
                outputDir_Stewart,
            };


            for (int i = 0; i < inputs.Count; i++)
            {
                string[] subdirs = Directory.GetDirectories(inputs[i] + "labels\\");
                foreach (string subdir in subdirs)
                {
                    int omitted = 0;
                    string[] files = Directory.GetFiles(subdir);
                    if (files.Length == 0)
                    {
                        string [] reward = Directory.GetDirectories(subdir);
                        files = Directory.GetFiles(reward[0]);
                    }
                    string groupName = URIUtilities.localDirectoryNameFromDirectory(subdir);
                    foreach (string file in files)
                    {
                        int tmp;
                        KITTIDetectionResultValidation.FilterGroundTruthFile(file, outputs[i] + groupName, out tmp);
                        omitted += tmp;
                    }
                    Console.WriteLine("{0}: omitted {1}", subdir, omitted);
                }
                
            }
        }
        public static void LoadAnnotations()
        {
            string folder     = @"C:\research\dataset\PascalVOC\VOCtrainval_11-May-2012\VOCdevkit\VOC2012\SegmentationImages\";
            string annofolder = @"C:\research\dataset\PascalVOC\VOCtrainval_11-May-2012\VOCdevkit\VOC2012\Annotations\";

            SortedDictionary <string, SortedDictionary <string, int> > corr = new SortedDictionary <string, SortedDictionary <string, int> >();

            string[] files = Directory.GetFiles(folder);
            foreach (string file in files)
            {
                string        filename = URIUtilities.filenameFromDirectoryNoExtension(file);
                string        annofile = annofolder + filename + ".xml";
                XElement      anno     = XElement.Load(annofile);
                List <string> names    = (from el in anno.Descendants("name")
                                          select(string) el).ToList <string>();

                names.Sort();
                for (int i = 0; i < names.Count; i++)
                {
                    for (int j = 0; j < i; j++)
                    {
                        string name1 = names[i];
                        string name2 = names[j];
                        if (!corr.ContainsKey(name1))
                        {
                            corr.Add(name1, new SortedDictionary <string, int>());
                        }
                        if (!corr[name1].ContainsKey(name2))
                        {
                            corr[name1].Add(name2, 0);
                        }
                        corr[name1][name2]++;
                    }
                }
            }

            foreach (string k in corr.Keys)
            {
                string line = k + " ";
                foreach (string v in corr.Keys)
                {
                    if (corr[k].ContainsKey(v))
                    {
                        line += corr[k][v] + " ";
                    }
                    else
                    {
                        line += "0 ";
                    }
                }
                Console.WriteLine(line);
            }
        }
Exemplo n.º 13
0
        public static void UploadFileFromRemoteFolder(BlobContainerManager source, string sourceContainerName, string sourceDirectoryName, BlobContainerManager destination, string destinationContainerName, string destinationDirectoryName, List <string> fileTypes)
        {
            List <String> filesToBeCopied = source.getURLList(sourceContainerName, sourceDirectoryName);

            foreach (string uri in filesToBeCopied)
            {
                string fileName = URIUtilities.filenameFromURI(uri);
                if (CheckFileExtensions.IsAllowedType(fileName, fileTypes))
                {
                    destination.UploadFromURI(uri, destinationContainerName, destinationDirectoryName, fileName);
                }
            }
        }
        public static bool AggregatedResultEqualsGroundTruth(string satyamUri, string resultString)
        {
            string fileName = URIUtilities.filenameFromURINoExtension(satyamUri);

            string VideoCategory = getVideoCategoryFromFileName(fileName);

            if (VideoCategory == "")
            {
                Console.WriteLine("this video doesn't have valid category in filename");
            }

            SingleObjectLabelingAggregatedResult result = JSonUtils.ConvertJSonToObject <SingleObjectLabelingAggregatedResult>(resultString);

            return(result.Category.Equals(VideoCategory, StringComparison.InvariantCultureIgnoreCase));
        }
Exemplo n.º 15
0
        private static void ValidateSatyamKITTIObjectCountingAggregationResult(List <SatyamAggregatedResultsTableEntry> aggResultEntries,
                                                                               //bool saveImage = false,
                                                                               out double totalError,
                                                                               out double totalGroundTruth,
                                                                               int MinHeight         = TaskConstants.OBJECT_COUNTING_VALIDATION_MIN_HEIGHT,
                                                                               int MaxOcclusion      = TaskConstants.OBJECT_COUNTING_VALIDATION_MAX_OCCLUSION,
                                                                               double Max_Truncation = TaskConstants.OBJECT_COUNTING_VALIDATION_MIN_TRUNCATION)
        {
            List <double> aggResultCounts   = new List <double>();
            List <int>    GroundTruthCounts = new List <int>();
            List <double> errors            = new List <double>();

            foreach (SatyamAggregatedResultsTableEntry aggResultEntry in aggResultEntries)
            {
                SatyamSaveAggregatedDataSatyam data = new SatyamSaveAggregatedDataSatyam(aggResultEntry);
                string fileName = URIUtilities.filenameFromURINoExtension(data.SatyamURI);
                KITTIDetectionGroundTruth GroundTruthObjects = new KITTIDetectionGroundTruth(KITTIDetectionResultValidation.GroundTruthLabelDirectory + fileName + ".txt", MinHeight, MaxOcclusion, Max_Truncation);

                string jobGUID      = aggResultEntry.JobGUID;
                int    taskID       = aggResultEntry.SatyamTaskTableEntryID;
                String resultString = data.AggregatedResultString;
                ObjectCountingAggregatedResult result = JSonUtils.ConvertJSonToObject <ObjectCountingAggregatedResult>(resultString);
                int GroundtruthCount = 0;
                for (int i = 0; i < GroundTruthObjects.objects.Count; i++)
                {
                    MultiObjectLocalizationAndLabelingResultSingleEntry obj = GroundTruthObjects.objects[i];
                    //if (obj.Category == "Car" || obj.Category == "Van" || obj.Category == "DontCare")
                    if (obj.Category == "Car" || obj.Category == "Van")
                    {
                        //if (!GroundTruthObjects.BlackListed[i])
                        //{
                        GroundtruthCount++;
                        //}
                    }
                }
                aggResultCounts.Add(result.Count);
                GroundTruthCounts.Add(GroundtruthCount);
                errors.Add(Math.Abs(GroundtruthCount - result.Count));
            }
            totalError       = errors.Sum();
            totalGroundTruth = GroundTruthCounts.Sum();
            double totalErrorRatio = totalError / totalGroundTruth;

            Console.WriteLine("Error: {0} / {1} = {2}", totalError, totalGroundTruth, totalErrorRatio);
            Console.WriteLine("Total Aggregated {0}", aggResultEntries.Count);
        }
Exemplo n.º 16
0
        public List <String> getURLListOfSpecificExtension(string containerName, string DirectoryName, List <string> extensions)
        {
            List <String> urlList    = getURLList(containerName, DirectoryName, flatList: true);
            List <String> returnList = new List <string>();

            foreach (String url in urlList)
            {
                string ext = URIUtilities.fileExtensionFromURI(url);
                foreach (string e in extensions)
                {
                    if (ext.Equals(e, StringComparison.InvariantCultureIgnoreCase))
                    {
                        returnList.Add(url);
                    }
                }
            }
            return(returnList);
        }
Exemplo n.º 17
0
        public static void TestExtractFrame()
        {
            string dataPath = @"C:\research\dataset\TomLaPorta\";

            string[] files = Directory.GetFiles(dataPath);
            foreach (string f in files)
            {
                string   fn           = URIUtilities.filenameFromDirectory(f);
                string[] ff           = fn.Split('.');
                string   fname        = ff[0];
                string   outputFolder = dataPath + fname + "\\";
                if (!Directory.Exists(outputFolder))
                {
                    Directory.CreateDirectory(outputFolder);
                }
                FFMpegWrappers.ExtractFrames(ffmpeg, f,
                                             outputFolder, fname, DateTime.MinValue, 0.1);
            }
        }
Exemplo n.º 18
0
        //public static void GroupFramesIntoChunksAndUploadChunks(string videoName, string CurrentDir, int noFramePerChunk, int noFramesOverlap, string satyamContainerName, string satyamDirectoryName)
        //{
        //    SatyamJobStorageAccountAccess satyamStorage = new SatyamJobStorageAccountAccess();
        //    List<string> files = Directory.GetFiles(CurrentDir).ToList();
        //    int chunkSequence = 0;
        //    List<List<string>> allFilesPerChunk = new List<List<string>>();
        //    for(int i=0;i<files.Count;i++)
        //    {

        //        int index = (int)Math.Ceiling((double)(i + 1) / (double)noFramePerChunk);
        //        if ( index > chunkSequence)
        //        {

        //            allFilesPerChunk.Add(new List<string>());
        //            chunkSequence++;
        //            if (chunkSequence > 1)
        //            {
        //                //starting the 2nd chunk, add frames before
        //                for (int j = 0; j < noFramesOverlap; j++)
        //                {
        //                    allFilesPerChunk[chunkSequence-1].Add(files[i - noFramesOverlap + j]);
        //                }
        //            }
        //        }
        //        allFilesPerChunk[index - 1].Add(files[i]);
        //    }
        //    for (int i = 0; i < allFilesPerChunk.Count; i++)
        //    {
        //        string subDir = CurrentDir + "\\" + i;
        //        Directory.CreateDirectory(subDir);
        //        foreach(string file in allFilesPerChunk[i])
        //        {
        //            string fileName = URIUtilities.filenameFromDirectory(file);
        //            File.Copy(file, subDir + "\\" + fileName);
        //        }
        //        satyamStorage.uploadALocalFolder(subDir, satyamContainerName, satyamDirectoryName +"/Video_" + videoName + "_startingFrame_" + noFramePerChunk * i);
        //    }
        //}

        public static List <string> GroupFramesIntoChunks(string CurrentDir, int noFramePerChunk, int noFramesOverlap = 0)
        {
            List <string>         files            = Directory.GetFiles(CurrentDir).ToList();
            int                   chunkSequence    = 0;
            List <List <string> > allFilesPerChunk = new List <List <string> >();

            for (int i = 0; i < files.Count; i++)
            {
                int index = (int)Math.Ceiling((double)(i + 1) / (double)noFramePerChunk);
                if (index > chunkSequence)
                {
                    allFilesPerChunk.Add(new List <string>());
                    chunkSequence++;
                    if (chunkSequence > 1)
                    {
                        //starting the 2nd chunk, add frames before
                        for (int j = 0; j < noFramesOverlap; j++)
                        {
                            allFilesPerChunk[chunkSequence - 1].Add(files[i - noFramesOverlap + j]);
                        }
                    }
                }
                allFilesPerChunk[index - 1].Add(files[i]);
            }

            List <string> chunkFolders = new List <string>();

            for (int i = 0; i < allFilesPerChunk.Count; i++)
            {
                string subDir = CurrentDir + "\\" + i;
                Directory.CreateDirectory(subDir);
                chunkFolders.Add(subDir);
                foreach (string file in allFilesPerChunk[i])
                {
                    string fileName = URIUtilities.filenameFromDirectory(file);
                    File.Copy(file, subDir + "\\" + fileName);
                }
            }

            return(chunkFolders);
        }
        public static void SaveAggregatedResultImagesLocally(List <SatyamAggregatedResultsTableEntry> entries, string directoryName)
        {
            if (!Directory.Exists(directoryName))
            {
                Directory.CreateDirectory(directoryName);
            }

            directoryName = directoryName + "\\Aggregated";

            if (!Directory.Exists(directoryName))
            {
                Directory.CreateDirectory(directoryName);
            }

            for (int i = 0; i < entries.Count; i++)
            {
                SatyamAggregatedResultsTableEntry entry        = entries[i];
                SatyamAggregatedResult            satyamResult = JSonUtils.ConvertJSonToObject <SatyamAggregatedResult>(entry.ResultString);
                SatyamTask task = JSonUtils.ConvertJSonToObject <SatyamTask>(satyamResult.TaskParameters);
                SatyamJob  job  = task.jobEntry;
                ImageSegmentationAggregatedResult res = JSonUtils.ConvertJSonToObject <ImageSegmentationAggregatedResult>(satyamResult.AggregatedResultString);


                //string fileName = URIUtilities.filenameFromURINoExtension(task.SatyamURI);

                //Image originalImage = ImageUtilities.getImageFromURI(task.SatyamURI);

                //Image ResultImage = DrawResultsOnImage(res.boxesAndCategories, originalImage);

                //fileName = fileName + "-AggregatedResult";
                //ImageUtilities.saveImage(ResultImage, directoryName, fileName);

                WebClient wb       = new WebClient();
                Image     im       = Image.FromStream(wb.OpenRead(res.metaData.PNG_URL));
                string    fileName = URIUtilities.filenameFromURI(res.metaData.PNG_URL);
                //wb.DownloadFile(directoryName + "\\" + fileName, res.metaData.PNG_URL);
                im.Save(directoryName + "\\" + fileName);
            }
        }
Exemplo n.º 20
0
        public static void getAmazonParametersFromURI(string OriginalUri, out string AssignmentID, out string HITID, out string workerID, out string reward)
        {
            OriginalUri = HttpUtility.UrlDecode(OriginalUri);
            string uri = HttpUtility.HtmlDecode(OriginalUri);
            Dictionary <string, string> dict = URIUtilities.getURIAttributes(uri);

            if (dict.ContainsKey("assignmentId"))
            {
                AssignmentID = dict["assignmentId"];
            }
            else
            {
                AssignmentID = "";
            }
            if (dict.ContainsKey("hitId"))
            {
                HITID = dict["hitId"];
            }
            else
            {
                HITID = "";
            }
            if (dict.ContainsKey("workerId"))
            {
                workerID = dict["workerId"];
            }
            else
            {
                workerID = "";
            }
            if (dict.ContainsKey("reward"))
            {
                reward = dict["reward"];
            }
            else
            {
                reward = "";
            }
        }
        public static void SaveAggregatedResultVideosLocally(List <SatyamAggregatedResultsTableEntry> entries, string directoryName)
        {
            directoryName = directoryName + "\\Aggregated";

            if (!Directory.Exists(directoryName))
            {
                Directory.CreateDirectory(directoryName);
            }
            SatyamJobStorageAccountAccess satyamStorage = new SatyamJobStorageAccountAccess();

            for (int i = 0; i < entries.Count; i++)
            {
                SatyamAggregatedResultsTableEntry entry           = entries[i];
                SatyamAggregatedResult            satyamAggResult = JSonUtils.ConvertJSonToObject <SatyamAggregatedResult>(entry.ResultString);
                SatyamTask aggTask = JSonUtils.ConvertJSonToObject <SatyamTask>(satyamAggResult.TaskParameters);
                MultiObjectTrackingSubmittedJob  job       = JSonUtils.ConvertJSonToObject <MultiObjectTrackingSubmittedJob>(aggTask.jobEntry.JobParameters);
                TrackletLabelingAggregatedResult aggresult = JSonUtils.ConvertJSonToObject <TrackletLabelingAggregatedResult>(satyamAggResult.AggregatedResultString);

                WebClient    wb             = new WebClient();
                Stream       aggTrackStream = wb.OpenRead(aggresult.AggregatedTrackletsString_URL);
                StreamReader reader         = new StreamReader(aggTrackStream);
                String       aggTrackString = reader.ReadToEnd();

                MultiObjectTrackingResult aggTracks = JSonUtils.ConvertJSonToObject <MultiObjectTrackingResult>(aggTrackString);

                string        blobDir   = URIUtilities.localDirectoryFullPathFromURI(aggTask.SatyamURI);
                List <string> ImageURLs = satyamStorage.getURLListOfSpecificExtensionUnderSubDirectoryByURI(blobDir, new List <string>()
                {
                    "jpg", "png"
                });


                string videoName = URIUtilities.localDirectoryNameFromURI(blobDir) + "_" + URIUtilities.filenameFromURINoExtension(aggTask.SatyamURI) + "_" + entry.ID;

                MultiObjectTrackingAnalyzer.generateVideoForEvaluation(ImageURLs, aggTracks, directoryName, videoName, job.FrameRate);
            }
        }
Exemplo n.º 22
0
        public static void AggregateWithParameterAndValidateObjectCountingResultByGUID(string guid,
                                                                                       //bool saveImage = false,
                                                                                       bool prepareTrainingSet = false, string outputDirectory = null,
                                                                                       int MinHeight           = TaskConstants.OBJECT_COUNTING_VALIDATION_MIN_HEIGHT,
                                                                                       int MaxOcclusion        = TaskConstants.OBJECT_COUNTING_VALIDATION_MAX_OCCLUSION,
                                                                                       double Max_Truncation   = TaskConstants.OBJECT_COUNTING_VALIDATION_MIN_TRUNCATION,
                                                                                       int MinResults          = TaskConstants.OBJECT_COUNTING_MTURK_MIN_RESULTS_TO_AGGREGATE,
                                                                                       int MaxResults          = TaskConstants.OBJECT_COUNTING_MTURK_MAX_RESULTS_TO_AGGREGATE,
                                                                                       double MAX_ABSOLUTE_COUNT_DEVIATION_LOWERBOUND = TaskConstants.OBJECT_COUNTING_MTURK_MAX_ABSOLUTE_COUNT_DEVIATION_LOWERBOUND,
                                                                                       double MAX_DEVIATION_FRACTION = TaskConstants.OBJECT_COUNTING_MTURK_MAX_DEVIATION_FRACTION,
                                                                                       double SUPER_MAJORITY_VALUE   = TaskConstants.OBJECT_COUNTING_MTURK_SUPER_MAJORITY_VALUE
                                                                                       )
        {
            if (!Directory.Exists(DirectoryConstants.defaultTempDirectory + guid))
            {
                Directory.CreateDirectory(DirectoryConstants.defaultTempDirectory + guid);
            }

            string configString = "Min_" + MinResults + "_Max_" + MaxResults + "_Dev_" + MAX_DEVIATION_FRACTION + "_Majority_" + SUPER_MAJORITY_VALUE;

            Console.WriteLine("Aggregating for param set " + configString);
            int noTerminatedTasks = 0;
            SatyamResultsTableAccess       resultsDB = new SatyamResultsTableAccess();
            List <SatyamResultsTableEntry> entries   = resultsDB.getEntriesByGUIDOrderByID(guid);

            resultsDB.close();

            SortedDictionary <DateTime, List <SatyamResultsTableEntry> > entriesBySubmitTime = SatyamResultValidationToolKit.SortResultsBySubmitTime(entries);

            int noTotalConverged = 0;
            //int noCorrect = 0;

            Dictionary <int, List <SatyamResultsTableEntry> > ResultsPerTask = new Dictionary <int, List <SatyamResultsTableEntry> >();
            List <int> aggregatedTasks = new List <int>();

            List <SatyamAggregatedResultsTableEntry> aggEntries     = new List <SatyamAggregatedResultsTableEntry>();
            Dictionary <int, int> noResultsNeededForAggregation     = SatyamResultsAnalysis.getNoResultsNeededForAggregationFromLog(configString, guid);
            Dictionary <int, int> noResultsNeededForAggregation_new = new Dictionary <int, int>();

            // play back by time
            foreach (DateTime t in entriesBySubmitTime.Keys)
            {
                //Console.WriteLine("Processing Results of time: {0}", t);
                foreach (SatyamResultsTableEntry entry in entriesBySubmitTime[t])
                {
                    SatyamResult satyamResult = JSonUtils.ConvertJSonToObject <SatyamResult>(entry.ResultString);
                    SatyamTask   task         = JSonUtils.ConvertJSonToObject <SatyamTask>(satyamResult.TaskParametersString);
                    SatyamJob    job          = task.jobEntry;
                    string       fileName     = URIUtilities.filenameFromURI(task.SatyamURI);


                    int taskEntryID = entry.SatyamTaskTableEntryID;
                    if (aggregatedTasks.Contains(taskEntryID))
                    {
                        continue;
                    }

                    if (!ResultsPerTask.ContainsKey(taskEntryID))
                    {
                        ResultsPerTask.Add(taskEntryID, new List <SatyamResultsTableEntry>());
                    }
                    ResultsPerTask[taskEntryID].Add(entry);

                    // check log if enough results are collected
                    if (noResultsNeededForAggregation != null && noResultsNeededForAggregation.ContainsKey(taskEntryID) &&
                        ResultsPerTask[taskEntryID].Count < noResultsNeededForAggregation[taskEntryID])
                    {
                        continue;
                    }

                    string aggResultString = ObjectCountingAggregator.GetAggregatedResultString(ResultsPerTask[taskEntryID], MinResults, MaxResults, MAX_ABSOLUTE_COUNT_DEVIATION_LOWERBOUND, MAX_DEVIATION_FRACTION, SUPER_MAJORITY_VALUE);

                    if (aggResultString == null)
                    {
                        continue;
                    }

                    SatyamAggregatedResultsTableEntry aggEntry = new SatyamAggregatedResultsTableEntry();
                    aggEntry.JobGUID                = ResultsPerTask[taskEntryID][0].JobGUID;
                    aggEntry.JobTemplateType        = ResultsPerTask[taskEntryID][0].JobTemplateType;
                    aggEntry.SatyamTaskTableEntryID = taskEntryID;
                    aggEntry.UserID       = ResultsPerTask[taskEntryID][0].UserID;
                    aggEntry.ResultString = aggResultString;
                    /// aggregation happen
                    /// // record logs
                    // record logs
                    if (noResultsNeededForAggregation == null || !noResultsNeededForAggregation.ContainsKey(taskEntryID))
                    {
                        noResultsNeededForAggregation_new.Add(taskEntryID, ResultsPerTask[taskEntryID].Count);
                    }
                    aggEntries.Add(aggEntry);
                    noTotalConverged++;
                    if (ResultsPerTask[taskEntryID].Count >= MaxResults)
                    {
                        noTerminatedTasks++;
                    }
                    aggregatedTasks.Add(taskEntryID);
                }
            }

            SatyamResultsAnalysis.RecordAggregationLog(noResultsNeededForAggregation_new, configString, guid);

            double totalError, totalGroundtruth;

            //ValidateSatyamKITTIObjectCountingAggregationResult(aggEntries, out totalError, out totalGroundtruth,
            //    MinHeight, MaxOcclusion, Max_Truncation);
            ValidateSatyamCARPKObjectCountingAggregationResult(aggEntries, out totalError, out totalGroundtruth);

            Console.WriteLine("noTotalConverged {0}", noTotalConverged);
            Console.WriteLine("noTerminatedTasks {0}", noTerminatedTasks);
            /// local file
            string outputString = String.Format("{0} {1} {2} {3} {4} {5} {6}\n", configString, totalError, totalGroundtruth, totalError / totalGroundtruth, noTotalConverged, noTerminatedTasks, ResultsPerTask.Count - noTotalConverged);
            string outputfile   = DirectoryConstants.defaultTempDirectory + guid + "\\resultSummary.txt";

            File.AppendAllText(outputfile, outputString);

            string approvalString = configString + "_PayDev_" + TaskConstants.OBJECT_COUNTING_MTURK_MAX_ABSOLUTE_COUNT_DEVIATION_FOR_PAYMENT +
                                    "_PayFrac_" + TaskConstants.OBJECT_COUNTING_MTURK_MAX_DEVIATION_FRACTION_FOR_PAYMENT;

            //for (double prob = 0; prob < 1;prob +=0.2)
            //{
            //    SatyamResultsAnalysis.AnalyzeApprovalRate(aggEntries, entries, guid, approvalString, anotherChanceProbablity: prob);
            //}
            //for (double ratio = 0; ratio < 1; ratio += 0.2)
            //{
            //    SatyamResultsAnalysis.AnalyzeApprovalRate(aggEntries, entriesBySubmitTime, noResultsNeededForAggregation, noResultsNeededForAggregation_new, guid, configString, approvalRatioThreshold: ratio);
            //}
            SatyamResultsAnalysis.AggregationAnalysis(aggEntries, entriesBySubmitTime, noResultsNeededForAggregation, noResultsNeededForAggregation_new, guid, configString);
        }
        public static void EvaluateAndPrintConfusionMatrixOfAggregatedResultEntries(List <SatyamAggregatedResultsTableEntry> aggResultEntries,
                                                                                    string outputFile,
                                                                                    out int noCorrect,
                                                                                    //out SortedDictionary<string, Dictionary<string, int>> confusionMatrix_res_groundtruth,
                                                                                    //out SortedDictionary<string, Dictionary<string, int>> confusionMatrix_groundtruth_res,
                                                                                    bool prepareDataForTraining = false, string outputDirectory = null)
        {
            noCorrect = 0;
            //SortedDictionary<string, Dictionary<string, int>> confusionMatrix_res_groundtruth = new SortedDictionary<string, Dictionary<string, int>>();
            //SortedDictionary<string, Dictionary<string, int>> confusionMatrix_groundtruth_res = new SortedDictionary<string, Dictionary<string, int>>();

            List <KeyValuePair <string, string> > detections_gts = new List <KeyValuePair <string, string> >();


            WebClient wc = new WebClient();

            foreach (SatyamAggregatedResultsTableEntry aggResultEntry in aggResultEntries)
            {
                SatyamSaveAggregatedDataSatyam       data      = new SatyamSaveAggregatedDataSatyam(aggResultEntry);
                SingleObjectLabelingAggregatedResult aggResult = JSonUtils.ConvertJSonToObject <SingleObjectLabelingAggregatedResult>(data.AggregatedResultString);
                string fileName = URIUtilities.filenameFromURI(data.SatyamURI);
                string VideoCategoryGroundTruth = getVideoCategoryFromFileName(fileName);

                detections_gts.Add(new KeyValuePair <string, string>(aggResult.Category, VideoCategoryGroundTruth));

                //if (!confusionMatrix_res_groundtruth.ContainsKey(aggResult.Category))
                //{
                //    confusionMatrix_res_groundtruth.Add(aggResult.Category, new Dictionary<string, int>());
                //}
                //if (!confusionMatrix_res_groundtruth[aggResult.Category].ContainsKey(VideoCategoryGroundTruth))
                //{
                //    confusionMatrix_res_groundtruth[aggResult.Category].Add(VideoCategoryGroundTruth, 0);
                //}
                //if (!confusionMatrix_groundtruth_res.ContainsKey(VideoCategoryGroundTruth))
                //{
                //    confusionMatrix_groundtruth_res.Add(VideoCategoryGroundTruth, new Dictionary<string, int>());
                //}
                //if (!confusionMatrix_groundtruth_res[VideoCategoryGroundTruth].ContainsKey(aggResult.Category))
                //{
                //    confusionMatrix_groundtruth_res[VideoCategoryGroundTruth].Add(aggResult.Category, 0);
                //}
                //confusionMatrix_res_groundtruth[aggResult.Category][VideoCategoryGroundTruth]++;
                //confusionMatrix_groundtruth_res[VideoCategoryGroundTruth][aggResult.Category]++;

                if (aggResult.Category.Equals(VideoCategoryGroundTruth, StringComparison.InvariantCultureIgnoreCase))
                {
                    noCorrect++;
                }
                else
                {
                    Console.WriteLine("{0}, Groundtruth: {1}, Aggregated: {2}, Votes: {3}",
                                      fileName, VideoCategoryGroundTruth, aggResult.Category,
                                      JSonUtils.ConvertObjectToJSon(aggResult.metaData));
                }



                // prepare training dataset
                if (prepareDataForTraining)
                {
                    if (!Directory.Exists(outputDirectory + "\\" + aggResult.Category))
                    {
                        Directory.CreateDirectory(outputDirectory + "\\" + aggResult.Category);
                    }
                    wc.DownloadFile(data.SatyamURI, outputDirectory + "\\" + aggResult.Category + "\\" + fileName);
                }
            }


            SortedDictionary <string, Dictionary <string, int> > confusionMatrix_res_groundtruth;
            SortedDictionary <string, Dictionary <string, int> > confusionMatrix_groundtruth_res;

            SatyamResultValidationToolKit.getConfusionMat(detections_gts, out confusionMatrix_res_groundtruth, out confusionMatrix_groundtruth_res);
            SatyamResultValidationToolKit.printConfusionMatrix(confusionMatrix_groundtruth_res, confusionMatrix_res_groundtruth, outputFile);
        }
        public static void SaveResultImagesLocally(List <SatyamResultsTableEntry> entries, string directoryName)
        {
            if (!Directory.Exists(directoryName))
            {
                Directory.CreateDirectory(directoryName);
            }

            directoryName = directoryName + "\\Raw";

            if (!Directory.Exists(directoryName))
            {
                Directory.CreateDirectory(directoryName);
            }

            // sort by task id
            SortedDictionary <int, List <SatyamResultsTableEntry> > taskResults = new SortedDictionary <int, List <SatyamResultsTableEntry> >();

            for (int i = 0; i < entries.Count; i++)
            {
                if (!taskResults.ContainsKey(entries[i].SatyamTaskTableEntryID))
                {
                    taskResults.Add(entries[i].SatyamTaskTableEntryID, new List <SatyamResultsTableEntry>());
                }
                taskResults[entries[i].SatyamTaskTableEntryID].Add(entries[i]);
            }

            foreach (int taskID in taskResults.Keys)
            {
                for (int i = 0; i < taskResults[taskID].Count; i++)
                {
                    SatyamResultsTableEntry entry        = taskResults[taskID][i];
                    SatyamResult            satyamResult = JSonUtils.ConvertJSonToObject <SatyamResult>(entry.ResultString);
                    SatyamTask task             = JSonUtils.ConvertJSonToObject <SatyamTask>(satyamResult.TaskParametersString);
                    SatyamJob  job              = task.jobEntry;
                    ImageSegmentationResult res = JSonUtils.ConvertJSonToObject <ImageSegmentationResult>(satyamResult.TaskResult);
                    if (res == null)
                    {
                        continue;
                    }

                    string fileName = URIUtilities.filenameFromURINoExtension(task.SatyamURI);



                    fileName = fileName + "-Result";
                    if (satyamResult.amazonInfo.AssignmentID != "")
                    {
                        fileName = fileName + "-" + satyamResult.amazonInfo.AssignmentID;
                    }
                    fileName += "-" + entry.ID;


                    if (File.Exists(directoryName + "\\" + fileName + ".jpg"))
                    {
                        continue;
                    }

                    Console.WriteLine("Saving Turker Result {0}", fileName);

                    Image originalImage = ImageUtilities.getImageFromURI(task.SatyamURI);

                    Image ResultImage = DrawResultsOnImage(res, originalImage);

                    byte[] png = ImageSegmentationResult.PolygonResult2Bitmap(res);

                    if (ResultImage == null)
                    {
                        continue;
                    }

                    ImageUtilities.saveImage(ResultImage, directoryName, fileName);

                    ImageUtilities.savePNGRawData(directoryName + "\\" + fileName + "_bitmap.jpg", originalImage.Width, originalImage.Height, png);
                }
            }
        }
        public static void AggregateWithParameterAndValidateSatyamVideoClassificationResultByGUID(string guid,
                                                                                                  int MinResults                    = TaskConstants.SINGLE_OBJECT_LABLING_MTURK_MIN_RESULTS_TO_AGGREGATE,
                                                                                                  int MaxResults                    = TaskConstants.SINGLE_OBJECT_LABLING_MTURK_MAX_RESULTS_TO_AGGREGATE,
                                                                                                  double MajorityThreshold          = TaskConstants.SINGLE_OBJECT_LABLING_MTURK_MAJORITY_THRESHOLD,
                                                                                                  bool prepareDataForTraining       = false, string outputDirectory = null,
                                                                                                  string confusingVideoListFilePath = null)
        {
            string configString = "Min_" + MinResults + "_Max_" + MaxResults + "_Thresh_" + MajorityThreshold;

            Console.WriteLine("Aggregating for param set " + configString);
            int noTerminatedTasks = 0;
            SatyamResultsTableAccess       resultsDB = new SatyamResultsTableAccess();
            List <SatyamResultsTableEntry> entries   = resultsDB.getEntriesByGUIDOrderByID(guid);

            resultsDB.close();


            SortedDictionary <DateTime, List <SatyamResultsTableEntry> > entriesBySubmitTime = SatyamResultValidationToolKit.SortResultsBySubmitTime(entries);

            int noTotalConverged = 0;
            int noCorrect        = 0;

            Dictionary <int, List <SatyamResultsTableEntry> > ResultsPerTask = new Dictionary <int, List <SatyamResultsTableEntry> >();
            List <int> aggregatedTasks = new List <int>();

            SortedDictionary <string, Dictionary <string, int> > confusionMatrix_res_groundtruth = new SortedDictionary <string, Dictionary <string, int> >();
            SortedDictionary <string, Dictionary <string, int> > confusionMatrix_groundtruth_res = new SortedDictionary <string, Dictionary <string, int> >();

            List <SatyamAggregatedResultsTableEntry> aggEntries = new List <SatyamAggregatedResultsTableEntry>();

            Dictionary <int, int> noResultsNeededForAggregation     = SatyamResultsAnalysis.getNoResultsNeededForAggregationFromLog(configString, guid);
            Dictionary <int, int> noResultsNeededForAggregation_new = new Dictionary <int, int>();

            foreach (DateTime t in entriesBySubmitTime.Keys)
            {
                //Console.WriteLine("Processing Results of time: {0}", t);
                foreach (SatyamResultsTableEntry entry in entriesBySubmitTime[t])
                {
                    SatyamResult satyamResult             = JSonUtils.ConvertJSonToObject <SatyamResult>(entry.ResultString);
                    SatyamTask   task                     = JSonUtils.ConvertJSonToObject <SatyamTask>(satyamResult.TaskParametersString);
                    SatyamJob    job                      = task.jobEntry;
                    string       fileName                 = URIUtilities.filenameFromURI(task.SatyamURI);
                    string       VideoCategoryGroundTruth = getVideoCategoryFromFileName(fileName);

                    int taskEntryID = entry.SatyamTaskTableEntryID;
                    if (aggregatedTasks.Contains(taskEntryID))
                    {
                        continue;
                    }

                    if (!ResultsPerTask.ContainsKey(taskEntryID))
                    {
                        ResultsPerTask.Add(taskEntryID, new List <SatyamResultsTableEntry>());
                    }

                    ResultsPerTask[taskEntryID].Add(entry);

                    // check log if enough results are collected
                    if (noResultsNeededForAggregation != null && noResultsNeededForAggregation.ContainsKey(taskEntryID) &&
                        ResultsPerTask[taskEntryID].Count < noResultsNeededForAggregation[taskEntryID])
                    {
                        continue;
                    }

                    //SingleObjectLabelingAggregatedResult aggResult = SingleObjectLabelingAggregator.getAggregatedResult(ResultsPerTask[taskEntryID], MinResults, MaxResult, MajorityThreshold);
                    string aggResultString = SingleObjectLabelingAggregator.GetAggregatedResultString(ResultsPerTask[taskEntryID], MinResults, MaxResults, MajorityThreshold);

                    if (aggResultString == null)
                    {
                        continue;
                    }

                    SatyamAggregatedResultsTableEntry aggEntry = new SatyamAggregatedResultsTableEntry();
                    aggEntry.JobGUID                = ResultsPerTask[taskEntryID][0].JobGUID;
                    aggEntry.JobTemplateType        = ResultsPerTask[taskEntryID][0].JobTemplateType;
                    aggEntry.SatyamTaskTableEntryID = taskEntryID;
                    aggEntry.UserID       = ResultsPerTask[taskEntryID][0].UserID;
                    aggEntry.ResultString = aggResultString;

                    /// aggregation happen
                    // record logs
                    if (noResultsNeededForAggregation == null || !noResultsNeededForAggregation.ContainsKey(taskEntryID))
                    {
                        noResultsNeededForAggregation_new.Add(taskEntryID, ResultsPerTask[taskEntryID].Count);
                    }

                    aggEntries.Add(aggEntry);
                    noTotalConverged++;
                    if (ResultsPerTask[taskEntryID].Count >= MaxResults)
                    {
                        noTerminatedTasks++;
                    }
                    aggregatedTasks.Add(taskEntryID);
                }
            }

            SatyamResultsAnalysis.RecordAggregationLog(noResultsNeededForAggregation_new, configString, guid);


            string outputmatFile = DirectoryConstants.defaultTempDirectory + guid + "\\" + configString + "_mat.txt";

            EvaluateAndPrintConfusionMatrixOfAggregatedResultEntries(aggEntries, outputmatFile, out noCorrect, prepareDataForTraining, outputDirectory);


            Console.WriteLine("noTotalConverged {0}", noTotalConverged);
            Console.WriteLine("noTerminatedTasks {0}", noTerminatedTasks);
            Console.WriteLine("Result: {0}/{1}, precision: {2}", noCorrect, noTotalConverged, (double)noCorrect / noTotalConverged);
            /// local file
            string outputString = String.Format("{0} {1} {2} {3} {4} {5}\n", configString, noCorrect, noTotalConverged, (double)noCorrect / noTotalConverged, noTerminatedTasks, ResultsPerTask.Count - noTotalConverged);
            string outputfile   = DirectoryConstants.defaultTempDirectory + guid + "\\resultSummary.txt";

            File.AppendAllText(outputfile, outputString);



            //for (double prob = 0; prob < 1;prob +=0.2)
            //{
            //    SatyamResultsAnalysis.AnalyzeApprovalRate(aggEntries, entries, guid, configString, anotherChanceProbablity: prob);
            //}
            //for (double ratio = 0; ratio < 1; ratio += 0.2)
            //{
            //    SatyamResultsAnalysis.AnalyzeApprovalRate(aggEntries, entriesBySubmitTime, noResultsNeededForAggregation, noResultsNeededForAggregation_new, guid, configString, approvalRatioThreshold: ratio);
            //}
            SatyamResultsAnalysis.AggregationAnalysis(aggEntries, entriesBySubmitTime, noResultsNeededForAggregation, noResultsNeededForAggregation_new, guid, configString);
        }
        public static MultiObjectTrackingResult stitchAllChunksAllObjectsOfOneVideo(List <SatyamAggregatedResultsTableEntry> entries, out List <string> ImageURLs, out int fps)
        {
            ImageURLs = new List <string>();
            fps       = 0;
            if (entries.Count == 0)
            {
                return(null);
            }

            SatyamJobStorageAccountAccess satyamStorage = new SatyamJobStorageAccountAccess();
            MultiObjectTrackingResult     stitched      = new MultiObjectTrackingResult();

            int totalFrameCounts = 0;

            // ensure the order is correct
            SortedDictionary <int, List <SatyamAggregatedResultsTableEntry> > sortedEntries = new SortedDictionary <int, List <SatyamAggregatedResultsTableEntry> >();
            List <int> idx = new List <int>();

            for (int i = 0; i < entries.Count; i++)
            {
                SatyamAggregatedResultsTableEntry entry           = entries[i];
                SatyamAggregatedResult            satyamAggResult = JSonUtils.ConvertJSonToObject <SatyamAggregatedResult>(entry.ResultString);
                SatyamTask aggTask = JSonUtils.ConvertJSonToObject <SatyamTask>(satyamAggResult.TaskParameters);
                string     video   = URIUtilities.localDirectoryNameFromURI(aggTask.SatyamURI);
                string[]   fields  = video.Split('_');

                int startingFrame = Convert.ToInt32(fields[fields.Length - 1]);
                if (!sortedEntries.ContainsKey(startingFrame))
                {
                    sortedEntries.Add(startingFrame, new List <SatyamAggregatedResultsTableEntry>());
                    idx.Add(startingFrame);
                }
                sortedEntries[startingFrame].Add(entries[i]);
            }


            idx.Sort();


            List <string> AggObjIds = new List <string>();

            for (int i = 0; i < idx.Count; i++)
            {
                int    noFramesOverlap = 0;
                string blobDir         = "";
                // grouping all objects that belong to the same chunk
                MultiObjectTrackingResult aggTracksOfAllObjectsPerChunk = new MultiObjectTrackingResult();

                List <string> objIds = new List <string>();
                for (int j = 0; j < sortedEntries[idx[i]].Count; j++)
                {
                    SatyamAggregatedResultsTableEntry entry           = sortedEntries[idx[i]][j];
                    SatyamAggregatedResult            satyamAggResult = JSonUtils.ConvertJSonToObject <SatyamAggregatedResult>(entry.ResultString);
                    SatyamTask aggTask = JSonUtils.ConvertJSonToObject <SatyamTask>(satyamAggResult.TaskParameters);
                    MultiObjectTrackingSubmittedJob job = JSonUtils.ConvertJSonToObject <MultiObjectTrackingSubmittedJob>(aggTask.jobEntry.JobParameters);
                    if (job.ChunkOverlap != 0.0)
                    {
                        noFramesOverlap = (int)(job.ChunkOverlap * job.FrameRate);
                    }
                    fps     = job.FrameRate;
                    blobDir = URIUtilities.localDirectoryFullPathFromURI(aggTask.SatyamURI);

                    string objId = URIUtilities.filenameFromURINoExtension(aggTask.SatyamURI);
                    objIds.Add(objId);

                    TrackletLabelingAggregatedResult aggresult = JSonUtils.ConvertJSonToObject <TrackletLabelingAggregatedResult>(satyamAggResult.AggregatedResultString);

                    WebClient    wb             = new WebClient();
                    Stream       aggTrackStream = wb.OpenRead(aggresult.AggregatedTrackletsString_URL);
                    StreamReader reader         = new StreamReader(aggTrackStream);
                    String       aggTrackString = reader.ReadToEnd();

                    MultiObjectTrackingResult aggTracks = JSonUtils.ConvertJSonToObject <MultiObjectTrackingResult>(aggTrackString);
                    if (aggTracksOfAllObjectsPerChunk.tracks.Count == 0)
                    {
                        aggTracksOfAllObjectsPerChunk = aggTracks;
                    }
                    else
                    {
                        for (int k = 0; k < aggTracks.tracks.Count; k++)
                        {
                            aggTracksOfAllObjectsPerChunk.tracks.Add(aggTracks.tracks[k]);
                        }
                    }
                }

                List <string> TraceURLs = satyamStorage.getURLListOfSpecificExtensionUnderSubDirectoryByURI(blobDir, new List <string>()
                {
                    "jpg", "png"
                });

                if (i == 0)
                {
                    ImageURLs.AddRange(TraceURLs);

                    stitched          = aggTracksOfAllObjectsPerChunk;
                    totalFrameCounts += TraceURLs.Count;
                    AggObjIds         = objIds;
                }
                else
                {
                    int noNewFrames = 0;
                    for (int j = noFramesOverlap; j < TraceURLs.Count; j++)
                    {
                        ImageURLs.Add(TraceURLs[j]);
                        noNewFrames++;
                    }

                    //stitched = MultiObjectTrackingAnalyzer.stitchTwoTracesByTubeletsOfOverlappingVideoChunk(stitched,
                    //    aggTracksOfAllObjectsPerChunk, totalFrameCounts, totalFrameCounts + noNewFrames, noFramesOverlap, fps);

                    // postpone the agg trace
                    double frameTimeInMiliSeconds       = (double)1000 / (double)fps;
                    double timeToPostponeInMilliSeconds = (double)(totalFrameCounts - noFramesOverlap) * frameTimeInMiliSeconds;
                    //TimeSpan timeSpanToPostponeInSeconds = new TimeSpan(0,0,0,0, (int)timeToPostponeInMilliSeconds);
                    TimeSpan timeSpanToPostponeInSeconds = DateTimeUtilities.getTimeSpanFromTotalMilliSeconds((int)timeToPostponeInMilliSeconds);
                    aggTracksOfAllObjectsPerChunk.postpone(timeSpanToPostponeInSeconds);

                    // overlap must be 0
                    for (int k = 0; k < aggTracksOfAllObjectsPerChunk.tracks.Count; k++)
                    {
                        if (!AggObjIds.Contains(objIds[k]))
                        {
                            stitched.tracks.Add(aggTracksOfAllObjectsPerChunk.tracks[k]);
                            AggObjIds.Add(objIds[k]);
                        }
                        else
                        {
                            // stitch the track for the same id
                            int tckIdx = AggObjIds.IndexOf(objIds[k]);
                            stitched.tracks[tckIdx] = CompressedTrack.stitchTwoAdjacentTrack(stitched.tracks[tckIdx], aggTracksOfAllObjectsPerChunk.tracks[k]);
                        }
                    }



                    totalFrameCounts += noNewFrames;
                }

                //debug
                //generateVideoForEvaluation(ImageURLs, stitched, directoryName + "_" + i, videoName, fps);
            }
            return(stitched);
        }
Exemplo n.º 27
0
        public static void AggregateWithParameter(string guid,
                                                  int MinResults = TaskConstants.TRACKLET_LABELING_MTURK_MIN_RESULTS_TO_AGGREGATE,
                                                  int MaxResults = TaskConstants.TRACKLET_LABELING_MTURK_MAX_RESULTS_TO_AGGREGATE,
                                                  double boxToleranceThreshold = TaskConstants.TRACKLET_LABELING_BOX_DEVIATION_THRESHOLD,
                                                  double ObjectCoverageApprovalThresholdPerVideo = TaskConstants.TRACKLET_LABELING_APPROVALRATIO_PER_VIDEO,
                                                  double BoxCoverageApprovalThresholdPerTrack    = TaskConstants.TRACKLET_LABELING_APPROVALRATIO_PER_TRACK,
                                                  int consensusNumber = TaskConstants.TRACKLET_LABELING_MIN_RESULTS_FOR_CONSENSUS,
                                                  double minTubeletIoUSimilarityThreshold = TaskConstants.TRACKLET_LABELING_MIN_TUBELET_SIMILARITY_THRESHOLD,
                                                  double attributeMajority      = TaskConstants.TRACKLET_LABELING_MTURK_ATTRIBUTE_MAJORITY_THRESHOLD,
                                                  bool allFalseAttributeInvalid = false
                                                  )
        {
            string configString = "Min_" + MinResults + "_Max_" + MaxResults + "_IoU_" + minTubeletIoUSimilarityThreshold + "_Ratio_" + ObjectCoverageApprovalThresholdPerVideo;

            Console.WriteLine("Aggregating for " + guid + " with param set " + configString);
            SatyamResultsTableAccess       resultsDB = new SatyamResultsTableAccess();
            List <SatyamResultsTableEntry> entries   = resultsDB.getEntriesByGUID(guid);

            resultsDB.close();

            SortedDictionary <DateTime, List <SatyamResultsTableEntry> > entriesBySubmitTime = SatyamResultValidationToolKit.SortResultsBySubmitTime_OneResultPerTurkerPerTask(entries);

            Dictionary <int, List <MultiObjectTrackingResult> > ResultsPerTask = new Dictionary <int, List <MultiObjectTrackingResult> >();
            List <int> aggregatedTasks = new List <int>();

            int noTotalConverged = 0;
            //int noCorrect = 0;
            int noTerminatedTasks = 0;

            List <SatyamAggregatedResultsTableEntry> aggEntries = new List <SatyamAggregatedResultsTableEntry>();

            Dictionary <int, int> noResultsNeededForAggregation     = SatyamResultsAnalysis.getNoResultsNeededForAggregationFromLog(configString, guid);
            Dictionary <int, int> noResultsNeededForAggregation_new = new Dictionary <int, int>();
            // play back by time
            Dictionary <int, List <string> > WorkersPerTask = new Dictionary <int, List <string> >();

            foreach (DateTime t in entriesBySubmitTime.Keys)
            {
                //Console.WriteLine("Processing Results of time: {0}", t);
                List <SatyamResultsTableEntry> ResultEntries = entriesBySubmitTime[t];
                foreach (SatyamResultsTableEntry entry in ResultEntries)
                {
                    SatyamResult satyamResult           = JSonUtils.ConvertJSonToObject <SatyamResult>(entry.ResultString);
                    SatyamTask   task                   = JSonUtils.ConvertJSonToObject <SatyamTask>(satyamResult.TaskParametersString);
                    MultiObjectTrackingSubmittedJob job = JSonUtils.ConvertJSonToObject <MultiObjectTrackingSubmittedJob>(task.jobEntry.JobParameters);
                    string fileName    = URIUtilities.filenameFromURINoExtension(task.SatyamURI);
                    int    taskEntryID = entry.SatyamTaskTableEntryID;
                    if (aggregatedTasks.Contains(taskEntryID))
                    {
                        continue;
                    }
                    if (!ResultsPerTask.ContainsKey(taskEntryID))
                    {
                        ResultsPerTask.Add(taskEntryID, new List <MultiObjectTrackingResult>());
                        WorkersPerTask.Add(taskEntryID, new List <string>());
                    }

                    // remove duplicate workers result
                    string workerID = satyamResult.amazonInfo.WorkerID;
                    if (WorkersPerTask[taskEntryID].Contains(workerID))
                    {
                        continue;
                    }
                    //enclose only non-duplicate results, one per each worker.
                    WorkersPerTask[taskEntryID].Add(workerID);



                    string   videoName        = URIUtilities.localDirectoryNameFromURI(task.SatyamURI);
                    string[] fields           = videoName.Split('_');
                    int      startingFrame    = Convert.ToInt32(fields[fields.Length - 1]);
                    int      maxChunkEndFrame = startingFrame + job.ChunkDuration * job.FrameRate;
                    int      noFrameOverlap   = (int)(job.ChunkOverlap * job.FrameRate);
                    if (startingFrame != 0)
                    {
                        startingFrame -= noFrameOverlap;
                    }



                    string blobDir = URIUtilities.localDirectoryFullPathFromURI(task.SatyamURI);
                    VATIC_DVA_CrowdsourcedResult taskr = TrackletLabelingAggregator.createVATIC_DVA_CrowdsourcedResultUsingSatyamBlobImageCount(satyamResult.TaskResult, blobDir, entry.SatyamTaskTableEntryID.ToString(), job.FrameRate);
                    //MultiObjectTrackingResult result_tmp = vatic_tmp.getCompressedTracksInTimeSegment();

                    //VATIC_DVA_CrowdsourcedResult taskr = new VATIC_DVA_CrowdsourcedResult(satyamResult.TaskResult, videoName, entry.ID.ToString(), ImageURLs.Count, job.FrameRate);
                    MultiObjectTrackingResult res = taskr.getCompressedTracksInTimeSegment();

                    if (allFalseAttributeInvalid)
                    {
                        if (TrackletLabelingAggregator.AllAttributeAllFalse(res))
                        {
                            continue;
                        }
                    }

                    ResultsPerTask[taskEntryID].Add(res);

                    // check log if enough results are collected

                    if (noResultsNeededForAggregation != null && noResultsNeededForAggregation.ContainsKey(taskEntryID) &&
                        ResultsPerTask[taskEntryID].Count < noResultsNeededForAggregation[taskEntryID])
                    {
                        continue;
                    }



                    // hack for masters
                    int tempMin = MinResults;
                    if (TaskConstants.masterGUIDs.Contains(entry.JobGUID))
                    {
                        tempMin = 1;
                    }

                    TrackletLabelingAggregatedResult aggResult = TrackletLabelingAggregator.getAggregatedResult(ResultsPerTask[taskEntryID],
                                                                                                                guid, task.SatyamURI,
                                                                                                                job.FrameRate, job.BoundaryLines,
                                                                                                                MinResults, MaxResults, boxToleranceThreshold,
                                                                                                                ObjectCoverageApprovalThresholdPerVideo,
                                                                                                                BoxCoverageApprovalThresholdPerTrack,
                                                                                                                consensusNumber, minTubeletIoUSimilarityThreshold,
                                                                                                                attributeMajority);

                    if (aggResult == null)
                    {
                        continue;
                    }

                    //////////////// aggregation happen
                    // record logs
                    if (noResultsNeededForAggregation == null || !noResultsNeededForAggregation.ContainsKey(taskEntryID))
                    {
                        noResultsNeededForAggregation_new.Add(taskEntryID, ResultsPerTask[taskEntryID].Count);
                    }



                    aggregatedTasks.Add(taskEntryID);
                    noTotalConverged++;
                    if (ResultsPerTask[taskEntryID].Count >= MaxResults)
                    {
                        noTerminatedTasks++;
                    }
                    SatyamAggregatedResult SatyamAggResult = new SatyamAggregatedResult();
                    SatyamAggResult.SatyamTaskTableEntryID = taskEntryID;
                    SatyamAggResult.AggregatedResultString = JSonUtils.ConvertObjectToJSon <TrackletLabelingAggregatedResult>(aggResult);
                    SatyamAggResult.TaskParameters         = JSonUtils.ConvertJSonToObject <SatyamResult>(entry.ResultString).TaskParametersString;

                    SatyamAggregatedResultsTableEntry aggEntry = new SatyamAggregatedResultsTableEntry();
                    aggEntry.SatyamTaskTableEntryID = taskEntryID;
                    aggEntry.JobGUID      = entry.JobGUID;
                    aggEntry.ResultString = JSonUtils.ConvertObjectToJSon <SatyamAggregatedResult>(SatyamAggResult);


                    aggEntries.Add(aggEntry);
                }
            }


            Console.WriteLine("Total_Aggregated_Tasks: {0}", noTotalConverged);
            Console.WriteLine("Total_Terminated_Tasks: {0}", noTerminatedTasks);

            SatyamResultsAnalysis.RecordAggregationLog(noResultsNeededForAggregation_new, configString, guid);

            TrackletLabelingAnalyzer.GroupEntriesByVideoNameAndStitchAndSaveAggregatedResultVideosLocally(aggEntries, DirectoryConstants.defaultTempDirectory + guid);
        }
Exemplo n.º 28
0
        /// <summary>
        /// Input should be a folder of videos and corresponding annotation file with the same name
        /// </summary>
        /// <param name="jobEntry"></param>
        public static bool ProcessAndUploadToAzureBlob(SatyamJobSubmissionsTableAccessEntry jobEntry)
        {
            // if the input is a folder of folders of frames, then copy directly
            SatyamJobStorageAccountAccess satyamStorage = new SatyamJobStorageAccountAccess();
            string    satyamContainerName = SatyamTaskGenerator.JobTemplateToSatyamContainerNameMap[jobEntry.JobTemplateType];
            string    GUID = jobEntry.JobGUID;
            string    satyamDirectoryName = GUID;
            SatyamJob job = JSonUtils.ConvertJSonToObject <SatyamJob>(jobEntry.JobParametersString);
            MultiObjectTrackingSubmittedJob jobParams = JSonUtils.ConvertJSonToObject <MultiObjectTrackingSubmittedJob>(job.JobParameters);

            BlobContainerManager bcm = new BlobContainerManager();
            string        status     = bcm.Connect(job.azureInformation.AzureBlobStorageConnectionString);
            List <string> FileTypes  = SatyamTaskGenerator.ValidFileTypesByTemplate[job.JobTemplateType];

            if (status != "SUCCESS")
            {
                return(false);
            }

            string guidFolder = DirectoryConstants.defaultTempDirectory + "\\" + GUID;

            Directory.CreateDirectory(guidFolder);
            int    chunkLength  = jobParams.ChunkDuration; // sec
            int    outputFPS    = jobParams.FrameRate;
            double chunkOverlap = jobParams.ChunkOverlap;  // sec

            var client = new WebClient();

            // sample to frames
            int           noFramePerChunk = (int)(chunkLength * outputFPS);
            int           noFrameOverlap  = (int)(chunkOverlap * outputFPS);
            List <string> videoUrls       = bcm.getURLList(job.azureInformation.AzureBlobStorageContainerName, job.azureInformation.AzureBlobStorageContainerDirectoryName);


            Dictionary <string, List <string> > videos = new Dictionary <string, List <string> >();

            foreach (string videolink in videoUrls)
            {
                string videoName = URIUtilities.filenameFromURINoExtension(videolink);

                if (!videos.ContainsKey(videoName))
                {
                    videos.Add(videoName, new List <string>());
                }
                videos[videoName].Add(videolink);
            }

            foreach (string videoName in videos.Keys)
            {
                // filter out those that doesn't provide a annotation file with it....
                if (videos[videoName].Count != 2)
                {
                    Console.WriteLine("Warning: Not 2 files provided for {0}.", videoName);
                    //Directory.Delete(guidFolder, true);
                    //return false;
                    continue;
                }

                string videoURL      = "";
                string annotationURL = "";
                foreach (string fileLink in videos[videoName])
                {
                    string extension = URIUtilities.fileExtensionFromURI(fileLink);
                    if (extension != "txt")
                    {
                        videoURL = fileLink;
                    }
                    else
                    {
                        annotationURL = fileLink;
                    }
                }

                string outputDirectory = guidFolder + "\\" + videoName;
                Directory.CreateDirectory(outputDirectory);

                string videoNameWithExtension = URIUtilities.filenameFromURI(videoURL);
                client.DownloadFile(videoURL, outputDirectory + "\\" + videoNameWithExtension);
                FFMpegWrappers.ExtractFrames(DirectoryConstants.ffmpeg, outputDirectory + "\\" + videoNameWithExtension, outputDirectory, videoName, DateTime.Now, outputFPS);

                File.Delete(outputDirectory + "\\" + videoNameWithExtension);


                List <string> chunkFolders = TrackingDataPreprocessor.GroupFramesIntoChunks(outputDirectory, noFramePerChunk);

                //parse VIRAT annotation file
                string annotationNameWithExtension = URIUtilities.filenameFromURI(annotationURL);
                client.DownloadFile(annotationURL, outputDirectory + "\\" + annotationNameWithExtension);

                parseAnnotationFileIntoChunkFolders(chunkFolders,
                                                    outputDirectory + "\\" + annotationNameWithExtension,
                                                    noFramePerChunk, outputFPS);

                //upload
                for (int i = 0; i < chunkFolders.Count; i++)
                {
                    string subDir = chunkFolders[i];
                    satyamStorage.uploadALocalFolder(subDir, satyamContainerName, satyamDirectoryName + "/Video_" + videoName + "_startingFrame_" + noFramePerChunk * i);
                }

                Directory.Delete(outputDirectory, true);
            }
            return(true);
        }
Exemplo n.º 29
0
        public static ImageSegmentationAggregatedResult_NoHoles getAggregatedResult(List <ImageSegmentationResult_NoHoles> results,
                                                                                    string SatyamURL,
                                                                                    string guid,
                                                                                    int MinResults = TaskConstants.IMAGE_SEGMENTATION_MTURK_MIN_RESULTS_TO_AGGREGATE,
                                                                                    int MaxResults = TaskConstants.IMAGE_SEGMENTATION_MTURK_MAX_RESULTS_TO_AGGREGATE,
                                                                                    double CategoryMajorityThreshold        = TaskConstants.IMAGE_SEGMENTATION_MTURK_MAJORITY_CATEGORY_THRESHOLD,
                                                                                    double PolygonBoundaryMajorityThreshold = TaskConstants.IMAGE_SEGMENTATION_MTURK_MAJORITY_POLYGON_BOUNDARY_THRESHOLD,
                                                                                    double ObjectsCoverageThreshold         = TaskConstants.IMAGE_SEGMENTATION_MTURK_OBJECT_COVERAGE_THRESHOLD_FOR_AGGREGATION_TERMINATION
                                                                                    )
        {
            if (results.Count < MinResults) //need at least three results!
            {
                return(null);
            }

            int ImageWidth  = results[0].imageWidth;
            int ImageHeight = results[0].imageHeight;

            byte[] PNG = new byte[ImageHeight * ImageWidth];


            ImageSegmentationAggregatedResult_NoHoles aggResult = new ImageSegmentationAggregatedResult_NoHoles();

            aggResult.metaData            = new ImageSegmentationAggregatedResultMetaData_NoHoles();
            aggResult.metaData.TotalCount = results.Count;

            for (int i = 0; i < PNG.Length; i++)
            {
                PNG[i] = 0;
            }

            aggResult.boxesAndCategories         = new ImageSegmentationResult_NoHoles();
            aggResult.boxesAndCategories.objects = new List <ImageSegmentationResultSingleEntry_NoHoles>();
            aggResult.boxesAndCategories.displayScaleReductionX = results[0].displayScaleReductionX;
            aggResult.boxesAndCategories.displayScaleReductionY = results[0].displayScaleReductionY;
            aggResult.boxesAndCategories.imageHeight            = ImageHeight;
            aggResult.boxesAndCategories.imageWidth             = ImageWidth;

            //first use multipartitie wieghted matching to associated the boxes disregarding the labels since
            //people might make mistake with lables but boxes are usually right
            List <List <GenericPolygon> > AllPolygons = new List <List <GenericPolygon> >();
            List <int> noPolygonsPerResult            = new List <int>();

            foreach (ImageSegmentationResult_NoHoles res in results)
            {
                AllPolygons.Add(new List <GenericPolygon>());

                if (res == null)
                {
                    noPolygonsPerResult.Add(0);
                    continue;
                }

                List <GenericPolygon> polygonList = AllPolygons[AllPolygons.Count - 1];

                foreach (ImageSegmentationResultSingleEntry_NoHoles entry in res.objects)
                {
                    polygonList.Add(entry.polygon);
                }
                noPolygonsPerResult.Add(polygonList.Count);
            }
            //now associate boxes across the various results
            List <MultipartiteWeightedMatch> polyAssociation = PolygonAssociation.computeGenericPolygonAssociations(AllPolygons);
            int noObjects            = polyAssociation.Count;
            int noAssociatedPolygons = 0;

            //how many of the drawn boxes for each result were actually associated by two or more people for each user?
            SortedDictionary <int, int> noMultipleAssociatedPolygonsPerResult = new SortedDictionary <int, int>();

            for (int i = 0; i < results.Count; i++)
            {
                noMultipleAssociatedPolygonsPerResult.Add(i, 0);
            }
            foreach (MultipartiteWeightedMatch match in polyAssociation)
            {
                if (match.elementList.Count > 1) //this has been corroborated by two people
                {
                    noAssociatedPolygons++;
                    foreach (KeyValuePair <int, int> entry in match.elementList)
                    {
                        noMultipleAssociatedPolygonsPerResult[entry.Key]++;
                    }
                }
            }

            //count how many people have a high association ratio
            int noHighAssociationRatio = 0;

            for (int i = 0; i < results.Count; i++)
            {
                if (noPolygonsPerResult[i] == 0)
                {
                    continue;
                }
                double ratio = (double)noMultipleAssociatedPolygonsPerResult[i] / (double)noPolygonsPerResult[i];
                if (ratio > ObjectsCoverageThreshold)
                {
                    noHighAssociationRatio++;
                }
            }
            if (noHighAssociationRatio < MinResults) //at least three people should have all their boxes highly corroborated by one other person
            {
                return(null);
            }

            //int noAcceptedPolygons = 0;

            for (int idx = 0; idx < polyAssociation.Count; idx++)
            {
                MultipartiteWeightedMatch match       = polyAssociation[idx];
                List <GenericPolygon>     polyList    = new List <GenericPolygon>();
                List <string>             identifiers = new List <string>();
                foreach (KeyValuePair <int, int> entry in match.elementList)
                {
                    polyList.Add(AllPolygons[entry.Key][entry.Value]);
                    identifiers.Add(entry.Key + "_" + entry.Value);
                }

                //GenericPolygon aggregatedPolygon = GetAggregatedGenericPolygon_Relaxation(polyList, ImageWidth, ImageHeight, PolygonBoundaryMajorityThreshold);
                //GenericPolygon aggregatedPolygon = GetAggregatedGenericPolygon_MajorityEdge(polyList, ImageWidth, ImageHeight, PolygonBoundaryMajorityThreshold);

                GenericPolygon aggregatedPolygon = new GenericPolygon();// dummy polygon
                byte[]         png = GetAggregatedGenericPolygon_PixelSweep(polyList, ImageWidth, ImageHeight, PolygonBoundaryMajorityThreshold);


                Dictionary <string, int> categoryNames = new Dictionary <string, int>();
                int    totalCount  = match.elementList.Count;
                int    maxCount    = 0;
                string maxCategory = "";
                foreach (KeyValuePair <int, int> entry in match.elementList)
                {
                    string category = results[entry.Key].objects[entry.Value].Category;
                    if (!categoryNames.ContainsKey(category))
                    {
                        categoryNames.Add(category, 0);
                    }
                    categoryNames[category]++;
                    if (maxCount < categoryNames[category])
                    {
                        maxCount    = categoryNames[category];
                        maxCategory = category;
                    }
                }
                double probability = ((double)maxCount + 1) / ((double)totalCount + 2);
                if (probability < CategoryMajorityThreshold && results.Count < MaxResults) //this is not a valid category need more work
                {
                    return(null);
                }


                // now we have one segment ready
                ImageSegmentationResultSingleEntry_NoHoles aggregated = new ImageSegmentationResultSingleEntry_NoHoles();
                aggregated.polygon  = aggregatedPolygon;
                aggregated.Category = maxCategory;

                aggResult.boxesAndCategories.objects.Add(aggregated);
                for (int i = 0; i < PNG.Length; i++)
                {
                    if (PNG[i] == 0 && png[i] != 0)
                    {
                        PNG[i] = (byte)(idx + 1);
                    }
                }
            }

            // save and upload to azure
            string filename = URIUtilities.filenameFromURINoExtension(SatyamURL);
            string filepath = DirectoryConstants.defaultTempDirectory + filename + "_aggregated.PNG";

            ImageUtilities.savePNGRawData(filepath, ImageWidth, ImageHeight, PNG);

            SatyamJobStorageAccountAccess blob = new SatyamJobStorageAccountAccess();
            string container     = SatyamTaskGenerator.JobTemplateToSatyamContainerNameMap[TaskConstants.Segmentation_Image_MTurk];
            string directoryPath = guid + "_aggregated";

            blob.UploadALocalFile(filepath, container, directoryPath);

            aggResult.metaData.PNG_URL = TaskConstants.AzureBlobURL + container + "/" + directoryPath + "/" + filename + "_aggregated.PNG";
            return(aggResult);
        }
Exemplo n.º 30
0
        private bool getNewRandomJob()
        {
            double price   = 0;
            bool   success = Double.TryParse(Hidden_Price.Value, out price);

            if (!success)
            {
                price = 0;
            }
            SatyamTaskTableAccess taskTableDB = new SatyamTaskTableAccess();
            SatyamTaskTableEntry  entry       = null;

            if (Submit_Button.Enabled == true)
            {
                //entry = taskTableDB.getMinimumTriedNewEntryForWorkerIDByTemplateAndPrice(Hidden_AmazonWorkerID.Value,
                //    TaskConstants.TrackletLabeling_MTurk, price);
                entry = taskTableDB.getTopKNewEntryForWorkerIDByTemplateAndPrice(50, Hidden_AmazonWorkerID.Value,
                                                                                 TaskConstants.TrackletLabeling_MTurk, price);
            }
            else
            {
                entry = taskTableDB.getMinimumTriedEntryByTemplate(TaskConstants.TrackletLabeling_MTurk);
            }

            taskTableDB.close();


            if (entry == null)
            {
                return(false);
            }
            taskTableDB = new SatyamTaskTableAccess();
            taskTableDB.IncrementDoneScore(entry.ID);
            taskTableDB.close();

            SatyamTask task = JSonUtils.ConvertJSonToObject <SatyamTask>(entry.TaskParametersString);

            SatyamJobStorageAccountAccess satyamStorage = new SatyamJobStorageAccountAccess();

            string        videoDir  = URIUtilities.localDirectoryFullPathFromURI(task.SatyamURI);
            List <string> ImageURLs = satyamStorage.getURLListOfSpecificExtensionUnderSubDirectoryByURI(videoDir, new List <string>()
            {
                "jpg"
            });

            string annotationFilePath = task.SatyamURI;

            //string urls = "";
            //for (int i=0;i<ImageURLs.Count;i++)
            //{
            //    urls += ImageURLs[i];
            //    if (i == ImageURLs.Count - 1) break;
            //    urls += ',';
            //}
            Hidden_ImageURLList.Value = ObjectsToStrings.ListString(ImageURLs, ',');

            SatyamJob jobDefinitionEntry        = task.jobEntry;
            MultiObjectTrackingSubmittedJob job = JSonUtils.ConvertJSonToObject <MultiObjectTrackingSubmittedJob>(jobDefinitionEntry.JobParameters);

            Dictionary <string, List <string> > subcategories = job.Categories;
            List <string> categories = subcategories.Keys.ToList();

            //CategorySelection_RakdioButtonList.Items.Clear();
            for (int i = 0; i < categories.Count; i++)
            {
                ListItem l = new ListItem(categories[i]);
                //CategorySelection_RadioButtonList.Items.Add(l);
            }

            if (job.Description != "")
            {
                //DescriptionPanel.Visible = true;
                //DescriptionTextPanel.Controls.Add(new LiteralControl(job.Description));
            }
            //Hidden_BoundaryLines.Value = JSonUtils.ConvertObjectToJSon(job.BoundaryLines);


            Hidden_TaskEntryString.Value = JSonUtils.ConvertObjectToJSon <SatyamTaskTableEntry>(entry);
            Hidden_PageLoadTime.Value    = DateTime.Now.ToString();


            // pass parameters from old template
            Slug_Hidden.Value          = "null";
            Start_Hidden.Value         = "0";
            Stop_Hidden.Value          = (ImageURLs.Count - 1).ToString();
            Skip_Hidden.Value          = "0";
            PerObject_Hidden.Value     = "0.1";
            Completion_Hidden.Value    = "0.5";
            BlowRadius_Hidden.Value    = "0";
            JobId_Hidden.Value         = "1";
            LabelString_Hidden.Value   = ObjectsToStrings.ListString(categories.ToList(), ',');
            Attributes_Hidden.Value    = ObjectsToStrings.DictionaryStringListString(subcategories, ',', ':', '_');
            Training_Hidden.Value      = "0";
            fps_Hidden.Value           = job.FrameRate.ToString();
            Hidden_ChunkDuration.Value = job.ChunkDuration.ToString();

            var web = new WebClient();

            System.Drawing.Image x = System.Drawing.Image.FromStream(web.OpenRead(ImageURLs[0]));
            ImageWidth_Hidden.Value  = x.Width.ToString();
            ImageHeight_Hidden.Value = x.Height.ToString();

            // image boundary for now
            //string[] region = new string[] { "0-0-1242-0-1242-375-0-375-0-0" };
            string[] region = new string[] { "0-0-" + x.Width + "-0-" + x.Width + "-" + x.Height + "-0-" + x.Height + "-0-0" };
            RegionString_Hidden.Value = ObjectsToStrings.ListString(region, ',');

            // temp test
            List <VATIC_Tracklet> prevTracesTemp = new List <VATIC_Tracklet>();

            WebClient     client = new WebClient();
            Stream        stream = client.OpenRead(annotationFilePath);
            StreamReader  reader = new StreamReader(stream);
            List <string> trace  = new List <string>();

            while (reader.Peek() >= 0)
            {
                string content = reader.ReadLine();
                trace.Add(content);
            }


            Dictionary <string, VATIC_Tracklet> tracklets = VATIC_Tracklet.ReadTrackletsFromVIRAT(trace);

            foreach (string id in tracklets.Keys)
            {
                //string output = JSonUtils.ConvertObjectToJSon(tracklets[id]);
                prevTracesTemp.Add(tracklets[id]);
            }
            string output = JSonUtils.ConvertObjectToJSon(prevTracesTemp);

            PreviousTrackString_Hidden.Value = output;

            return(true);
        }