public static void SaveResultVideosLocally(List <SatyamResultsTableEntry> entries, string directoryName, int fps = 10) { directoryName = directoryName + "\\Raw"; if (!Directory.Exists(directoryName)) { Directory.CreateDirectory(directoryName); } SatyamJobStorageAccountAccess satyamStorage = new SatyamJobStorageAccountAccess(); for (int i = 0; i < entries.Count; i++) { SatyamResultsTableEntry entry = entries[i]; SatyamResult satyamResult = JSonUtils.ConvertJSonToObject <SatyamResult>(entry.ResultString); SatyamTask task = JSonUtils.ConvertJSonToObject <SatyamTask>(satyamResult.TaskParametersString); MultiObjectTrackingSubmittedJob job = JSonUtils.ConvertJSonToObject <MultiObjectTrackingSubmittedJob>(task.jobEntry.JobParameters); string blobDir = URIUtilities.localDirectoryFullPathFromURI(task.SatyamURI); //VATIC_DVA_CrowdsourcedResult taskr = new VATIC_DVA_CrowdsourcedResult(satyamResult.TaskResult, task.SatyamURI, start, end, task.SatyamJobSubmissionsTableEntryID.ToString()); VATIC_DVA_CrowdsourcedResult taskr = MultiObjectTrackingAggregator.createVATIC_DVA_CrowdsourcedResultUsingSatyamBlobImageCount(satyamResult.TaskResult, blobDir, entry.ID.ToString(), job.FrameRate); MultiObjectTrackingResult res = taskr.getCompressedTracksInTimeSegment(); List <string> ImageURLs = satyamStorage.getURLListOfSpecificExtensionUnderSubDirectoryByURI(blobDir, new List <string>() { "jpg", "png" }); string videoName = URIUtilities.localDirectoryNameFromURI(blobDir) + "_" + URIUtilities.filenameFromURINoExtension(task.SatyamURI) + "_" + entry.ID; MultiObjectTrackingAnalyzer.generateVideoForEvaluation(ImageURLs, res, directoryName, videoName, job.FrameRate); } }
public static void GroupEntriesByVideoNameAndStitchAndSaveAggregatedResultVideosLocally(List <SatyamAggregatedResultsTableEntry> entries, string directoryName) { if (entries.Count == 0) { return; } directoryName = directoryName + "\\AggregatedStitched"; if (!Directory.Exists(directoryName)) { Directory.CreateDirectory(directoryName); } SortedDictionary <string, List <SatyamAggregatedResultsTableEntry> > entriesPerVideo = MultiObjectTrackingAnalyzer.GroupChunksByVideoName(entries); foreach (string videoname in entriesPerVideo.Keys) { List <string> ImageURLs; int fps; MultiObjectTrackingResult stitched = stitchAllChunksAllObjectsOfOneVideo(entriesPerVideo[videoname], out ImageURLs, out fps); if (stitched != null) { // incase video is too large MultiObjectTrackingAnalyzer.generateVideoFramesForEvaluation(ImageURLs, stitched, directoryName, videoname, fps); //MultiObjectTrackingAnalyzer.generateVideoForEvaluation(ImageURLs, stitched, directoryName, videoname, fps); } } }
public static void generateVideoForEvaluation(List <string> ImageURLs, MultiObjectTrackingResult ctts, String directory, String videoName, int fps) { if (Directory.Exists(directory + "\\" + videoName)) { return; } if (File.Exists(directory + "\\" + videoName + ".mp4")) { return; } List <Image> imageList = new List <Image>(); var wc = new WebClient(); foreach (string uri in ImageURLs) { Image x = Image.FromStream(wc.OpenRead(uri)); imageList.Add(x); } List <DateTime> frameTimes = new List <DateTime>(); double frameTimeSpanInMiliseconds = (double)1000 / (double)fps; //double frameTimeSpanInMiliseconds = (double)(ChunkDuration) / (double)(ImageURLs.Count) * 1000; DateTime start = ctts.VideoSegmentStartTime; for (int i = 0; i < ImageURLs.Count; i++) { DateTime t; t = start.AddMilliseconds(frameTimeSpanInMiliseconds * i); frameTimes.Add(t); } generateVideoForEvaluation(imageList, frameTimes, ctts, videoName, directory); }
public static void SaveKITTIResultVideosLocally(List <SatyamResultsTableEntry> entries, string directoryName, int fps = 10) { directoryName = directoryName + "\\Raw"; if (!Directory.Exists(directoryName)) { Directory.CreateDirectory(directoryName); } SatyamJobStorageAccountAccess satyamStorage = new SatyamJobStorageAccountAccess(); for (int i = 0; i < entries.Count; i++) { SatyamResultsTableEntry entry = entries[i]; SatyamResult satyamResult = JSonUtils.ConvertJSonToObject <SatyamResult>(entry.ResultString); SatyamTask task = JSonUtils.ConvertJSonToObject <SatyamTask>(satyamResult.TaskParametersString); MultiObjectTrackingSubmittedJob job = JSonUtils.ConvertJSonToObject <MultiObjectTrackingSubmittedJob>(task.jobEntry.JobParameters); //VATIC_DVA_CrowdsourcedResult taskr = VATIC_DVA_CrowdsourcedResult.createVATIC_DVA_CrowdsourcedResultUsingSatyamBlobImageCount(satyamResult.TaskResult, task.SatyamURI, task.SatyamJobSubmissionsTableEntryID.ToString(), job.FrameRate); //MultiObjectTrackingResult res = taskr.getCompressedTracksInTimeSegment(); string[] names = task.SatyamURI.Split('/'); string videoName = names[names.Length - 2] + "_" + entry.ID; string[] fields = videoName.Split('_'); string videoSequence = fields[fields.Length - 5]; int startingFrame = Convert.ToInt32(fields[fields.Length - 2]); int maxChunkEndFrame = startingFrame + job.ChunkDuration * job.FrameRate; int noFrameOverlap = (int)(job.ChunkOverlap * job.FrameRate); if (startingFrame != 0) { startingFrame -= noFrameOverlap; } if (entry.JobGUID == "1e43a983-548d-4a2e-8161-5537eb985902") { videoSequence = getCorrectSequenceNo(videoSequence); } string videoFrameDir = DirectoryConstants.KITTITrackingImages + videoSequence; List <string> files = Directory.GetFiles(videoFrameDir).ToList(); List <string> ImageURLs = new List <string>(); for (int j = startingFrame; j < files.Count && j < maxChunkEndFrame; j++) { ImageURLs.Add(files[j]); } if (ImageURLs.Count == 0) { continue; } VATIC_DVA_CrowdsourcedResult taskr = new VATIC_DVA_CrowdsourcedResult(satyamResult.TaskResult, videoName, entry.ID.ToString(), ImageURLs.Count, job.FrameRate); MultiObjectTrackingResult res = taskr.getCompressedTracksInTimeSegment(); generateVideoForEvaluation(ImageURLs, res, directoryName, videoName, job.FrameRate); } }
public static void SaveResultVideosLocally(List <SatyamResultsTableEntry> entries, string directoryName, int fps = 10) { directoryName = directoryName + "\\Raw"; if (!Directory.Exists(directoryName)) { Directory.CreateDirectory(directoryName); } SatyamJobStorageAccountAccess satyamStorage = new SatyamJobStorageAccountAccess(); for (int i = 0; i < entries.Count; i++) { SatyamResultsTableEntry entry = entries[i]; SatyamResult satyamResult = JSonUtils.ConvertJSonToObject <SatyamResult>(entry.ResultString); SatyamTask task = JSonUtils.ConvertJSonToObject <SatyamTask>(satyamResult.TaskParametersString); MultiObjectTrackingSubmittedJob job = JSonUtils.ConvertJSonToObject <MultiObjectTrackingSubmittedJob>(task.jobEntry.JobParameters); //VATIC_DVA_CrowdsourcedResult taskr = new VATIC_DVA_CrowdsourcedResult(satyamResult.TaskResult, task.SatyamURI, start, end, task.SatyamJobSubmissionsTableEntryID.ToString()); VATIC_DVA_CrowdsourcedResult taskr = MultiObjectTrackingAggregator.createVATIC_DVA_CrowdsourcedResultUsingSatyamBlobImageCount(satyamResult.TaskResult, task.SatyamURI, entry.ID.ToString(), job.FrameRate); MultiObjectTrackingResult res = taskr.getCompressedTracksInTimeSegment(); List <string> ImageURLs = satyamStorage.getURLListOfSubDirectoryByURL(task.SatyamURI); string[] names = task.SatyamURI.Split('/'); string videoName = names[names.Length - 2] + "_" + entry.ID; generateVideoForEvaluation(ImageURLs, res, directoryName, videoName, job.FrameRate); } }
public static double[,] computeTrackSimilarityMatrix(MultiObjectTrackingResult cts1, MultiObjectTrackingResult cts2, ICompressedTrackSimilarityMetric metric) { double[,] ret = new double[cts1.tracks.Count, cts2.tracks.Count]; for (int i = 0; i < cts1.tracks.Count; i++) { CompressedTrack t1 = cts1.tracks[i]; for (int j = 0; j < cts2.tracks.Count; j++) { CompressedTrack t2 = cts2.tracks[j]; ret[i, j] = metric.getMetric(t1, t2); } } return(ret); }
public static void SaveAggregatedResultVideosLocally(List <SatyamAggregatedResultsTableEntry> entries, string directoryName) { directoryName = directoryName + "\\Aggregated"; if (!Directory.Exists(directoryName)) { Directory.CreateDirectory(directoryName); } SatyamJobStorageAccountAccess satyamStorage = new SatyamJobStorageAccountAccess(); for (int i = 0; i < entries.Count; i++) { SatyamAggregatedResultsTableEntry entry = entries[i]; SatyamAggregatedResult satyamAggResult = JSonUtils.ConvertJSonToObject <SatyamAggregatedResult>(entry.ResultString); SatyamTask aggTask = JSonUtils.ConvertJSonToObject <SatyamTask>(satyamAggResult.TaskParameters); MultiObjectTrackingSubmittedJob job = JSonUtils.ConvertJSonToObject <MultiObjectTrackingSubmittedJob>(aggTask.jobEntry.JobParameters); TrackletLabelingAggregatedResult aggresult = JSonUtils.ConvertJSonToObject <TrackletLabelingAggregatedResult>(satyamAggResult.AggregatedResultString); WebClient wb = new WebClient(); Stream aggTrackStream = wb.OpenRead(aggresult.AggregatedTrackletsString_URL); StreamReader reader = new StreamReader(aggTrackStream); String aggTrackString = reader.ReadToEnd(); MultiObjectTrackingResult aggTracks = JSonUtils.ConvertJSonToObject <MultiObjectTrackingResult>(aggTrackString); string blobDir = URIUtilities.localDirectoryFullPathFromURI(aggTask.SatyamURI); List <string> ImageURLs = satyamStorage.getURLListOfSpecificExtensionUnderSubDirectoryByURI(blobDir, new List <string>() { "jpg", "png" }); string videoName = URIUtilities.localDirectoryNameFromURI(blobDir) + "_" + URIUtilities.filenameFromURINoExtension(aggTask.SatyamURI) + "_" + entry.ID; MultiObjectTrackingAnalyzer.generateVideoForEvaluation(ImageURLs, aggTracks, directoryName, videoName, job.FrameRate); } }
public static void AggregateWithParameter(string guid, int MinResults = TaskConstants.TRACKLET_LABELING_MTURK_MIN_RESULTS_TO_AGGREGATE, int MaxResults = TaskConstants.TRACKLET_LABELING_MTURK_MAX_RESULTS_TO_AGGREGATE, double boxToleranceThreshold = TaskConstants.TRACKLET_LABELING_BOX_DEVIATION_THRESHOLD, double ObjectCoverageApprovalThresholdPerVideo = TaskConstants.TRACKLET_LABELING_APPROVALRATIO_PER_VIDEO, double BoxCoverageApprovalThresholdPerTrack = TaskConstants.TRACKLET_LABELING_APPROVALRATIO_PER_TRACK, int consensusNumber = TaskConstants.TRACKLET_LABELING_MIN_RESULTS_FOR_CONSENSUS, double minTubeletIoUSimilarityThreshold = TaskConstants.TRACKLET_LABELING_MIN_TUBELET_SIMILARITY_THRESHOLD, double attributeMajority = TaskConstants.TRACKLET_LABELING_MTURK_ATTRIBUTE_MAJORITY_THRESHOLD, bool allFalseAttributeInvalid = false ) { string configString = "Min_" + MinResults + "_Max_" + MaxResults + "_IoU_" + minTubeletIoUSimilarityThreshold + "_Ratio_" + ObjectCoverageApprovalThresholdPerVideo; Console.WriteLine("Aggregating for " + guid + " with param set " + configString); SatyamResultsTableAccess resultsDB = new SatyamResultsTableAccess(); List <SatyamResultsTableEntry> entries = resultsDB.getEntriesByGUID(guid); resultsDB.close(); SortedDictionary <DateTime, List <SatyamResultsTableEntry> > entriesBySubmitTime = SatyamResultValidationToolKit.SortResultsBySubmitTime_OneResultPerTurkerPerTask(entries); Dictionary <int, List <MultiObjectTrackingResult> > ResultsPerTask = new Dictionary <int, List <MultiObjectTrackingResult> >(); List <int> aggregatedTasks = new List <int>(); int noTotalConverged = 0; //int noCorrect = 0; int noTerminatedTasks = 0; List <SatyamAggregatedResultsTableEntry> aggEntries = new List <SatyamAggregatedResultsTableEntry>(); Dictionary <int, int> noResultsNeededForAggregation = SatyamResultsAnalysis.getNoResultsNeededForAggregationFromLog(configString, guid); Dictionary <int, int> noResultsNeededForAggregation_new = new Dictionary <int, int>(); // play back by time Dictionary <int, List <string> > WorkersPerTask = new Dictionary <int, List <string> >(); foreach (DateTime t in entriesBySubmitTime.Keys) { //Console.WriteLine("Processing Results of time: {0}", t); List <SatyamResultsTableEntry> ResultEntries = entriesBySubmitTime[t]; foreach (SatyamResultsTableEntry entry in ResultEntries) { SatyamResult satyamResult = JSonUtils.ConvertJSonToObject <SatyamResult>(entry.ResultString); SatyamTask task = JSonUtils.ConvertJSonToObject <SatyamTask>(satyamResult.TaskParametersString); MultiObjectTrackingSubmittedJob job = JSonUtils.ConvertJSonToObject <MultiObjectTrackingSubmittedJob>(task.jobEntry.JobParameters); string fileName = URIUtilities.filenameFromURINoExtension(task.SatyamURI); int taskEntryID = entry.SatyamTaskTableEntryID; if (aggregatedTasks.Contains(taskEntryID)) { continue; } if (!ResultsPerTask.ContainsKey(taskEntryID)) { ResultsPerTask.Add(taskEntryID, new List <MultiObjectTrackingResult>()); WorkersPerTask.Add(taskEntryID, new List <string>()); } // remove duplicate workers result string workerID = satyamResult.amazonInfo.WorkerID; if (WorkersPerTask[taskEntryID].Contains(workerID)) { continue; } //enclose only non-duplicate results, one per each worker. WorkersPerTask[taskEntryID].Add(workerID); string videoName = URIUtilities.localDirectoryNameFromURI(task.SatyamURI); string[] fields = videoName.Split('_'); int startingFrame = Convert.ToInt32(fields[fields.Length - 1]); int maxChunkEndFrame = startingFrame + job.ChunkDuration * job.FrameRate; int noFrameOverlap = (int)(job.ChunkOverlap * job.FrameRate); if (startingFrame != 0) { startingFrame -= noFrameOverlap; } string blobDir = URIUtilities.localDirectoryFullPathFromURI(task.SatyamURI); VATIC_DVA_CrowdsourcedResult taskr = TrackletLabelingAggregator.createVATIC_DVA_CrowdsourcedResultUsingSatyamBlobImageCount(satyamResult.TaskResult, blobDir, entry.SatyamTaskTableEntryID.ToString(), job.FrameRate); //MultiObjectTrackingResult result_tmp = vatic_tmp.getCompressedTracksInTimeSegment(); //VATIC_DVA_CrowdsourcedResult taskr = new VATIC_DVA_CrowdsourcedResult(satyamResult.TaskResult, videoName, entry.ID.ToString(), ImageURLs.Count, job.FrameRate); MultiObjectTrackingResult res = taskr.getCompressedTracksInTimeSegment(); if (allFalseAttributeInvalid) { if (TrackletLabelingAggregator.AllAttributeAllFalse(res)) { continue; } } ResultsPerTask[taskEntryID].Add(res); // check log if enough results are collected if (noResultsNeededForAggregation != null && noResultsNeededForAggregation.ContainsKey(taskEntryID) && ResultsPerTask[taskEntryID].Count < noResultsNeededForAggregation[taskEntryID]) { continue; } // hack for masters int tempMin = MinResults; if (TaskConstants.masterGUIDs.Contains(entry.JobGUID)) { tempMin = 1; } TrackletLabelingAggregatedResult aggResult = TrackletLabelingAggregator.getAggregatedResult(ResultsPerTask[taskEntryID], guid, task.SatyamURI, job.FrameRate, job.BoundaryLines, MinResults, MaxResults, boxToleranceThreshold, ObjectCoverageApprovalThresholdPerVideo, BoxCoverageApprovalThresholdPerTrack, consensusNumber, minTubeletIoUSimilarityThreshold, attributeMajority); if (aggResult == null) { continue; } //////////////// aggregation happen // record logs if (noResultsNeededForAggregation == null || !noResultsNeededForAggregation.ContainsKey(taskEntryID)) { noResultsNeededForAggregation_new.Add(taskEntryID, ResultsPerTask[taskEntryID].Count); } aggregatedTasks.Add(taskEntryID); noTotalConverged++; if (ResultsPerTask[taskEntryID].Count >= MaxResults) { noTerminatedTasks++; } SatyamAggregatedResult SatyamAggResult = new SatyamAggregatedResult(); SatyamAggResult.SatyamTaskTableEntryID = taskEntryID; SatyamAggResult.AggregatedResultString = JSonUtils.ConvertObjectToJSon <TrackletLabelingAggregatedResult>(aggResult); SatyamAggResult.TaskParameters = JSonUtils.ConvertJSonToObject <SatyamResult>(entry.ResultString).TaskParametersString; SatyamAggregatedResultsTableEntry aggEntry = new SatyamAggregatedResultsTableEntry(); aggEntry.SatyamTaskTableEntryID = taskEntryID; aggEntry.JobGUID = entry.JobGUID; aggEntry.ResultString = JSonUtils.ConvertObjectToJSon <SatyamAggregatedResult>(SatyamAggResult); aggEntries.Add(aggEntry); } } Console.WriteLine("Total_Aggregated_Tasks: {0}", noTotalConverged); Console.WriteLine("Total_Terminated_Tasks: {0}", noTerminatedTasks); SatyamResultsAnalysis.RecordAggregationLog(noResultsNeededForAggregation_new, configString, guid); TrackletLabelingAnalyzer.GroupEntriesByVideoNameAndStitchAndSaveAggregatedResultVideosLocally(aggEntries, DirectoryConstants.defaultTempDirectory + guid); }
public static MultiObjectTrackingResult stitchAllChunksAllObjectsOfOneVideo(List <SatyamAggregatedResultsTableEntry> entries, out List <string> ImageURLs, out int fps) { ImageURLs = new List <string>(); fps = 0; if (entries.Count == 0) { return(null); } SatyamJobStorageAccountAccess satyamStorage = new SatyamJobStorageAccountAccess(); MultiObjectTrackingResult stitched = new MultiObjectTrackingResult(); int totalFrameCounts = 0; // ensure the order is correct SortedDictionary <int, List <SatyamAggregatedResultsTableEntry> > sortedEntries = new SortedDictionary <int, List <SatyamAggregatedResultsTableEntry> >(); List <int> idx = new List <int>(); for (int i = 0; i < entries.Count; i++) { SatyamAggregatedResultsTableEntry entry = entries[i]; SatyamAggregatedResult satyamAggResult = JSonUtils.ConvertJSonToObject <SatyamAggregatedResult>(entry.ResultString); SatyamTask aggTask = JSonUtils.ConvertJSonToObject <SatyamTask>(satyamAggResult.TaskParameters); string video = URIUtilities.localDirectoryNameFromURI(aggTask.SatyamURI); string[] fields = video.Split('_'); int startingFrame = Convert.ToInt32(fields[fields.Length - 1]); if (!sortedEntries.ContainsKey(startingFrame)) { sortedEntries.Add(startingFrame, new List <SatyamAggregatedResultsTableEntry>()); idx.Add(startingFrame); } sortedEntries[startingFrame].Add(entries[i]); } idx.Sort(); List <string> AggObjIds = new List <string>(); for (int i = 0; i < idx.Count; i++) { int noFramesOverlap = 0; string blobDir = ""; // grouping all objects that belong to the same chunk MultiObjectTrackingResult aggTracksOfAllObjectsPerChunk = new MultiObjectTrackingResult(); List <string> objIds = new List <string>(); for (int j = 0; j < sortedEntries[idx[i]].Count; j++) { SatyamAggregatedResultsTableEntry entry = sortedEntries[idx[i]][j]; SatyamAggregatedResult satyamAggResult = JSonUtils.ConvertJSonToObject <SatyamAggregatedResult>(entry.ResultString); SatyamTask aggTask = JSonUtils.ConvertJSonToObject <SatyamTask>(satyamAggResult.TaskParameters); MultiObjectTrackingSubmittedJob job = JSonUtils.ConvertJSonToObject <MultiObjectTrackingSubmittedJob>(aggTask.jobEntry.JobParameters); if (job.ChunkOverlap != 0.0) { noFramesOverlap = (int)(job.ChunkOverlap * job.FrameRate); } fps = job.FrameRate; blobDir = URIUtilities.localDirectoryFullPathFromURI(aggTask.SatyamURI); string objId = URIUtilities.filenameFromURINoExtension(aggTask.SatyamURI); objIds.Add(objId); TrackletLabelingAggregatedResult aggresult = JSonUtils.ConvertJSonToObject <TrackletLabelingAggregatedResult>(satyamAggResult.AggregatedResultString); WebClient wb = new WebClient(); Stream aggTrackStream = wb.OpenRead(aggresult.AggregatedTrackletsString_URL); StreamReader reader = new StreamReader(aggTrackStream); String aggTrackString = reader.ReadToEnd(); MultiObjectTrackingResult aggTracks = JSonUtils.ConvertJSonToObject <MultiObjectTrackingResult>(aggTrackString); if (aggTracksOfAllObjectsPerChunk.tracks.Count == 0) { aggTracksOfAllObjectsPerChunk = aggTracks; } else { for (int k = 0; k < aggTracks.tracks.Count; k++) { aggTracksOfAllObjectsPerChunk.tracks.Add(aggTracks.tracks[k]); } } } List <string> TraceURLs = satyamStorage.getURLListOfSpecificExtensionUnderSubDirectoryByURI(blobDir, new List <string>() { "jpg", "png" }); if (i == 0) { ImageURLs.AddRange(TraceURLs); stitched = aggTracksOfAllObjectsPerChunk; totalFrameCounts += TraceURLs.Count; AggObjIds = objIds; } else { int noNewFrames = 0; for (int j = noFramesOverlap; j < TraceURLs.Count; j++) { ImageURLs.Add(TraceURLs[j]); noNewFrames++; } //stitched = MultiObjectTrackingAnalyzer.stitchTwoTracesByTubeletsOfOverlappingVideoChunk(stitched, // aggTracksOfAllObjectsPerChunk, totalFrameCounts, totalFrameCounts + noNewFrames, noFramesOverlap, fps); // postpone the agg trace double frameTimeInMiliSeconds = (double)1000 / (double)fps; double timeToPostponeInMilliSeconds = (double)(totalFrameCounts - noFramesOverlap) * frameTimeInMiliSeconds; //TimeSpan timeSpanToPostponeInSeconds = new TimeSpan(0,0,0,0, (int)timeToPostponeInMilliSeconds); TimeSpan timeSpanToPostponeInSeconds = DateTimeUtilities.getTimeSpanFromTotalMilliSeconds((int)timeToPostponeInMilliSeconds); aggTracksOfAllObjectsPerChunk.postpone(timeSpanToPostponeInSeconds); // overlap must be 0 for (int k = 0; k < aggTracksOfAllObjectsPerChunk.tracks.Count; k++) { if (!AggObjIds.Contains(objIds[k])) { stitched.tracks.Add(aggTracksOfAllObjectsPerChunk.tracks[k]); AggObjIds.Add(objIds[k]); } else { // stitch the track for the same id int tckIdx = AggObjIds.IndexOf(objIds[k]); stitched.tracks[tckIdx] = CompressedTrack.stitchTwoAdjacentTrack(stitched.tracks[tckIdx], aggTracksOfAllObjectsPerChunk.tracks[k]); } } totalFrameCounts += noNewFrames; } //debug //generateVideoForEvaluation(ImageURLs, stitched, directoryName + "_" + i, videoName, fps); } return(stitched); }
public static void generateVideoForEvaluation(List <Image> Images, List <DateTime> dateTimeList, MultiObjectTrackingResult ctts, String videoName, String directory) { List <Image> imagesWithTracks = new List <Image>(); for (int i = 0; i < Images.Count; i++) { List <BoundingBox> locations = new List <BoundingBox>(); List <string> labels = new List <string>(); Dictionary <string, List <bool> > attributes = new Dictionary <string, List <bool> >(); attributes.Add("occlusion", new List <bool>()); if (ctts.tracks.Count != 0) { foreach (string key in ctts.tracks[0].booleanAttributeTracks.Keys) { if (!attributes.ContainsKey(key)) { attributes.Add(key, new List <bool>()); } } } List <int> idx = new List <int>(); for (int j = 0; j < ctts.tracks.Count; j++) { CompressedTrack ct = ctts.tracks[j]; SpaceTime st = ct.getSpaceTimeAt(dateTimeList[i]); BooleanAttribute outofview_attr = ct.getAttributeAt("outofview", dateTimeList[i]); if (st != null && outofview_attr != null && !outofview_attr.value) { BoundingBox l = st.region; locations.Add(l); labels.Add(ctts.tracks[j].label); foreach (string key in attributes.Keys) { attributes[key].Add(ct.getAttributeAt(key, dateTimeList[i]).value); } idx.Add(j); } } Image new_image = generateTrackImage(Images[i], labels, locations, attributes, idx); imagesWithTracks.Add(new_image); } Console.WriteLine("Saving " + directory + "\\" + videoName); FFMpegWrappers.generateVideoFromFrames(imagesWithTracks, videoName, directory); //Directory.CreateDirectory(directory + "\\" + videoName); //Console.WriteLine("Saving " + directory + "\\" + videoName); //for (int i = 0; i < imagesWithTracks.Count; i++) //{ // imagesWithTracks[i].Save(directory + "\\" + videoName + "\\img" + i.ToString("000") + ".jpg"); //} //Console.WriteLine("done"); }
// if the stitched video is too long public static void generateVideoFramesForEvaluation(List <string> ImageURLs, MultiObjectTrackingResult ctts, String directory, String videoName, int fps) { if (Directory.Exists(directory + "\\" + videoName)) { return; } Directory.CreateDirectory(directory + "\\" + videoName); Console.WriteLine("Saving " + directory + "\\" + videoName); //List<Image> imageList = new List<Image>(); var wc = new WebClient(); //List<DateTime> frameTimes = new List<DateTime>(); double frameTimeSpanInMiliseconds = (double)1000 / (double)fps; //double frameTimeSpanInMiliseconds = (double)(ChunkDuration) / (double)(ImageURLs.Count) * 1000; DateTime start = ctts.VideoSegmentStartTime; for (int i = 0; i < ImageURLs.Count; i++) { DateTime t; t = start.AddMilliseconds(frameTimeSpanInMiliseconds * i); //frameTimes.Add(t); Image x = Image.FromStream(wc.OpenRead(ImageURLs[i])); //imageList.Add(x); List <BoundingBox> locations = new List <BoundingBox>(); List <string> labels = new List <string>(); Dictionary <string, List <bool> > attributes = new Dictionary <string, List <bool> >(); attributes.Add("occlusion", new List <bool>()); List <int> idx = new List <int>(); if (ctts.tracks.Count != 0) { foreach (string key in ctts.tracks[0].booleanAttributeTracks.Keys) { if (!attributes.ContainsKey(key)) { attributes.Add(key, new List <bool>()); } } } for (int j = 0; j < ctts.tracks.Count; j++) { CompressedTrack ct = ctts.tracks[j]; SpaceTime st = ct.getSpaceTimeAt(t); BooleanAttribute outofview_attr = ct.getAttributeAt("outofview", t); if (st != null && outofview_attr != null && !outofview_attr.value) { BoundingBox l = st.region; locations.Add(l); labels.Add(ctts.tracks[j].label); //attributes["occlusion"].Add(ct.getAttributeAt("occlusion", t).value); foreach (string key in attributes.Keys) { attributes[key].Add(ct.getAttributeAt(key, t).value); } idx.Add(j); } } Image new_image = generateTrackImage(x, labels, locations, attributes, idx); new_image.Save(directory + "\\" + videoName + "\\img" + i.ToString("000") + ".jpg"); } FFMpegWrappers.generateVideoFromFolderofFrames(videoName, directory + "\\" + videoName + "\\"); Console.WriteLine("done"); }
public static MultiObjectTrackingResult stitchTwoTracesByTubeletsOfOverlappingVideoChunk(MultiObjectTrackingResult currentTrace, MultiObjectTrackingResult nextTrace, int totalFrameCountsBeforeStitching, int totalFrameCountsAfterStitching, int noFramesOverlap, int fps = 10) { //if (noFramesOverlap == 0) return null; double frameTimeInMiliSeconds = (double)1000 / (double)fps; List <MultiObjectTrackingResult> compressedOverlappingTracks = new List <MultiObjectTrackingResult>(); DateTime overlapStart = currentTrace.VideoSegmentStartTime.AddMilliseconds(frameTimeInMiliSeconds * (totalFrameCountsBeforeStitching - noFramesOverlap)); DateTime overlapEnd = currentTrace.VideoSegmentStartTime.AddMilliseconds(frameTimeInMiliSeconds * (totalFrameCountsBeforeStitching - 1)); //DateTime nextTrace_overlapStart = nextTrace.VideoSegmentStartTime; //DateTime nextTrace_overlapEnd = nextTrace.VideoSegmentStartTime.AddMilliseconds(frameTimeInMiliSeconds * (noFramesOverlap-1)); //MultiObjectTrackingResult nextTrace_Overlap = nextTrace.getSubTimeSegment(nextTrace_overlapStart, nextTrace_overlapEnd, fps); //compressedOverlappingTracks.Add(nextTrace_Overlap); List <List <string> > CrossChunkEntitiesAnnotationStrings = new List <List <string> >(); List <List <string> > SingleChunkEntitiesAnnotationStrings = new List <List <string> >(); for (int i = 0; i < 2; i++) { //boundingboxes.Add(new List<BoundingBox>()); CrossChunkEntitiesAnnotationStrings.Add(new List <string>()); SingleChunkEntitiesAnnotationStrings.Add(new List <string>()); } double timeToPostponeInMilliSeconds = (double)(totalFrameCountsBeforeStitching - noFramesOverlap) * frameTimeInMiliSeconds; //TimeSpan timeSpanToPostponeInSeconds = new TimeSpan(0,0,0,0, (int)timeToPostponeInMilliSeconds); TimeSpan timeSpanToPostponeInSeconds = DateTimeUtilities.getTimeSpanFromTotalMilliSeconds((int)timeToPostponeInMilliSeconds); //// get the end of the first trace and the begining + overlap of the second trace //DateTime CurrentTraceSampleFrameTime = currentTrace.VideoSegmentStartTime.AddMilliseconds(frameTimeInMiliSeconds * (totalFrameCountsBeforeStitching - 1)); //DateTime NextTraceSampleFrameTime = nextTrace.VideoSegmentStartTime.AddMilliseconds(frameTimeInMiliSeconds * (Math.Max(noFramesOverlap - 1, 0))); // current trace List <string> currentTrace_filteredAnnotationStrings = new List <string>(); foreach (CompressedTrack entity in currentTrace.tracks) { // get a snapshot frame at start and end of current trace VATIC_DVA_Frame currentTraceOverlapStartFrame = entity.getFrameAt(overlapStart); string annotationString = entity.unCompressToTrackAnnotationString(); string[] fields = annotationString.Split(':'); List <string> TimeFilteredSegmentStringList = new List <string>(); TimeFilteredSegmentStringList.Add(fields[0]); // adds the label back if (currentTraceOverlapStartFrame != null) { TimeFilteredSegmentStringList.Add(currentTraceOverlapStartFrame.ToAnnotationString()); // adds a starting frame } // remove the label of the second string int count = 0; for (int j = 1; j < fields.Length; j++) { // remove the overlapping part. DateTime time = DateTimeUtilities.getDateTimeFromString(fields[j].Split(',')[0]); if (time <= overlapStart || time > overlapEnd) { continue; } TimeFilteredSegmentStringList.Add(fields[j]); count++; } if (count == 0) { // not a cross chunk one SingleChunkEntitiesAnnotationStrings[0].Add(annotationString); continue; } else { //a cross chunk one CrossChunkEntitiesAnnotationStrings[0].Add(annotationString); // construct for association string filteredAnnotationString = ObjectsToStrings.ListString(TimeFilteredSegmentStringList, ':'); currentTrace_filteredAnnotationStrings.Add(filteredAnnotationString); } } string currentOverlap_totalFilteredAnnotationString = ObjectsToStrings.ListString(currentTrace_filteredAnnotationStrings, '|'); MultiObjectTrackingResult currentTrace_Overlap = MultiObjectTrackingResult.ConvertAnnotationStringToMultiObjectTrackingResult(overlapStart, currentOverlap_totalFilteredAnnotationString, "current", "0", noFramesOverlap, fps); compressedOverlappingTracks.Add(currentTrace_Overlap); // next trace /// postpone the timestamp of the second trace nextTrace.postpone(timeSpanToPostponeInSeconds); List <string> nextTrace_filteredAnnotationStrings = new List <string>(); foreach (CompressedTrack entity in nextTrace.tracks) { VATIC_DVA_Frame nextTraceOverlapStartFrame = entity.getFrameAt(overlapStart); VATIC_DVA_Frame nextTraceOverlapEndFrame = entity.getFrameAt(overlapEnd); string annotationString = entity.unCompressToTrackAnnotationString(); string[] fields = annotationString.Split(':'); List <string> TimeFilteredSegmentStringList = new List <string>(); TimeFilteredSegmentStringList.Add(fields[0]); // adds the label back if (nextTraceOverlapStartFrame != null) { TimeFilteredSegmentStringList.Add(nextTraceOverlapStartFrame.ToAnnotationString()); // adds a starting frame } // remove the label of the second string int count = 0; for (int j = 1; j < fields.Length; j++) { // remove the overlapping part. DateTime time = DateTimeUtilities.getDateTimeFromString(fields[j].Split(',')[0]); if (time <= overlapStart || time >= overlapEnd) { continue; } TimeFilteredSegmentStringList.Add(fields[j]); count++; } if (count == 0) { // not a cross chunk one SingleChunkEntitiesAnnotationStrings[1].Add(annotationString); continue; } else { //a cross chunk one CrossChunkEntitiesAnnotationStrings[1].Add(annotationString); // construct for association if (nextTraceOverlapEndFrame != null) { TimeFilteredSegmentStringList.Add(nextTraceOverlapEndFrame.ToAnnotationString()); // adds a ending frame } string filteredAnnotationString = ObjectsToStrings.ListString(TimeFilteredSegmentStringList, ':'); nextTrace_filteredAnnotationStrings.Add(filteredAnnotationString); } } string nextOverlap_totalFilteredAnnotationString = ObjectsToStrings.ListString(nextTrace_filteredAnnotationStrings, '|'); MultiObjectTrackingResult nextTrace_Overlap = MultiObjectTrackingResult.ConvertAnnotationStringToMultiObjectTrackingResult(overlapStart, nextOverlap_totalFilteredAnnotationString, "next", "1", noFramesOverlap, fps); compressedOverlappingTracks.Add(nextTrace_Overlap); List <MultipartiteWeightedMatch> association = TrackletsAssociation.AssociateTracklets(compressedOverlappingTracks); DateTime NewFrameStartTime = currentTrace.VideoSegmentStartTime + DateTimeUtilities.getTimeSpanFromTotalMilliSeconds((int)(totalFrameCountsBeforeStitching * frameTimeInMiliSeconds)); string totalStitchedAnnotationString = stitchAnnotationStringByAssociation(CrossChunkEntitiesAnnotationStrings, SingleChunkEntitiesAnnotationStrings, association, NewFrameStartTime); MultiObjectTrackingResult ret = MultiObjectTrackingResult.ConvertAnnotationStringToMultiObjectTrackingResult(currentTrace.VideoSegmentStartTime, totalStitchedAnnotationString, currentTrace.cameraId, currentTrace.UID, totalFrameCountsAfterStitching, fps); return(ret); }
public static MultiObjectTrackingResult stitchTwoTracesByOneFrameBoundingBoxes(MultiObjectTrackingResult currentTrace, MultiObjectTrackingResult nextTrace, int totalFrameCountsBeforeStitching, int totalFrameCountsAfterStitching, int noFramesOverlap = 0, int fps = 10) { List <List <BoundingBox> > boundingboxes = new List <List <BoundingBox> >(); List <List <string> > CrossChunkEntitiesAnnotationStrings = new List <List <string> >(); List <List <string> > SingleChunkEntitiesAnnotationStrings = new List <List <string> >(); for (int i = 0; i < 2; i++) { boundingboxes.Add(new List <BoundingBox>()); CrossChunkEntitiesAnnotationStrings.Add(new List <string>()); SingleChunkEntitiesAnnotationStrings.Add(new List <string>()); } double frameTimeInMiliSeconds = (double)1000 / (double)fps; double timeToPostponeInMilliSeconds = (double)(totalFrameCountsBeforeStitching - noFramesOverlap) * frameTimeInMiliSeconds; //TimeSpan timeSpanToPostponeInSeconds = new TimeSpan(0,0,0,0, (int)timeToPostponeInMilliSeconds); TimeSpan timeSpanToPostponeInSeconds = DateTimeUtilities.getTimeSpanFromTotalMilliSeconds((int)timeToPostponeInMilliSeconds); // get the end of the first trace and the begining + overlap of the second trace DateTime CurrentTraceSampleFrameTime = currentTrace.VideoSegmentStartTime.AddMilliseconds(frameTimeInMiliSeconds * (totalFrameCountsBeforeStitching - 1)); DateTime NextTraceSampleFrameTime = nextTrace.VideoSegmentStartTime.AddMilliseconds(frameTimeInMiliSeconds * (Math.Max(noFramesOverlap - 1, 0))); // current trace foreach (CompressedTrack entity in currentTrace.tracks) { SpaceTime st = entity.getSpaceTimeAt(CurrentTraceSampleFrameTime); BooleanAttribute outofview_attr = entity.getAttributeAt("outofview", CurrentTraceSampleFrameTime); if (st != null && outofview_attr != null && !outofview_attr.value) { BoundingBox box = st.region; boundingboxes[0].Add(box); CrossChunkEntitiesAnnotationStrings[0].Add(entity.unCompressToTrackAnnotationString()); } else { SingleChunkEntitiesAnnotationStrings[0].Add(entity.unCompressToTrackAnnotationString()); } } // next trace nextTrace.VideoSegmentStartTime += timeSpanToPostponeInSeconds; nextTrace.VideoSegmentEndTime += timeSpanToPostponeInSeconds; foreach (CompressedTrack entity in nextTrace.tracks) { // get the frame first before changing the time stmaps. SpaceTime st = entity.getSpaceTimeAt(NextTraceSampleFrameTime); BooleanAttribute outofview_attr = entity.getAttributeAt("outofview", entity.endTime); /// postpone the timestamp of the second trace entity.startTime += timeSpanToPostponeInSeconds; entity.endTime += timeSpanToPostponeInSeconds; entity.spaceTimeTrack.startTime += timeSpanToPostponeInSeconds; entity.spaceTimeTrack.endTime += timeSpanToPostponeInSeconds; //foreach (SpaceTime st in entity.spaceTimeTrack.space_time_track) for (int i = 0; i < entity.spaceTimeTrack.space_time_track.Count; i++) { entity.spaceTimeTrack.space_time_track[i].time += timeSpanToPostponeInSeconds; } //foreach (CompressedBooleanAttributeTrack booleanAttributeTrack in entity.booleanAttributeTracks.Values) foreach (string key in entity.booleanAttributeTracks.Keys) { entity.booleanAttributeTracks[key].startTime += timeSpanToPostponeInSeconds; entity.booleanAttributeTracks[key].endTime += timeSpanToPostponeInSeconds; //foreach (BooleanAttribute ba in entity.booleanAttributeTracks[key].attribute_track) for (int i = 0; i < entity.booleanAttributeTracks[key].attribute_track.Count; i++) { entity.booleanAttributeTracks[key].attribute_track[i].time += timeSpanToPostponeInSeconds; } } if (st != null && outofview_attr != null && !outofview_attr.value) { BoundingBox box = st.region; boundingboxes[1].Add(box); CrossChunkEntitiesAnnotationStrings[1].Add(entity.unCompressToTrackAnnotationString()); } else { SingleChunkEntitiesAnnotationStrings[1].Add(entity.unCompressToTrackAnnotationString()); } } List <MultipartiteWeightedMatch> association = BoundingBoxAssociation.computeBoundingBoxAssociations(boundingboxes); DateTime NewFrameStartTime = currentTrace.VideoSegmentStartTime + DateTimeUtilities.getTimeSpanFromTotalMilliSeconds((int)(totalFrameCountsBeforeStitching * frameTimeInMiliSeconds)); string totalStitchedAnnotationString = stitchAnnotationStringByAssociation(CrossChunkEntitiesAnnotationStrings, SingleChunkEntitiesAnnotationStrings, association, NewFrameStartTime); MultiObjectTrackingResult ret = MultiObjectTrackingResult.ConvertAnnotationStringToMultiObjectTrackingResult(currentTrace.VideoSegmentStartTime, totalStitchedAnnotationString, currentTrace.cameraId, currentTrace.UID, totalFrameCountsAfterStitching, fps); return(ret); }
public static MultiObjectTrackingResult stitchAllChunksOfOneVideo(List <SatyamAggregatedResultsTableEntry> entries, out List <string> ImageURLs, out int fps) { ImageURLs = new List <string>(); fps = 0; if (entries.Count == 0) { return(null); } SatyamJobStorageAccountAccess satyamStorage = new SatyamJobStorageAccountAccess(); MultiObjectTrackingResult stitched = new MultiObjectTrackingResult(); int totalFrameCounts = 0; // ensure the order is correct SortedDictionary <int, SatyamAggregatedResultsTableEntry> sortedEntries = new SortedDictionary <int, SatyamAggregatedResultsTableEntry>(); List <int> idx = new List <int>(); for (int i = 0; i < entries.Count; i++) { SatyamAggregatedResultsTableEntry entry = entries[i]; SatyamAggregatedResult satyamAggResult = JSonUtils.ConvertJSonToObject <SatyamAggregatedResult>(entry.ResultString); SatyamTask aggTask = JSonUtils.ConvertJSonToObject <SatyamTask>(satyamAggResult.TaskParameters); string video = URIUtilities.localDirectoryNameFromURI(aggTask.SatyamURI); string[] fields = video.Split('_'); int startingFrame = Convert.ToInt32(fields[fields.Length - 1]); if (!sortedEntries.ContainsKey(startingFrame)) { sortedEntries.Add(startingFrame, entries[i]); idx.Add(startingFrame); } } idx.Sort(); for (int i = 0; i < idx.Count; i++) { //SatyamAggregatedResultsTableEntry entry = entries[i]; SatyamAggregatedResultsTableEntry entry = sortedEntries[idx[i]]; SatyamAggregatedResult satyamAggResult = JSonUtils.ConvertJSonToObject <SatyamAggregatedResult>(entry.ResultString); SatyamTask aggTask = JSonUtils.ConvertJSonToObject <SatyamTask>(satyamAggResult.TaskParameters); MultiObjectTrackingSubmittedJob job = JSonUtils.ConvertJSonToObject <MultiObjectTrackingSubmittedJob>(aggTask.jobEntry.JobParameters); MultiObjectTrackingAggregatedResult aggresult = JSonUtils.ConvertJSonToObject <MultiObjectTrackingAggregatedResult>(satyamAggResult.AggregatedResultString); int noFramesOverlap = 0; if (job.ChunkOverlap != 0.0) { noFramesOverlap = (int)(job.ChunkOverlap * job.FrameRate); } List <string> TraceURLs = satyamStorage.getURLListOfSubDirectoryByURL(aggTask.SatyamURI); if (i == 0) { ImageURLs.AddRange(TraceURLs); string[] names = aggTask.SatyamURI.Split('/'); fps = job.FrameRate; stitched = aggresult.tracklets; totalFrameCounts += TraceURLs.Count; } else { int noNewFrames = 0; for (int j = noFramesOverlap; j < TraceURLs.Count; j++) { ImageURLs.Add(TraceURLs[j]); noNewFrames++; } //stitched = stitchTwoTracesByOneFrameBoundingBoxes(stitched, aggresult.tracklets, totalFrameCounts, totalFrameCounts+ noNewFrames, noFramesOverlap,fps); stitched = stitchTwoTracesByTubeletsOfOverlappingVideoChunk(stitched, aggresult.tracklets, totalFrameCounts, totalFrameCounts + noNewFrames, noFramesOverlap, fps); totalFrameCounts += noNewFrames; } //debug //generateVideoForEvaluation(ImageURLs, stitched, directoryName + "_" + i, videoName, fps); } return(stitched); }