public static void SaveResultVideosLocally(List <SatyamResultsTableEntry> entries, string directoryName, int fps = 10)
{
directoryName = directoryName + "\\Raw";
if (!Directory.Exists(directoryName))
{
Directory.CreateDirectory(directoryName);
}
SatyamJobStorageAccountAccess satyamStorage = new SatyamJobStorageAccountAccess();
for (int i = 0; i < entries.Count; i++)
{
SatyamResultsTableEntry entry = entries[i];
SatyamResult satyamResult = JSonUtils.ConvertJSonToObject <SatyamResult>(entry.ResultString);
SatyamTask task = JSonUtils.ConvertJSonToObject <SatyamTask>(satyamResult.TaskParametersString);
MultiObjectTrackingSubmittedJob job = JSonUtils.ConvertJSonToObject <MultiObjectTrackingSubmittedJob>(task.jobEntry.JobParameters);
string blobDir = URIUtilities.localDirectoryFullPathFromURI(task.SatyamURI);
//VATIC_DVA_CrowdsourcedResult taskr = new VATIC_DVA_CrowdsourcedResult(satyamResult.TaskResult, task.SatyamURI, start, end, task.SatyamJobSubmissionsTableEntryID.ToString());
VATIC_DVA_CrowdsourcedResult taskr = MultiObjectTrackingAggregator.createVATIC_DVA_CrowdsourcedResultUsingSatyamBlobImageCount(satyamResult.TaskResult, blobDir, entry.ID.ToString(), job.FrameRate);
MultiObjectTrackingResult res = taskr.getCompressedTracksInTimeSegment();
List <string> ImageURLs = satyamStorage.getURLListOfSpecificExtensionUnderSubDirectoryByURI(blobDir, new List <string>()
{
"jpg", "png"
});
string videoName = URIUtilities.localDirectoryNameFromURI(blobDir) + "_" + URIUtilities.filenameFromURINoExtension(task.SatyamURI) + "_" + entry.ID;
MultiObjectTrackingAnalyzer.generateVideoForEvaluation(ImageURLs, res, directoryName, videoName, job.FrameRate);
}
}
public static void GroupEntriesByVideoNameAndStitchAndSaveAggregatedResultVideosLocally(List <SatyamAggregatedResultsTableEntry> entries, string directoryName)
{
if (entries.Count == 0)
{
return;
}
directoryName = directoryName + "\\AggregatedStitched";
if (!Directory.Exists(directoryName))
{
Directory.CreateDirectory(directoryName);
}
SortedDictionary <string, List <SatyamAggregatedResultsTableEntry> > entriesPerVideo = MultiObjectTrackingAnalyzer.GroupChunksByVideoName(entries);
foreach (string videoname in entriesPerVideo.Keys)
{
List <string> ImageURLs;
int fps;
MultiObjectTrackingResult stitched = stitchAllChunksAllObjectsOfOneVideo(entriesPerVideo[videoname], out ImageURLs, out fps);
if (stitched != null)
{
// incase video is too large
MultiObjectTrackingAnalyzer.generateVideoFramesForEvaluation(ImageURLs, stitched, directoryName, videoname, fps);
//MultiObjectTrackingAnalyzer.generateVideoForEvaluation(ImageURLs, stitched, directoryName, videoname, fps);
}
}
}
public static void generateVideoForEvaluation(List <string> ImageURLs, MultiObjectTrackingResult ctts, String directory, String videoName, int fps)
{
if (Directory.Exists(directory + "\\" + videoName))
{
return;
}
if (File.Exists(directory + "\\" + videoName + ".mp4"))
{
return;
}
List <Image> imageList = new List <Image>();
var wc = new WebClient();
foreach (string uri in ImageURLs)
{
Image x = Image.FromStream(wc.OpenRead(uri));
imageList.Add(x);
}
List <DateTime> frameTimes = new List <DateTime>();
double frameTimeSpanInMiliseconds = (double)1000 / (double)fps;
//double frameTimeSpanInMiliseconds = (double)(ChunkDuration) / (double)(ImageURLs.Count) * 1000;
DateTime start = ctts.VideoSegmentStartTime;
for (int i = 0; i < ImageURLs.Count; i++)
{
DateTime t;
t = start.AddMilliseconds(frameTimeSpanInMiliseconds * i);
frameTimes.Add(t);
}
generateVideoForEvaluation(imageList, frameTimes, ctts, videoName, directory);
}
public static void SaveKITTIResultVideosLocally(List <SatyamResultsTableEntry> entries, string directoryName, int fps = 10)
{
directoryName = directoryName + "\\Raw";
if (!Directory.Exists(directoryName))
{
Directory.CreateDirectory(directoryName);
}
SatyamJobStorageAccountAccess satyamStorage = new SatyamJobStorageAccountAccess();
for (int i = 0; i < entries.Count; i++)
{
SatyamResultsTableEntry entry = entries[i];
SatyamResult satyamResult = JSonUtils.ConvertJSonToObject <SatyamResult>(entry.ResultString);
SatyamTask task = JSonUtils.ConvertJSonToObject <SatyamTask>(satyamResult.TaskParametersString);
MultiObjectTrackingSubmittedJob job = JSonUtils.ConvertJSonToObject <MultiObjectTrackingSubmittedJob>(task.jobEntry.JobParameters);
//VATIC_DVA_CrowdsourcedResult taskr = VATIC_DVA_CrowdsourcedResult.createVATIC_DVA_CrowdsourcedResultUsingSatyamBlobImageCount(satyamResult.TaskResult, task.SatyamURI, task.SatyamJobSubmissionsTableEntryID.ToString(), job.FrameRate);
//MultiObjectTrackingResult res = taskr.getCompressedTracksInTimeSegment();
string[] names = task.SatyamURI.Split('/');
string videoName = names[names.Length - 2] + "_" + entry.ID;
string[] fields = videoName.Split('_');
string videoSequence = fields[fields.Length - 5];
int startingFrame = Convert.ToInt32(fields[fields.Length - 2]);
int maxChunkEndFrame = startingFrame + job.ChunkDuration * job.FrameRate;
int noFrameOverlap = (int)(job.ChunkOverlap * job.FrameRate);
if (startingFrame != 0)
{
startingFrame -= noFrameOverlap;
}
if (entry.JobGUID == "1e43a983-548d-4a2e-8161-5537eb985902")
{
videoSequence = getCorrectSequenceNo(videoSequence);
}
string videoFrameDir = DirectoryConstants.KITTITrackingImages + videoSequence;
List <string> files = Directory.GetFiles(videoFrameDir).ToList();
List <string> ImageURLs = new List <string>();
for (int j = startingFrame; j < files.Count && j < maxChunkEndFrame; j++)
{
ImageURLs.Add(files[j]);
}
if (ImageURLs.Count == 0)
{
continue;
}
VATIC_DVA_CrowdsourcedResult taskr = new VATIC_DVA_CrowdsourcedResult(satyamResult.TaskResult, videoName, entry.ID.ToString(), ImageURLs.Count, job.FrameRate);
MultiObjectTrackingResult res = taskr.getCompressedTracksInTimeSegment();
generateVideoForEvaluation(ImageURLs, res, directoryName, videoName, job.FrameRate);
}
}
public static void SaveResultVideosLocally(List <SatyamResultsTableEntry> entries, string directoryName, int fps = 10)
{
directoryName = directoryName + "\\Raw";
if (!Directory.Exists(directoryName))
{
Directory.CreateDirectory(directoryName);
}
SatyamJobStorageAccountAccess satyamStorage = new SatyamJobStorageAccountAccess();
for (int i = 0; i < entries.Count; i++)
{
SatyamResultsTableEntry entry = entries[i];
SatyamResult satyamResult = JSonUtils.ConvertJSonToObject <SatyamResult>(entry.ResultString);
SatyamTask task = JSonUtils.ConvertJSonToObject <SatyamTask>(satyamResult.TaskParametersString);
MultiObjectTrackingSubmittedJob job = JSonUtils.ConvertJSonToObject <MultiObjectTrackingSubmittedJob>(task.jobEntry.JobParameters);
//VATIC_DVA_CrowdsourcedResult taskr = new VATIC_DVA_CrowdsourcedResult(satyamResult.TaskResult, task.SatyamURI, start, end, task.SatyamJobSubmissionsTableEntryID.ToString());
VATIC_DVA_CrowdsourcedResult taskr = MultiObjectTrackingAggregator.createVATIC_DVA_CrowdsourcedResultUsingSatyamBlobImageCount(satyamResult.TaskResult, task.SatyamURI, entry.ID.ToString(), job.FrameRate);
MultiObjectTrackingResult res = taskr.getCompressedTracksInTimeSegment();
List <string> ImageURLs = satyamStorage.getURLListOfSubDirectoryByURL(task.SatyamURI);
string[] names = task.SatyamURI.Split('/');
string videoName = names[names.Length - 2] + "_" + entry.ID;
generateVideoForEvaluation(ImageURLs, res, directoryName, videoName, job.FrameRate);
}
}
public static double[,] computeTrackSimilarityMatrix(MultiObjectTrackingResult cts1, MultiObjectTrackingResult cts2, ICompressedTrackSimilarityMetric metric)
{
double[,] ret = new double[cts1.tracks.Count, cts2.tracks.Count];
for (int i = 0; i < cts1.tracks.Count; i++)
{
CompressedTrack t1 = cts1.tracks[i];
for (int j = 0; j < cts2.tracks.Count; j++)
{
CompressedTrack t2 = cts2.tracks[j];
ret[i, j] = metric.getMetric(t1, t2);
}
}
return(ret);
}
public static void SaveAggregatedResultVideosLocally(List <SatyamAggregatedResultsTableEntry> entries, string directoryName)
{
directoryName = directoryName + "\\Aggregated";
if (!Directory.Exists(directoryName))
{
Directory.CreateDirectory(directoryName);
}
SatyamJobStorageAccountAccess satyamStorage = new SatyamJobStorageAccountAccess();
for (int i = 0; i < entries.Count; i++)
{
SatyamAggregatedResultsTableEntry entry = entries[i];
SatyamAggregatedResult satyamAggResult = JSonUtils.ConvertJSonToObject <SatyamAggregatedResult>(entry.ResultString);
SatyamTask aggTask = JSonUtils.ConvertJSonToObject <SatyamTask>(satyamAggResult.TaskParameters);
MultiObjectTrackingSubmittedJob job = JSonUtils.ConvertJSonToObject <MultiObjectTrackingSubmittedJob>(aggTask.jobEntry.JobParameters);
TrackletLabelingAggregatedResult aggresult = JSonUtils.ConvertJSonToObject <TrackletLabelingAggregatedResult>(satyamAggResult.AggregatedResultString);
WebClient wb = new WebClient();
Stream aggTrackStream = wb.OpenRead(aggresult.AggregatedTrackletsString_URL);
StreamReader reader = new StreamReader(aggTrackStream);
String aggTrackString = reader.ReadToEnd();
MultiObjectTrackingResult aggTracks = JSonUtils.ConvertJSonToObject <MultiObjectTrackingResult>(aggTrackString);
string blobDir = URIUtilities.localDirectoryFullPathFromURI(aggTask.SatyamURI);
List <string> ImageURLs = satyamStorage.getURLListOfSpecificExtensionUnderSubDirectoryByURI(blobDir, new List <string>()
{
"jpg", "png"
});
string videoName = URIUtilities.localDirectoryNameFromURI(blobDir) + "_" + URIUtilities.filenameFromURINoExtension(aggTask.SatyamURI) + "_" + entry.ID;
MultiObjectTrackingAnalyzer.generateVideoForEvaluation(ImageURLs, aggTracks, directoryName, videoName, job.FrameRate);
}
}
public static void AggregateWithParameter(string guid,
int MinResults = TaskConstants.TRACKLET_LABELING_MTURK_MIN_RESULTS_TO_AGGREGATE,
int MaxResults = TaskConstants.TRACKLET_LABELING_MTURK_MAX_RESULTS_TO_AGGREGATE,
double boxToleranceThreshold = TaskConstants.TRACKLET_LABELING_BOX_DEVIATION_THRESHOLD,
double ObjectCoverageApprovalThresholdPerVideo = TaskConstants.TRACKLET_LABELING_APPROVALRATIO_PER_VIDEO,
double BoxCoverageApprovalThresholdPerTrack = TaskConstants.TRACKLET_LABELING_APPROVALRATIO_PER_TRACK,
int consensusNumber = TaskConstants.TRACKLET_LABELING_MIN_RESULTS_FOR_CONSENSUS,
double minTubeletIoUSimilarityThreshold = TaskConstants.TRACKLET_LABELING_MIN_TUBELET_SIMILARITY_THRESHOLD,
double attributeMajority = TaskConstants.TRACKLET_LABELING_MTURK_ATTRIBUTE_MAJORITY_THRESHOLD,
bool allFalseAttributeInvalid = false
)
{
string configString = "Min_" + MinResults + "_Max_" + MaxResults + "_IoU_" + minTubeletIoUSimilarityThreshold + "_Ratio_" + ObjectCoverageApprovalThresholdPerVideo;
Console.WriteLine("Aggregating for " + guid + " with param set " + configString);
SatyamResultsTableAccess resultsDB = new SatyamResultsTableAccess();
List <SatyamResultsTableEntry> entries = resultsDB.getEntriesByGUID(guid);
resultsDB.close();
SortedDictionary <DateTime, List <SatyamResultsTableEntry> > entriesBySubmitTime = SatyamResultValidationToolKit.SortResultsBySubmitTime_OneResultPerTurkerPerTask(entries);
Dictionary <int, List <MultiObjectTrackingResult> > ResultsPerTask = new Dictionary <int, List <MultiObjectTrackingResult> >();
List <int> aggregatedTasks = new List <int>();
int noTotalConverged = 0;
//int noCorrect = 0;
int noTerminatedTasks = 0;
List <SatyamAggregatedResultsTableEntry> aggEntries = new List <SatyamAggregatedResultsTableEntry>();
Dictionary <int, int> noResultsNeededForAggregation = SatyamResultsAnalysis.getNoResultsNeededForAggregationFromLog(configString, guid);
Dictionary <int, int> noResultsNeededForAggregation_new = new Dictionary <int, int>();
// play back by time
Dictionary <int, List <string> > WorkersPerTask = new Dictionary <int, List <string> >();
foreach (DateTime t in entriesBySubmitTime.Keys)
{
//Console.WriteLine("Processing Results of time: {0}", t);
List <SatyamResultsTableEntry> ResultEntries = entriesBySubmitTime[t];
foreach (SatyamResultsTableEntry entry in ResultEntries)
{
SatyamResult satyamResult = JSonUtils.ConvertJSonToObject <SatyamResult>(entry.ResultString);
SatyamTask task = JSonUtils.ConvertJSonToObject <SatyamTask>(satyamResult.TaskParametersString);
MultiObjectTrackingSubmittedJob job = JSonUtils.ConvertJSonToObject <MultiObjectTrackingSubmittedJob>(task.jobEntry.JobParameters);
string fileName = URIUtilities.filenameFromURINoExtension(task.SatyamURI);
int taskEntryID = entry.SatyamTaskTableEntryID;
if (aggregatedTasks.Contains(taskEntryID))
{
continue;
}
if (!ResultsPerTask.ContainsKey(taskEntryID))
{
ResultsPerTask.Add(taskEntryID, new List <MultiObjectTrackingResult>());
WorkersPerTask.Add(taskEntryID, new List <string>());
}
// remove duplicate workers result
string workerID = satyamResult.amazonInfo.WorkerID;
if (WorkersPerTask[taskEntryID].Contains(workerID))
{
continue;
}
//enclose only non-duplicate results, one per each worker.
WorkersPerTask[taskEntryID].Add(workerID);
string videoName = URIUtilities.localDirectoryNameFromURI(task.SatyamURI);
string[] fields = videoName.Split('_');
int startingFrame = Convert.ToInt32(fields[fields.Length - 1]);
int maxChunkEndFrame = startingFrame + job.ChunkDuration * job.FrameRate;
int noFrameOverlap = (int)(job.ChunkOverlap * job.FrameRate);
if (startingFrame != 0)
{
startingFrame -= noFrameOverlap;
}
string blobDir = URIUtilities.localDirectoryFullPathFromURI(task.SatyamURI);
VATIC_DVA_CrowdsourcedResult taskr = TrackletLabelingAggregator.createVATIC_DVA_CrowdsourcedResultUsingSatyamBlobImageCount(satyamResult.TaskResult, blobDir, entry.SatyamTaskTableEntryID.ToString(), job.FrameRate);
//MultiObjectTrackingResult result_tmp = vatic_tmp.getCompressedTracksInTimeSegment();
//VATIC_DVA_CrowdsourcedResult taskr = new VATIC_DVA_CrowdsourcedResult(satyamResult.TaskResult, videoName, entry.ID.ToString(), ImageURLs.Count, job.FrameRate);
MultiObjectTrackingResult res = taskr.getCompressedTracksInTimeSegment();
if (allFalseAttributeInvalid)
{
if (TrackletLabelingAggregator.AllAttributeAllFalse(res))
{
continue;
}
}
ResultsPerTask[taskEntryID].Add(res);
// check log if enough results are collected
if (noResultsNeededForAggregation != null && noResultsNeededForAggregation.ContainsKey(taskEntryID) &&
ResultsPerTask[taskEntryID].Count < noResultsNeededForAggregation[taskEntryID])
{
continue;
}
// hack for masters
int tempMin = MinResults;
if (TaskConstants.masterGUIDs.Contains(entry.JobGUID))
{
tempMin = 1;
}
TrackletLabelingAggregatedResult aggResult = TrackletLabelingAggregator.getAggregatedResult(ResultsPerTask[taskEntryID],
guid, task.SatyamURI,
job.FrameRate, job.BoundaryLines,
MinResults, MaxResults, boxToleranceThreshold,
ObjectCoverageApprovalThresholdPerVideo,
BoxCoverageApprovalThresholdPerTrack,
consensusNumber, minTubeletIoUSimilarityThreshold,
attributeMajority);
if (aggResult == null)
{
continue;
}
//////////////// aggregation happen
// record logs
if (noResultsNeededForAggregation == null || !noResultsNeededForAggregation.ContainsKey(taskEntryID))
{
noResultsNeededForAggregation_new.Add(taskEntryID, ResultsPerTask[taskEntryID].Count);
}
aggregatedTasks.Add(taskEntryID);
noTotalConverged++;
if (ResultsPerTask[taskEntryID].Count >= MaxResults)
{
noTerminatedTasks++;
}
SatyamAggregatedResult SatyamAggResult = new SatyamAggregatedResult();
SatyamAggResult.SatyamTaskTableEntryID = taskEntryID;
SatyamAggResult.AggregatedResultString = JSonUtils.ConvertObjectToJSon <TrackletLabelingAggregatedResult>(aggResult);
SatyamAggResult.TaskParameters = JSonUtils.ConvertJSonToObject <SatyamResult>(entry.ResultString).TaskParametersString;
SatyamAggregatedResultsTableEntry aggEntry = new SatyamAggregatedResultsTableEntry();
aggEntry.SatyamTaskTableEntryID = taskEntryID;
aggEntry.JobGUID = entry.JobGUID;
aggEntry.ResultString = JSonUtils.ConvertObjectToJSon <SatyamAggregatedResult>(SatyamAggResult);
aggEntries.Add(aggEntry);
}
}
Console.WriteLine("Total_Aggregated_Tasks: {0}", noTotalConverged);
Console.WriteLine("Total_Terminated_Tasks: {0}", noTerminatedTasks);
SatyamResultsAnalysis.RecordAggregationLog(noResultsNeededForAggregation_new, configString, guid);
TrackletLabelingAnalyzer.GroupEntriesByVideoNameAndStitchAndSaveAggregatedResultVideosLocally(aggEntries, DirectoryConstants.defaultTempDirectory + guid);
}
public static MultiObjectTrackingResult stitchAllChunksAllObjectsOfOneVideo(List <SatyamAggregatedResultsTableEntry> entries, out List <string> ImageURLs, out int fps)
{
ImageURLs = new List <string>();
fps = 0;
if (entries.Count == 0)
{
return(null);
}
SatyamJobStorageAccountAccess satyamStorage = new SatyamJobStorageAccountAccess();
MultiObjectTrackingResult stitched = new MultiObjectTrackingResult();
int totalFrameCounts = 0;
// ensure the order is correct
SortedDictionary <int, List <SatyamAggregatedResultsTableEntry> > sortedEntries = new SortedDictionary <int, List <SatyamAggregatedResultsTableEntry> >();
List <int> idx = new List <int>();
for (int i = 0; i < entries.Count; i++)
{
SatyamAggregatedResultsTableEntry entry = entries[i];
SatyamAggregatedResult satyamAggResult = JSonUtils.ConvertJSonToObject <SatyamAggregatedResult>(entry.ResultString);
SatyamTask aggTask = JSonUtils.ConvertJSonToObject <SatyamTask>(satyamAggResult.TaskParameters);
string video = URIUtilities.localDirectoryNameFromURI(aggTask.SatyamURI);
string[] fields = video.Split('_');
int startingFrame = Convert.ToInt32(fields[fields.Length - 1]);
if (!sortedEntries.ContainsKey(startingFrame))
{
sortedEntries.Add(startingFrame, new List <SatyamAggregatedResultsTableEntry>());
idx.Add(startingFrame);
}
sortedEntries[startingFrame].Add(entries[i]);
}
idx.Sort();
List <string> AggObjIds = new List <string>();
for (int i = 0; i < idx.Count; i++)
{
int noFramesOverlap = 0;
string blobDir = "";
// grouping all objects that belong to the same chunk
MultiObjectTrackingResult aggTracksOfAllObjectsPerChunk = new MultiObjectTrackingResult();
List <string> objIds = new List <string>();
for (int j = 0; j < sortedEntries[idx[i]].Count; j++)
{
SatyamAggregatedResultsTableEntry entry = sortedEntries[idx[i]][j];
SatyamAggregatedResult satyamAggResult = JSonUtils.ConvertJSonToObject <SatyamAggregatedResult>(entry.ResultString);
SatyamTask aggTask = JSonUtils.ConvertJSonToObject <SatyamTask>(satyamAggResult.TaskParameters);
MultiObjectTrackingSubmittedJob job = JSonUtils.ConvertJSonToObject <MultiObjectTrackingSubmittedJob>(aggTask.jobEntry.JobParameters);
if (job.ChunkOverlap != 0.0)
{
noFramesOverlap = (int)(job.ChunkOverlap * job.FrameRate);
}
fps = job.FrameRate;
blobDir = URIUtilities.localDirectoryFullPathFromURI(aggTask.SatyamURI);
string objId = URIUtilities.filenameFromURINoExtension(aggTask.SatyamURI);
objIds.Add(objId);
TrackletLabelingAggregatedResult aggresult = JSonUtils.ConvertJSonToObject <TrackletLabelingAggregatedResult>(satyamAggResult.AggregatedResultString);
WebClient wb = new WebClient();
Stream aggTrackStream = wb.OpenRead(aggresult.AggregatedTrackletsString_URL);
StreamReader reader = new StreamReader(aggTrackStream);
String aggTrackString = reader.ReadToEnd();
MultiObjectTrackingResult aggTracks = JSonUtils.ConvertJSonToObject <MultiObjectTrackingResult>(aggTrackString);
if (aggTracksOfAllObjectsPerChunk.tracks.Count == 0)
{
aggTracksOfAllObjectsPerChunk = aggTracks;
}
else
{
for (int k = 0; k < aggTracks.tracks.Count; k++)
{
aggTracksOfAllObjectsPerChunk.tracks.Add(aggTracks.tracks[k]);
}
}
}
List <string> TraceURLs = satyamStorage.getURLListOfSpecificExtensionUnderSubDirectoryByURI(blobDir, new List <string>()
{
"jpg", "png"
});
if (i == 0)
{
ImageURLs.AddRange(TraceURLs);
stitched = aggTracksOfAllObjectsPerChunk;
totalFrameCounts += TraceURLs.Count;
AggObjIds = objIds;
}
else
{
int noNewFrames = 0;
for (int j = noFramesOverlap; j < TraceURLs.Count; j++)
{
ImageURLs.Add(TraceURLs[j]);
noNewFrames++;
}
//stitched = MultiObjectTrackingAnalyzer.stitchTwoTracesByTubeletsOfOverlappingVideoChunk(stitched,
// aggTracksOfAllObjectsPerChunk, totalFrameCounts, totalFrameCounts + noNewFrames, noFramesOverlap, fps);
// postpone the agg trace
double frameTimeInMiliSeconds = (double)1000 / (double)fps;
double timeToPostponeInMilliSeconds = (double)(totalFrameCounts - noFramesOverlap) * frameTimeInMiliSeconds;
//TimeSpan timeSpanToPostponeInSeconds = new TimeSpan(0,0,0,0, (int)timeToPostponeInMilliSeconds);
TimeSpan timeSpanToPostponeInSeconds = DateTimeUtilities.getTimeSpanFromTotalMilliSeconds((int)timeToPostponeInMilliSeconds);
aggTracksOfAllObjectsPerChunk.postpone(timeSpanToPostponeInSeconds);
// overlap must be 0
for (int k = 0; k < aggTracksOfAllObjectsPerChunk.tracks.Count; k++)
{
if (!AggObjIds.Contains(objIds[k]))
{
stitched.tracks.Add(aggTracksOfAllObjectsPerChunk.tracks[k]);
AggObjIds.Add(objIds[k]);
}
else
{
// stitch the track for the same id
int tckIdx = AggObjIds.IndexOf(objIds[k]);
stitched.tracks[tckIdx] = CompressedTrack.stitchTwoAdjacentTrack(stitched.tracks[tckIdx], aggTracksOfAllObjectsPerChunk.tracks[k]);
}
}
totalFrameCounts += noNewFrames;
}
//debug
//generateVideoForEvaluation(ImageURLs, stitched, directoryName + "_" + i, videoName, fps);
}
return(stitched);
}
public static void generateVideoForEvaluation(List <Image> Images, List <DateTime> dateTimeList, MultiObjectTrackingResult ctts, String videoName, String directory)
{
List <Image> imagesWithTracks = new List <Image>();
for (int i = 0; i < Images.Count; i++)
{
List <BoundingBox> locations = new List <BoundingBox>();
List <string> labels = new List <string>();
Dictionary <string, List <bool> > attributes = new Dictionary <string, List <bool> >();
attributes.Add("occlusion", new List <bool>());
if (ctts.tracks.Count != 0)
{
foreach (string key in ctts.tracks[0].booleanAttributeTracks.Keys)
{
if (!attributes.ContainsKey(key))
{
attributes.Add(key, new List <bool>());
}
}
}
List <int> idx = new List <int>();
for (int j = 0; j < ctts.tracks.Count; j++)
{
CompressedTrack ct = ctts.tracks[j];
SpaceTime st = ct.getSpaceTimeAt(dateTimeList[i]);
BooleanAttribute outofview_attr = ct.getAttributeAt("outofview", dateTimeList[i]);
if (st != null && outofview_attr != null && !outofview_attr.value)
{
BoundingBox l = st.region;
locations.Add(l);
labels.Add(ctts.tracks[j].label);
foreach (string key in attributes.Keys)
{
attributes[key].Add(ct.getAttributeAt(key, dateTimeList[i]).value);
}
idx.Add(j);
}
}
Image new_image = generateTrackImage(Images[i], labels, locations, attributes, idx);
imagesWithTracks.Add(new_image);
}
Console.WriteLine("Saving " + directory + "\\" + videoName);
FFMpegWrappers.generateVideoFromFrames(imagesWithTracks, videoName, directory);
//Directory.CreateDirectory(directory + "\\" + videoName);
//Console.WriteLine("Saving " + directory + "\\" + videoName);
//for (int i = 0; i < imagesWithTracks.Count; i++)
//{
// imagesWithTracks[i].Save(directory + "\\" + videoName + "\\img" + i.ToString("000") + ".jpg");
//}
//Console.WriteLine("done");
}
// if the stitched video is too long
public static void generateVideoFramesForEvaluation(List <string> ImageURLs, MultiObjectTrackingResult ctts, String directory, String videoName, int fps)
{
if (Directory.Exists(directory + "\\" + videoName))
{
return;
}
Directory.CreateDirectory(directory + "\\" + videoName);
Console.WriteLine("Saving " + directory + "\\" + videoName);
//List<Image> imageList = new List<Image>();
var wc = new WebClient();
//List<DateTime> frameTimes = new List<DateTime>();
double frameTimeSpanInMiliseconds = (double)1000 / (double)fps;
//double frameTimeSpanInMiliseconds = (double)(ChunkDuration) / (double)(ImageURLs.Count) * 1000;
DateTime start = ctts.VideoSegmentStartTime;
for (int i = 0; i < ImageURLs.Count; i++)
{
DateTime t;
t = start.AddMilliseconds(frameTimeSpanInMiliseconds * i);
//frameTimes.Add(t);
Image x = Image.FromStream(wc.OpenRead(ImageURLs[i]));
//imageList.Add(x);
List <BoundingBox> locations = new List <BoundingBox>();
List <string> labels = new List <string>();
Dictionary <string, List <bool> > attributes = new Dictionary <string, List <bool> >();
attributes.Add("occlusion", new List <bool>());
List <int> idx = new List <int>();
if (ctts.tracks.Count != 0)
{
foreach (string key in ctts.tracks[0].booleanAttributeTracks.Keys)
{
if (!attributes.ContainsKey(key))
{
attributes.Add(key, new List <bool>());
}
}
}
for (int j = 0; j < ctts.tracks.Count; j++)
{
CompressedTrack ct = ctts.tracks[j];
SpaceTime st = ct.getSpaceTimeAt(t);
BooleanAttribute outofview_attr = ct.getAttributeAt("outofview", t);
if (st != null && outofview_attr != null && !outofview_attr.value)
{
BoundingBox l = st.region;
locations.Add(l);
labels.Add(ctts.tracks[j].label);
//attributes["occlusion"].Add(ct.getAttributeAt("occlusion", t).value);
foreach (string key in attributes.Keys)
{
attributes[key].Add(ct.getAttributeAt(key, t).value);
}
idx.Add(j);
}
}
Image new_image = generateTrackImage(x, labels, locations, attributes, idx);
new_image.Save(directory + "\\" + videoName + "\\img" + i.ToString("000") + ".jpg");
}
FFMpegWrappers.generateVideoFromFolderofFrames(videoName, directory + "\\" + videoName + "\\");
Console.WriteLine("done");
}
public static MultiObjectTrackingResult stitchTwoTracesByTubeletsOfOverlappingVideoChunk(MultiObjectTrackingResult currentTrace, MultiObjectTrackingResult nextTrace,
int totalFrameCountsBeforeStitching, int totalFrameCountsAfterStitching, int noFramesOverlap, int fps = 10)
{
//if (noFramesOverlap == 0) return null;
double frameTimeInMiliSeconds = (double)1000 / (double)fps;
List <MultiObjectTrackingResult> compressedOverlappingTracks = new List <MultiObjectTrackingResult>();
DateTime overlapStart = currentTrace.VideoSegmentStartTime.AddMilliseconds(frameTimeInMiliSeconds * (totalFrameCountsBeforeStitching - noFramesOverlap));
DateTime overlapEnd = currentTrace.VideoSegmentStartTime.AddMilliseconds(frameTimeInMiliSeconds * (totalFrameCountsBeforeStitching - 1));
//DateTime nextTrace_overlapStart = nextTrace.VideoSegmentStartTime;
//DateTime nextTrace_overlapEnd = nextTrace.VideoSegmentStartTime.AddMilliseconds(frameTimeInMiliSeconds * (noFramesOverlap-1));
//MultiObjectTrackingResult nextTrace_Overlap = nextTrace.getSubTimeSegment(nextTrace_overlapStart, nextTrace_overlapEnd, fps);
//compressedOverlappingTracks.Add(nextTrace_Overlap);
List <List <string> > CrossChunkEntitiesAnnotationStrings = new List <List <string> >();
List <List <string> > SingleChunkEntitiesAnnotationStrings = new List <List <string> >();
for (int i = 0; i < 2; i++)
{
//boundingboxes.Add(new List<BoundingBox>());
CrossChunkEntitiesAnnotationStrings.Add(new List <string>());
SingleChunkEntitiesAnnotationStrings.Add(new List <string>());
}
double timeToPostponeInMilliSeconds = (double)(totalFrameCountsBeforeStitching - noFramesOverlap) * frameTimeInMiliSeconds;
//TimeSpan timeSpanToPostponeInSeconds = new TimeSpan(0,0,0,0, (int)timeToPostponeInMilliSeconds);
TimeSpan timeSpanToPostponeInSeconds = DateTimeUtilities.getTimeSpanFromTotalMilliSeconds((int)timeToPostponeInMilliSeconds);
//// get the end of the first trace and the begining + overlap of the second trace
//DateTime CurrentTraceSampleFrameTime = currentTrace.VideoSegmentStartTime.AddMilliseconds(frameTimeInMiliSeconds * (totalFrameCountsBeforeStitching - 1));
//DateTime NextTraceSampleFrameTime = nextTrace.VideoSegmentStartTime.AddMilliseconds(frameTimeInMiliSeconds * (Math.Max(noFramesOverlap - 1, 0)));
// current trace
List <string> currentTrace_filteredAnnotationStrings = new List <string>();
foreach (CompressedTrack entity in currentTrace.tracks)
{
// get a snapshot frame at start and end of current trace
VATIC_DVA_Frame currentTraceOverlapStartFrame = entity.getFrameAt(overlapStart);
string annotationString = entity.unCompressToTrackAnnotationString();
string[] fields = annotationString.Split(':');
List <string> TimeFilteredSegmentStringList = new List <string>();
TimeFilteredSegmentStringList.Add(fields[0]); // adds the label back
if (currentTraceOverlapStartFrame != null)
{
TimeFilteredSegmentStringList.Add(currentTraceOverlapStartFrame.ToAnnotationString()); // adds a starting frame
}
// remove the label of the second string
int count = 0;
for (int j = 1; j < fields.Length; j++)
{
// remove the overlapping part.
DateTime time = DateTimeUtilities.getDateTimeFromString(fields[j].Split(',')[0]);
if (time <= overlapStart || time > overlapEnd)
{
continue;
}
TimeFilteredSegmentStringList.Add(fields[j]);
count++;
}
if (count == 0)
{
// not a cross chunk one
SingleChunkEntitiesAnnotationStrings[0].Add(annotationString);
continue;
}
else
{
//a cross chunk one
CrossChunkEntitiesAnnotationStrings[0].Add(annotationString);
// construct for association
string filteredAnnotationString = ObjectsToStrings.ListString(TimeFilteredSegmentStringList, ':');
currentTrace_filteredAnnotationStrings.Add(filteredAnnotationString);
}
}
string currentOverlap_totalFilteredAnnotationString = ObjectsToStrings.ListString(currentTrace_filteredAnnotationStrings, '|');
MultiObjectTrackingResult currentTrace_Overlap = MultiObjectTrackingResult.ConvertAnnotationStringToMultiObjectTrackingResult(overlapStart, currentOverlap_totalFilteredAnnotationString, "current", "0", noFramesOverlap, fps);
compressedOverlappingTracks.Add(currentTrace_Overlap);
// next trace
/// postpone the timestamp of the second trace
nextTrace.postpone(timeSpanToPostponeInSeconds);
List <string> nextTrace_filteredAnnotationStrings = new List <string>();
foreach (CompressedTrack entity in nextTrace.tracks)
{
VATIC_DVA_Frame nextTraceOverlapStartFrame = entity.getFrameAt(overlapStart);
VATIC_DVA_Frame nextTraceOverlapEndFrame = entity.getFrameAt(overlapEnd);
string annotationString = entity.unCompressToTrackAnnotationString();
string[] fields = annotationString.Split(':');
List <string> TimeFilteredSegmentStringList = new List <string>();
TimeFilteredSegmentStringList.Add(fields[0]); // adds the label back
if (nextTraceOverlapStartFrame != null)
{
TimeFilteredSegmentStringList.Add(nextTraceOverlapStartFrame.ToAnnotationString()); // adds a starting frame
}
// remove the label of the second string
int count = 0;
for (int j = 1; j < fields.Length; j++)
{
// remove the overlapping part.
DateTime time = DateTimeUtilities.getDateTimeFromString(fields[j].Split(',')[0]);
if (time <= overlapStart || time >= overlapEnd)
{
continue;
}
TimeFilteredSegmentStringList.Add(fields[j]);
count++;
}
if (count == 0)
{
// not a cross chunk one
SingleChunkEntitiesAnnotationStrings[1].Add(annotationString);
continue;
}
else
{
//a cross chunk one
CrossChunkEntitiesAnnotationStrings[1].Add(annotationString);
// construct for association
if (nextTraceOverlapEndFrame != null)
{
TimeFilteredSegmentStringList.Add(nextTraceOverlapEndFrame.ToAnnotationString()); // adds a ending frame
}
string filteredAnnotationString = ObjectsToStrings.ListString(TimeFilteredSegmentStringList, ':');
nextTrace_filteredAnnotationStrings.Add(filteredAnnotationString);
}
}
string nextOverlap_totalFilteredAnnotationString = ObjectsToStrings.ListString(nextTrace_filteredAnnotationStrings, '|');
MultiObjectTrackingResult nextTrace_Overlap = MultiObjectTrackingResult.ConvertAnnotationStringToMultiObjectTrackingResult(overlapStart, nextOverlap_totalFilteredAnnotationString, "next", "1", noFramesOverlap, fps);
compressedOverlappingTracks.Add(nextTrace_Overlap);
List <MultipartiteWeightedMatch> association = TrackletsAssociation.AssociateTracklets(compressedOverlappingTracks);
DateTime NewFrameStartTime = currentTrace.VideoSegmentStartTime + DateTimeUtilities.getTimeSpanFromTotalMilliSeconds((int)(totalFrameCountsBeforeStitching * frameTimeInMiliSeconds));
string totalStitchedAnnotationString = stitchAnnotationStringByAssociation(CrossChunkEntitiesAnnotationStrings, SingleChunkEntitiesAnnotationStrings,
association, NewFrameStartTime);
MultiObjectTrackingResult ret = MultiObjectTrackingResult.ConvertAnnotationStringToMultiObjectTrackingResult(currentTrace.VideoSegmentStartTime, totalStitchedAnnotationString, currentTrace.cameraId, currentTrace.UID, totalFrameCountsAfterStitching, fps);
return(ret);
}
public static MultiObjectTrackingResult stitchTwoTracesByOneFrameBoundingBoxes(MultiObjectTrackingResult currentTrace, MultiObjectTrackingResult nextTrace,
int totalFrameCountsBeforeStitching, int totalFrameCountsAfterStitching, int noFramesOverlap = 0, int fps = 10)
{
List <List <BoundingBox> > boundingboxes = new List <List <BoundingBox> >();
List <List <string> > CrossChunkEntitiesAnnotationStrings = new List <List <string> >();
List <List <string> > SingleChunkEntitiesAnnotationStrings = new List <List <string> >();
for (int i = 0; i < 2; i++)
{
boundingboxes.Add(new List <BoundingBox>());
CrossChunkEntitiesAnnotationStrings.Add(new List <string>());
SingleChunkEntitiesAnnotationStrings.Add(new List <string>());
}
double frameTimeInMiliSeconds = (double)1000 / (double)fps;
double timeToPostponeInMilliSeconds = (double)(totalFrameCountsBeforeStitching - noFramesOverlap) * frameTimeInMiliSeconds;
//TimeSpan timeSpanToPostponeInSeconds = new TimeSpan(0,0,0,0, (int)timeToPostponeInMilliSeconds);
TimeSpan timeSpanToPostponeInSeconds = DateTimeUtilities.getTimeSpanFromTotalMilliSeconds((int)timeToPostponeInMilliSeconds);
// get the end of the first trace and the begining + overlap of the second trace
DateTime CurrentTraceSampleFrameTime = currentTrace.VideoSegmentStartTime.AddMilliseconds(frameTimeInMiliSeconds * (totalFrameCountsBeforeStitching - 1));
DateTime NextTraceSampleFrameTime = nextTrace.VideoSegmentStartTime.AddMilliseconds(frameTimeInMiliSeconds * (Math.Max(noFramesOverlap - 1, 0)));
// current trace
foreach (CompressedTrack entity in currentTrace.tracks)
{
SpaceTime st = entity.getSpaceTimeAt(CurrentTraceSampleFrameTime);
BooleanAttribute outofview_attr = entity.getAttributeAt("outofview", CurrentTraceSampleFrameTime);
if (st != null && outofview_attr != null && !outofview_attr.value)
{
BoundingBox box = st.region;
boundingboxes[0].Add(box);
CrossChunkEntitiesAnnotationStrings[0].Add(entity.unCompressToTrackAnnotationString());
}
else
{
SingleChunkEntitiesAnnotationStrings[0].Add(entity.unCompressToTrackAnnotationString());
}
}
// next trace
nextTrace.VideoSegmentStartTime += timeSpanToPostponeInSeconds;
nextTrace.VideoSegmentEndTime += timeSpanToPostponeInSeconds;
foreach (CompressedTrack entity in nextTrace.tracks)
{
// get the frame first before changing the time stmaps.
SpaceTime st = entity.getSpaceTimeAt(NextTraceSampleFrameTime);
BooleanAttribute outofview_attr = entity.getAttributeAt("outofview", entity.endTime);
/// postpone the timestamp of the second trace
entity.startTime += timeSpanToPostponeInSeconds;
entity.endTime += timeSpanToPostponeInSeconds;
entity.spaceTimeTrack.startTime += timeSpanToPostponeInSeconds;
entity.spaceTimeTrack.endTime += timeSpanToPostponeInSeconds;
//foreach (SpaceTime st in entity.spaceTimeTrack.space_time_track)
for (int i = 0; i < entity.spaceTimeTrack.space_time_track.Count; i++)
{
entity.spaceTimeTrack.space_time_track[i].time += timeSpanToPostponeInSeconds;
}
//foreach (CompressedBooleanAttributeTrack booleanAttributeTrack in entity.booleanAttributeTracks.Values)
foreach (string key in entity.booleanAttributeTracks.Keys)
{
entity.booleanAttributeTracks[key].startTime += timeSpanToPostponeInSeconds;
entity.booleanAttributeTracks[key].endTime += timeSpanToPostponeInSeconds;
//foreach (BooleanAttribute ba in entity.booleanAttributeTracks[key].attribute_track)
for (int i = 0; i < entity.booleanAttributeTracks[key].attribute_track.Count; i++)
{
entity.booleanAttributeTracks[key].attribute_track[i].time += timeSpanToPostponeInSeconds;
}
}
if (st != null && outofview_attr != null && !outofview_attr.value)
{
BoundingBox box = st.region;
boundingboxes[1].Add(box);
CrossChunkEntitiesAnnotationStrings[1].Add(entity.unCompressToTrackAnnotationString());
}
else
{
SingleChunkEntitiesAnnotationStrings[1].Add(entity.unCompressToTrackAnnotationString());
}
}
List <MultipartiteWeightedMatch> association = BoundingBoxAssociation.computeBoundingBoxAssociations(boundingboxes);
DateTime NewFrameStartTime = currentTrace.VideoSegmentStartTime + DateTimeUtilities.getTimeSpanFromTotalMilliSeconds((int)(totalFrameCountsBeforeStitching * frameTimeInMiliSeconds));
string totalStitchedAnnotationString = stitchAnnotationStringByAssociation(CrossChunkEntitiesAnnotationStrings, SingleChunkEntitiesAnnotationStrings,
association, NewFrameStartTime);
MultiObjectTrackingResult ret = MultiObjectTrackingResult.ConvertAnnotationStringToMultiObjectTrackingResult(currentTrace.VideoSegmentStartTime, totalStitchedAnnotationString, currentTrace.cameraId, currentTrace.UID, totalFrameCountsAfterStitching, fps);
return(ret);
}
public static MultiObjectTrackingResult stitchAllChunksOfOneVideo(List <SatyamAggregatedResultsTableEntry> entries, out List <string> ImageURLs, out int fps)
{
ImageURLs = new List <string>();
fps = 0;
if (entries.Count == 0)
{
return(null);
}
SatyamJobStorageAccountAccess satyamStorage = new SatyamJobStorageAccountAccess();
MultiObjectTrackingResult stitched = new MultiObjectTrackingResult();
int totalFrameCounts = 0;
// ensure the order is correct
SortedDictionary <int, SatyamAggregatedResultsTableEntry> sortedEntries = new SortedDictionary <int, SatyamAggregatedResultsTableEntry>();
List <int> idx = new List <int>();
for (int i = 0; i < entries.Count; i++)
{
SatyamAggregatedResultsTableEntry entry = entries[i];
SatyamAggregatedResult satyamAggResult = JSonUtils.ConvertJSonToObject <SatyamAggregatedResult>(entry.ResultString);
SatyamTask aggTask = JSonUtils.ConvertJSonToObject <SatyamTask>(satyamAggResult.TaskParameters);
string video = URIUtilities.localDirectoryNameFromURI(aggTask.SatyamURI);
string[] fields = video.Split('_');
int startingFrame = Convert.ToInt32(fields[fields.Length - 1]);
if (!sortedEntries.ContainsKey(startingFrame))
{
sortedEntries.Add(startingFrame, entries[i]);
idx.Add(startingFrame);
}
}
idx.Sort();
for (int i = 0; i < idx.Count; i++)
{
//SatyamAggregatedResultsTableEntry entry = entries[i];
SatyamAggregatedResultsTableEntry entry = sortedEntries[idx[i]];
SatyamAggregatedResult satyamAggResult = JSonUtils.ConvertJSonToObject <SatyamAggregatedResult>(entry.ResultString);
SatyamTask aggTask = JSonUtils.ConvertJSonToObject <SatyamTask>(satyamAggResult.TaskParameters);
MultiObjectTrackingSubmittedJob job = JSonUtils.ConvertJSonToObject <MultiObjectTrackingSubmittedJob>(aggTask.jobEntry.JobParameters);
MultiObjectTrackingAggregatedResult aggresult = JSonUtils.ConvertJSonToObject <MultiObjectTrackingAggregatedResult>(satyamAggResult.AggregatedResultString);
int noFramesOverlap = 0;
if (job.ChunkOverlap != 0.0)
{
noFramesOverlap = (int)(job.ChunkOverlap * job.FrameRate);
}
List <string> TraceURLs = satyamStorage.getURLListOfSubDirectoryByURL(aggTask.SatyamURI);
if (i == 0)
{
ImageURLs.AddRange(TraceURLs);
string[] names = aggTask.SatyamURI.Split('/');
fps = job.FrameRate;
stitched = aggresult.tracklets;
totalFrameCounts += TraceURLs.Count;
}
else
{
int noNewFrames = 0;
for (int j = noFramesOverlap; j < TraceURLs.Count; j++)
{
ImageURLs.Add(TraceURLs[j]);
noNewFrames++;
}
//stitched = stitchTwoTracesByOneFrameBoundingBoxes(stitched, aggresult.tracklets, totalFrameCounts, totalFrameCounts+ noNewFrames, noFramesOverlap,fps);
stitched = stitchTwoTracesByTubeletsOfOverlappingVideoChunk(stitched, aggresult.tracklets, totalFrameCounts, totalFrameCounts + noNewFrames, noFramesOverlap, fps);
totalFrameCounts += noNewFrames;
}
//debug
//generateVideoForEvaluation(ImageURLs, stitched, directoryName + "_" + i, videoName, fps);
}
return(stitched);
}
