public static bool PreprocessSubmittedData(SatyamJobSubmissionsTableAccessEntry jobEntry)
{
string satyamContainerName = JobTemplateToSatyamContainerNameMap[jobEntry.JobTemplateType];
string GUID = jobEntry.JobGUID;
string satyamDirectoryName = GUID;
SatyamJob job = JSonUtils.ConvertJSonToObject <SatyamJob>(jobEntry.JobParametersString);
bool success = false;
switch (job.JobTemplateType)
{
case TaskConstants.TrackletLabeling:
case TaskConstants.TrackletLabeling_MTurk:
success = VATIC_Tracklet.ProcessAndUploadToAzureBlob(jobEntry);
break;
case TaskConstants.Tracking:
case TaskConstants.Tracking_MTurk:
success = TrackingDataPreprocessor.ProcessAndUploadToAzureBlob(jobEntry);
break;
case TaskConstants.Classification_Video:
case TaskConstants.Classification_Video_MTurk:
success = VideoClassficationPreprocessor.ProcessAndUploadToAzureBlob(jobEntry);
break;
case TaskConstants.Classification_Image:
case TaskConstants.Classification_Image_MTurk:
case TaskConstants.Counting_Image:
case TaskConstants.Counting_Image_MTurk:
case TaskConstants.Counting_Video:
case TaskConstants.Counting_Video_MTurk:
case TaskConstants.Detection_Image:
case TaskConstants.Detection_Image_MTurk:
case TaskConstants.Segmentation_Image:
case TaskConstants.Segmentation_Image_MTurk:
case TaskConstants.OpenEndedQuestion_Image:
case TaskConstants.OpenEndedQuestion_Image_MTurk:
default:
// all the images to satyam location
success = DefaultDataPreprocessor.copyDataFromUserBlobToSatyamBlob(jobEntry);
break;
}
return(success);
}
/// <summary>
/// Input should be a folder of videos and corresponding annotation file with the same name
/// </summary>
/// <param name="jobEntry"></param>
public static bool ProcessAndUploadToAzureBlob(SatyamJobSubmissionsTableAccessEntry jobEntry)
{
// if the input is a folder of folders of frames, then copy directly
SatyamJobStorageAccountAccess satyamStorage = new SatyamJobStorageAccountAccess();
string satyamContainerName = SatyamTaskGenerator.JobTemplateToSatyamContainerNameMap[jobEntry.JobTemplateType];
string GUID = jobEntry.JobGUID;
string satyamDirectoryName = GUID;
SatyamJob job = JSonUtils.ConvertJSonToObject <SatyamJob>(jobEntry.JobParametersString);
MultiObjectTrackingSubmittedJob jobParams = JSonUtils.ConvertJSonToObject <MultiObjectTrackingSubmittedJob>(job.JobParameters);
BlobContainerManager bcm = new BlobContainerManager();
string status = bcm.Connect(job.azureInformation.AzureBlobStorageConnectionString);
List <string> FileTypes = SatyamTaskGenerator.ValidFileTypesByTemplate[job.JobTemplateType];
if (status != "SUCCESS")
{
return(false);
}
string guidFolder = DirectoryConstants.defaultTempDirectory + "\\" + GUID;
Directory.CreateDirectory(guidFolder);
int chunkLength = jobParams.ChunkDuration; // sec
int outputFPS = jobParams.FrameRate;
double chunkOverlap = jobParams.ChunkOverlap; // sec
var client = new WebClient();
// sample to frames
int noFramePerChunk = (int)(chunkLength * outputFPS);
int noFrameOverlap = (int)(chunkOverlap * outputFPS);
List <string> videoUrls = bcm.getURLList(job.azureInformation.AzureBlobStorageContainerName, job.azureInformation.AzureBlobStorageContainerDirectoryName);
Dictionary <string, List <string> > videos = new Dictionary <string, List <string> >();
foreach (string videolink in videoUrls)
{
string videoName = URIUtilities.filenameFromURINoExtension(videolink);
if (!videos.ContainsKey(videoName))
{
videos.Add(videoName, new List <string>());
}
videos[videoName].Add(videolink);
}
foreach (string videoName in videos.Keys)
{
// filter out those that doesn't provide a annotation file with it....
if (videos[videoName].Count != 2)
{
Console.WriteLine("Warning: Not 2 files provided for {0}.", videoName);
//Directory.Delete(guidFolder, true);
//return false;
continue;
}
string videoURL = "";
string annotationURL = "";
foreach (string fileLink in videos[videoName])
{
string extension = URIUtilities.fileExtensionFromURI(fileLink);
if (extension != "txt")
{
videoURL = fileLink;
}
else
{
annotationURL = fileLink;
}
}
string outputDirectory = guidFolder + "\\" + videoName;
Directory.CreateDirectory(outputDirectory);
string videoNameWithExtension = URIUtilities.filenameFromURI(videoURL);
client.DownloadFile(videoURL, outputDirectory + "\\" + videoNameWithExtension);
FFMpegWrappers.ExtractFrames(DirectoryConstants.ffmpeg, outputDirectory + "\\" + videoNameWithExtension, outputDirectory, videoName, DateTime.Now, outputFPS);
File.Delete(outputDirectory + "\\" + videoNameWithExtension);
List <string> chunkFolders = TrackingDataPreprocessor.GroupFramesIntoChunks(outputDirectory, noFramePerChunk);
//parse VIRAT annotation file
string annotationNameWithExtension = URIUtilities.filenameFromURI(annotationURL);
client.DownloadFile(annotationURL, outputDirectory + "\\" + annotationNameWithExtension);
parseAnnotationFileIntoChunkFolders(chunkFolders,
outputDirectory + "\\" + annotationNameWithExtension,
noFramePerChunk, outputFPS);
//upload
for (int i = 0; i < chunkFolders.Count; i++)
{
string subDir = chunkFolders[i];
satyamStorage.uploadALocalFolder(subDir, satyamContainerName, satyamDirectoryName + "/Video_" + videoName + "_startingFrame_" + noFramePerChunk * i);
}
Directory.Delete(outputDirectory, true);
}
return(true);
}
