public static void PopulateTasksIntoTaskTable(SatyamJobSubmissionsTableAccessEntry job)
{
List <SatyamTask> tasks = generate(job);
if (tasks == null)
{
return;
}
SatyamTaskTableAccess dbAccess = new SatyamTaskTableAccess();
foreach (SatyamTask task in tasks)
{
String JobTemplateType = job.JobTemplateType;
String UserID = job.UserID;
String JobGUID = job.JobGUID;
String JsonString = JSonUtils.ConvertObjectToJSon <SatyamTask>(task);
DateTime SubmitTime = job.JobSubmitTime;
double price = task.jobEntry.amazonHITInformation.Price;
dbAccess.AddEntry(JobTemplateType, UserID, JobGUID, JsonString, SubmitTime, price);
}
dbAccess.close();
// add the total task number to job submission table
SatyamJobSubmissionsTableAccess jobDB = new SatyamJobSubmissionsTableAccess();
jobDB.UpdateEntryProgress(job.JobGUID, tasks.Count.ToString());
jobDB.close();
}
public static bool PreprocessSubmittedData(SatyamJobSubmissionsTableAccessEntry jobEntry)
{
string satyamContainerName = JobTemplateToSatyamContainerNameMap[jobEntry.JobTemplateType];
string GUID = jobEntry.JobGUID;
string satyamDirectoryName = GUID;
SatyamJob job = JSonUtils.ConvertJSonToObject <SatyamJob>(jobEntry.JobParametersString);
bool success = false;
switch (job.JobTemplateType)
{
case TaskConstants.TrackletLabeling:
case TaskConstants.TrackletLabeling_MTurk:
success = VATIC_Tracklet.ProcessAndUploadToAzureBlob(jobEntry);
break;
case TaskConstants.Tracking:
case TaskConstants.Tracking_MTurk:
success = TrackingDataPreprocessor.ProcessAndUploadToAzureBlob(jobEntry);
break;
case TaskConstants.Classification_Video:
case TaskConstants.Classification_Video_MTurk:
success = VideoClassficationPreprocessor.ProcessAndUploadToAzureBlob(jobEntry);
break;
case TaskConstants.Classification_Image:
case TaskConstants.Classification_Image_MTurk:
case TaskConstants.Counting_Image:
case TaskConstants.Counting_Image_MTurk:
case TaskConstants.Counting_Video:
case TaskConstants.Counting_Video_MTurk:
case TaskConstants.Detection_Image:
case TaskConstants.Detection_Image_MTurk:
case TaskConstants.Segmentation_Image:
case TaskConstants.Segmentation_Image_MTurk:
case TaskConstants.OpenEndedQuestion_Image:
case TaskConstants.OpenEndedQuestion_Image_MTurk:
default:
// all the images to satyam location
success = DefaultDataPreprocessor.copyDataFromUserBlobToSatyamBlob(jobEntry);
break;
}
return(success);
}
public static bool copyDataFromUserBlobToSatyamBlob(SatyamJobSubmissionsTableAccessEntry jobEntry)
{
string satyamContainerName = SatyamTaskGenerator.JobTemplateToSatyamContainerNameMap[jobEntry.JobTemplateType];
string GUID = jobEntry.JobGUID;
string satyamDirectoryName = GUID;
SatyamJob job = JSonUtils.ConvertJSonToObject <SatyamJob>(jobEntry.JobParametersString);
BlobContainerManager bcm = new BlobContainerManager();
string status = bcm.Connect(job.azureInformation.AzureBlobStorageConnectionString);
List <string> FileTypes = SatyamTaskGenerator.ValidFileTypesByTemplate[job.JobTemplateType];
if (status != "SUCCESS")
{
return(false);
}
SatyamJobStorageAccountAccess satyamStorage = new SatyamJobStorageAccountAccess();
satyamStorage.copyFilesFromAnotherAzureBlob(bcm, job.azureInformation.AzureBlobStorageContainerName, job.azureInformation.AzureBlobStorageContainerDirectoryName, satyamContainerName, satyamDirectoryName, FileTypes);
return(true);
}
public static string PopulateTasksIntoTaskTableDebug(SatyamJobSubmissionsTableAccessEntry job)
{
string debugString = "In PopulateTasksIntoTaskTable\n";
List <SatyamTask> tasks = generateDebug(job, debugString, out debugString);
return(debugString);
//SatyamTaskTableAccess dbAccess = new SatyamTaskTableAccess();
//foreach (SatyamTask task in tasks)
//{
// String JobTemplateType = job.JobTemplateType;
// String UserID = job.UserID;
// String JobGUID = job.JobGUID;
// String JsonString = JSonUtils.ConvertObjectToJSon<SatyamTask>(task);
// DateTime SubmitTime = job.JobSubmitTime;
// dbAccess.AddEntry(JobTemplateType, UserID, JobGUID, JsonString, SubmitTime);
//}
//dbAccess.close();
}
public static bool ProcessAndUploadToAzureBlob(SatyamJobSubmissionsTableAccessEntry jobEntry)
{
// Assumpltion: the input is a folder videos, convert to mp4, and upload to satyam blob
SatyamJobStorageAccountAccess satyamStorage = new SatyamJobStorageAccountAccess();
string satyamContainerName = SatyamTaskGenerator.JobTemplateToSatyamContainerNameMap[jobEntry.JobTemplateType];
string GUID = jobEntry.JobGUID;
string satyamDirectoryName = GUID;
SatyamJob job = JSonUtils.ConvertJSonToObject <SatyamJob>(jobEntry.JobParametersString);
SingleObjectLabelingSubmittedJob jobParams = JSonUtils.ConvertJSonToObject <SingleObjectLabelingSubmittedJob>(job.JobParameters);
BlobContainerManager bcm = new BlobContainerManager();
string status = bcm.Connect(job.azureInformation.AzureBlobStorageConnectionString);
List <string> FileTypes = SatyamTaskGenerator.ValidFileTypesByTemplate[job.JobTemplateType];
if (status != "SUCCESS")
{
return(false);
}
string guidFolder = DirectoryConstants.defaultTempDirectory + "\\" + GUID;
Directory.CreateDirectory(guidFolder);
//var client = new WebClient();
List <string> videoUrls = bcm.getURLList(job.azureInformation.AzureBlobStorageContainerName, job.azureInformation.AzureBlobStorageContainerDirectoryName);
foreach (string video in videoUrls)
{
FFMpegWrappers.ConvertVideoURLToMP4(DirectoryConstants.ffmpeg, video, guidFolder);
}
satyamStorage.uploadALocalFolder(guidFolder, satyamContainerName, satyamDirectoryName);
return(true);
}
// If the results were not good enough, or aggregation method changed, retargeted, users might want to reopen the job
// All we need to do: (IN STRICT ORDER)
// clear all results back to inconclusive,
// remove all aggregated results
// restore the task table as it was for the guid,
// change the job status back to launched, and
// WARNING:
// for safety, this has to be run atomically synchronously, without any parallel process. for now.
public static void reopenJobForMoreResults(string guid)
{
SatyamJobSubmissionsTableAccess jobDB = new SatyamJobSubmissionsTableAccess();
SatyamJobSubmissionsTableAccessEntry jobEntry = jobDB.getEntryByJobGIUD(guid);
jobDB.close();
if (jobEntry.JobStatus != JobStatus.completed)
{
Console.WriteLine("Job not completed yet!");
return;
}
SatyamAggregatedResultsTableAccess aggResultDB = new SatyamAggregatedResultsTableAccess();
bool delSuccess = aggResultDB.DeleteEntriesByGUID(guid);
aggResultDB.close();
if (!delSuccess)
{
Console.WriteLine("Delete Agg Result DB Failed");
return;
}
SatyamResultsTableAccess resultDB = new SatyamResultsTableAccess();
if (!resultDB.UpdateStatusByGUID(guid, ResultStatus.inconclusive))
{
Console.WriteLine("Update Result DB Failed");
//resultDB.close();
//return;
}
List <SatyamResultsTableEntry> results = resultDB.getEntriesByGUID(guid);
resultDB.close();
Dictionary <int, SatyamTask> taskParamsByTaskID = new Dictionary <int, SatyamTask>();
foreach (SatyamResultsTableEntry result in results)
{
if (taskParamsByTaskID.ContainsKey(result.SatyamTaskTableEntryID))
{
continue;
}
SatyamResult satyamRes = JSonUtils.ConvertJSonToObject <SatyamResult>(result.ResultString);
SatyamTask task = JSonUtils.ConvertJSonToObject <SatyamTask>(satyamRes.TaskParametersString);
taskParamsByTaskID.Add(result.SatyamTaskTableEntryID, task);
}
SatyamTaskTableAccess taskDB = new SatyamTaskTableAccess();
foreach (int taskID in taskParamsByTaskID.Keys)
{
SatyamTask task = taskParamsByTaskID[taskID];
SatyamJob job = task.jobEntry;
bool suc = taskDB.AddEntryWithSpecificID(taskID, job.JobTemplateType, job.UserID, job.JobGUIDString, JSonUtils.ConvertObjectToJSon(task),
job.JobSubmitTime, job.amazonHITInformation.Price);
if (!suc)
{
Console.WriteLine("Update Task Table Failed");
taskDB.close();
return;
}
}
taskDB.close();
jobDB = new SatyamJobSubmissionsTableAccess();
bool success = jobDB.UpdateEntryStatus(guid, JobStatus.launched);
jobDB.close();
if (!success)
{
Console.WriteLine("Update Job Entry Failed");
return;
}
}
public static List <SatyamTask> generateDebug(SatyamJobSubmissionsTableAccessEntry jobEntry, string debugString, out string newDebugString)
{
newDebugString = debugString + "In Generate \n";
BlobContainerManager containerManager = new BlobContainerManager();
newDebugString += "About to connect to Satayam\n";
string statusString = containerManager.ConnectDebug(SatyamJobStorageAccountAccess.connection_string);
newDebugString += "The connetion status is : " + statusString;
return(null);
//SatyamJobStorageAccountAccess satyamStorage = new SatyamJobStorageAccountAccess();
//newDebugString += "Got a new storage account\n";
//return null;
//string satyamContainerName = JobTemplateToSatyamContainerNameMap[jobEntry.JobTemplateType];
//string GUID = jobEntry.JobGUID;
//string satyamDirectoryName = GUID;
//newDebugString += "About to get object from JSon\n";
//return null;
//SatyamJob job = JSonUtils.ConvertJSonToObject<SatyamJob>(jobEntry.JobParametersString);
//newDebugString += "Got the job from JSon\n";
//return null;
//BlobContainerManager bcm = new BlobContainerManager();
//string status = bcm.Connect(job.azureInformation.AzureBlobStorageConnectionString);
//if (status != "SUCCESS")
//{
// return null;
//}
////first copy all the images to satyam location
//List<string> FileTypes = ValidFileTypesByTemplate[job.JobTemplateType];
//satyamStorage.copyFilesFromAnotherAzureBlob(bcm, job.azureInformation.AzureBlobStorageContainerName, job.azureInformation.AzureBlobStorageContainerDirectoryName, satyamContainerName, satyamDirectoryName, FileTypes);
////Now get all the URIs at the remote location
//List<string> FullURIList = bcm.getURLList(job.azureInformation.AzureBlobStorageContainerName, job.azureInformation.AzureBlobStorageContainerDirectoryName);
//List<string> URIList = new List<string>();
//foreach (string uri in FullURIList)
//{
// string filename = URIUtilities.filenameFromURI(uri);
// if (CheckFileExtensions.IsAllowedType(filename, FileTypes))
// {
// URIList.Add(uri);
// }
//}
//List<string> satyamURIList = satyamStorage.getURLList(satyamContainerName, satyamDirectoryName);
////Now create a dictionary mapping filename to URI for both the locations
//Dictionary<string, string> fileNameToURI = new Dictionary<string, string>();
//Dictionary<string, string> satyamFileNameToURI = new Dictionary<string, string>();
//for (int i = 0; i < URIList.Count; i++)
//{
// string fileName = URIUtilities.filenameFromURI(URIList[i]);
// string satyamFileName = URIUtilities.filenameFromURI(satyamURIList[i]);
// fileNameToURI.Add(fileName, URIList[i]);
// satyamFileNameToURI.Add(satyamFileName, satyamURIList[i]);
//}
//List<string> fileNameList = fileNameToURI.Keys.ToList();
////now create a task for each of these one by one
//List<SatyamTask> tasks = new List<SatyamTask>();
//foreach (string filename in fileNameList)
//{
// SatyamTask task = new SatyamTask();
// task.jobEntry = job;
// //task.OriginalURI = fileNameToURI[filename];
// task.SatyamURI = satyamFileNameToURI[filename];
// tasks.Add(task);
//}
//return tasks;
}
public static List <SatyamTask> generate(SatyamJobSubmissionsTableAccessEntry jobEntry)
{
SatyamJobStorageAccountAccess satyamStorage = new SatyamJobStorageAccountAccess();
string satyamContainerName = JobTemplateToSatyamContainerNameMap[jobEntry.JobTemplateType];
string GUID = jobEntry.JobGUID;
string satyamDirectoryName = GUID;
SatyamJob job = JSonUtils.ConvertJSonToObject <SatyamJob>(jobEntry.JobParametersString);
//BlobContainerManager bcm = new BlobContainerManager();
//string status = bcm.Connect(job.azureInformation.AzureBlobStorageConnectionString);
//if (status != "SUCCESS")
//{
// return null;
//}
////first copy all the images to satyam location
//List<string> FileTypes = ValidFileTypesByTemplate[job.JobTemplateType];
////Now get all the URIs at the remote location that are relevant
//List<string> FullURIList = bcm.getURLList(job.azureInformation.AzureBlobStorageContainerName, job.azureInformation.AzureBlobStorageContainerDirectoryName);
//List<string> URIList = new List<string>();
//List<string> fileNameList = new List<string>();
//foreach(string uri in FullURIList)
//{
// string filename = URIUtilities.filenameFromURI(uri);
// if(CheckFileExtensions.IsAllowedType(filename,FileTypes))
// {
// URIList.Add(uri);
// fileNameList.Add(filename);
// }
//}
////Now create a dictionary mapping filename to URI for both the locations
//Dictionary<string, string> fileNameToURI = new Dictionary<string, string>();
//for (int i = 0; i < URIList.Count; i++)
//{
// string fileName = URIUtilities.filenameFromURI(URIList[i]);
// fileNameToURI.Add(fileName, URIList[i]);
//}
List <string> satyamURIList = new List <string>();
switch (job.JobTemplateType)
{
case TaskConstants.Tracking:
case TaskConstants.Tracking_MTurk:
satyamURIList = satyamStorage.getImmediateNextLevelURLList(satyamContainerName, satyamDirectoryName); // list only immediate next level subblobs, directories / files
break;
case TaskConstants.TrackletLabeling:
case TaskConstants.TrackletLabeling_MTurk:
satyamURIList = satyamStorage.getURLListOfSpecificExtension(satyamContainerName, satyamDirectoryName, new List <string>()
{
"txt"
}); // list only the annotation files per chunk per object
break;
case TaskConstants.Classification_Image:
case TaskConstants.Classification_Image_MTurk:
case TaskConstants.Classification_Video:
case TaskConstants.Classification_Video_MTurk:
case TaskConstants.Counting_Image:
case TaskConstants.Counting_Image_MTurk:
case TaskConstants.Counting_Video:
case TaskConstants.Counting_Video_MTurk:
case TaskConstants.Detection_Image:
case TaskConstants.Detection_Image_MTurk:
case TaskConstants.Segmentation_Image:
case TaskConstants.Segmentation_Image_MTurk:
case TaskConstants.OpenEndedQuestion_Image:
case TaskConstants.OpenEndedQuestion_Image_MTurk:
default:
// all the images to satyam location
satyamURIList = satyamStorage.getURLList(satyamContainerName, satyamDirectoryName);
break;
}
//Dictionary<string, string> satyamFileNameToURI = new Dictionary<string, string>();
//for (int i = 0; i < satyamURIList.Count; i++)
//{
// string satyamFileName = URIUtilities.filenameFromURI(satyamURIList[i]);
// satyamFileNameToURI.Add(satyamFileName, satyamURIList[i]);
//}
//List<string> fileNameList = fileNameToURI.Keys.ToList();
//now create a task for each of these one by one
List <SatyamTask> tasks = new List <SatyamTask>();
foreach (string uri in satyamURIList)
{
SatyamTask task = new SatyamTask();
task.jobEntry = job;
//task.OriginalURI = fileNameToURI[filename];
task.SatyamURI = uri;
tasks.Add(task);
}
return(tasks);
}
/// <summary>
/// Input should be a folder of videos and corresponding annotation file with the same name
/// </summary>
/// <param name="jobEntry"></param>
public static bool ProcessAndUploadToAzureBlob(SatyamJobSubmissionsTableAccessEntry jobEntry)
{
// if the input is a folder of folders of frames, then copy directly
SatyamJobStorageAccountAccess satyamStorage = new SatyamJobStorageAccountAccess();
string satyamContainerName = SatyamTaskGenerator.JobTemplateToSatyamContainerNameMap[jobEntry.JobTemplateType];
string GUID = jobEntry.JobGUID;
string satyamDirectoryName = GUID;
SatyamJob job = JSonUtils.ConvertJSonToObject <SatyamJob>(jobEntry.JobParametersString);
MultiObjectTrackingSubmittedJob jobParams = JSonUtils.ConvertJSonToObject <MultiObjectTrackingSubmittedJob>(job.JobParameters);
BlobContainerManager bcm = new BlobContainerManager();
string status = bcm.Connect(job.azureInformation.AzureBlobStorageConnectionString);
List <string> FileTypes = SatyamTaskGenerator.ValidFileTypesByTemplate[job.JobTemplateType];
if (status != "SUCCESS")
{
return(false);
}
string guidFolder = DirectoryConstants.defaultTempDirectory + "\\" + GUID;
Directory.CreateDirectory(guidFolder);
int chunkLength = jobParams.ChunkDuration; // sec
int outputFPS = jobParams.FrameRate;
double chunkOverlap = jobParams.ChunkOverlap; // sec
var client = new WebClient();
// sample to frames
int noFramePerChunk = (int)(chunkLength * outputFPS);
int noFrameOverlap = (int)(chunkOverlap * outputFPS);
List <string> videoUrls = bcm.getURLList(job.azureInformation.AzureBlobStorageContainerName, job.azureInformation.AzureBlobStorageContainerDirectoryName);
Dictionary <string, List <string> > videos = new Dictionary <string, List <string> >();
foreach (string videolink in videoUrls)
{
string videoName = URIUtilities.filenameFromURINoExtension(videolink);
if (!videos.ContainsKey(videoName))
{
videos.Add(videoName, new List <string>());
}
videos[videoName].Add(videolink);
}
foreach (string videoName in videos.Keys)
{
// filter out those that doesn't provide a annotation file with it....
if (videos[videoName].Count != 2)
{
Console.WriteLine("Warning: Not 2 files provided for {0}.", videoName);
//Directory.Delete(guidFolder, true);
//return false;
continue;
}
string videoURL = "";
string annotationURL = "";
foreach (string fileLink in videos[videoName])
{
string extension = URIUtilities.fileExtensionFromURI(fileLink);
if (extension != "txt")
{
videoURL = fileLink;
}
else
{
annotationURL = fileLink;
}
}
string outputDirectory = guidFolder + "\\" + videoName;
Directory.CreateDirectory(outputDirectory);
string videoNameWithExtension = URIUtilities.filenameFromURI(videoURL);
client.DownloadFile(videoURL, outputDirectory + "\\" + videoNameWithExtension);
FFMpegWrappers.ExtractFrames(DirectoryConstants.ffmpeg, outputDirectory + "\\" + videoNameWithExtension, outputDirectory, videoName, DateTime.Now, outputFPS);
File.Delete(outputDirectory + "\\" + videoNameWithExtension);
List <string> chunkFolders = TrackingDataPreprocessor.GroupFramesIntoChunks(outputDirectory, noFramePerChunk);
//parse VIRAT annotation file
string annotationNameWithExtension = URIUtilities.filenameFromURI(annotationURL);
client.DownloadFile(annotationURL, outputDirectory + "\\" + annotationNameWithExtension);
parseAnnotationFileIntoChunkFolders(chunkFolders,
outputDirectory + "\\" + annotationNameWithExtension,
noFramePerChunk, outputFPS);
//upload
for (int i = 0; i < chunkFolders.Count; i++)
{
string subDir = chunkFolders[i];
satyamStorage.uploadALocalFolder(subDir, satyamContainerName, satyamDirectoryName + "/Video_" + videoName + "_startingFrame_" + noFramePerChunk * i);
}
Directory.Delete(outputDirectory, true);
}
return(true);
}
public static bool ProcessAndUploadToAzureBlob(SatyamJobSubmissionsTableAccessEntry jobEntry)
{
// if the input is a folder of folders of frames, then copy directly
SatyamJobStorageAccountAccess satyamStorage = new SatyamJobStorageAccountAccess();
string satyamContainerName = SatyamTaskGenerator.JobTemplateToSatyamContainerNameMap[jobEntry.JobTemplateType];
string GUID = jobEntry.JobGUID;
string satyamDirectoryName = GUID;
SatyamJob job = JSonUtils.ConvertJSonToObject <SatyamJob>(jobEntry.JobParametersString);
MultiObjectTrackingSubmittedJob jobParams = JSonUtils.ConvertJSonToObject <MultiObjectTrackingSubmittedJob>(job.JobParameters);
BlobContainerManager bcm = new BlobContainerManager();
string status = bcm.Connect(job.azureInformation.AzureBlobStorageConnectionString);
List <string> FileTypes = SatyamTaskGenerator.ValidFileTypesByTemplate[job.JobTemplateType];
if (status != "SUCCESS")
{
return(false);
}
string guidFolder = DirectoryConstants.defaultTempDirectory + "\\" + GUID;
Directory.CreateDirectory(guidFolder);
int chunkLength = jobParams.ChunkDuration; // sec
int outputFPS = jobParams.FrameRate;
double chunkOverlap = jobParams.ChunkOverlap; // sec
var client = new WebClient();
if (jobParams.DataSrcFormat == DataFormat.Video)
{
// sample to frames
int noFramePerChunk = (int)(chunkLength * outputFPS);
int noFrameOverlap = (int)(chunkOverlap * outputFPS);
List <string> videoUrls = bcm.getURLList(job.azureInformation.AzureBlobStorageContainerName, job.azureInformation.AzureBlobStorageContainerDirectoryName);
foreach (string video in videoUrls)
{
string videoName = URIUtilities.filenameFromURINoExtension(video);
string videonameExtension = URIUtilities.filenameFromURI(video);
string outputDirectory = guidFolder + "\\" + videoName;
Directory.CreateDirectory(outputDirectory);
client.DownloadFile(video, outputDirectory + "\\" + videonameExtension);
FFMpegWrappers.ExtractFrames(DirectoryConstants.ffmpeg, outputDirectory + "\\" + videonameExtension, outputDirectory, videoName, DateTime.Now, outputFPS);
Console.WriteLine("deleting downloaded file...");
File.Delete(outputDirectory + "\\" + videonameExtension);
GroupFramesIntoChunksAndUploadChunks(videoName, outputDirectory, noFramePerChunk, noFrameOverlap, satyamContainerName, satyamDirectoryName);
Directory.Delete(outputDirectory, true);
}
}
if (jobParams.DataSrcFormat == DataFormat.VideoFrame)
{
int noFramePerChunk = (int)(chunkLength * outputFPS);//just use one fps for now, assume input should've already downsampled
int noFrameOverlap = (int)(chunkOverlap * outputFPS);
// chunk according to parameters
List <string> frameUrls = bcm.getURLList(job.azureInformation.AzureBlobStorageContainerName, job.azureInformation.AzureBlobStorageContainerDirectoryName);
Dictionary <string, List <string> > framesPerVideo = new Dictionary <string, List <string> >();
foreach (string url in frameUrls)
{
// assumed hierarchy: blob/directory.../videoname/frameid.jpg
//string frameName = URIUtilities.filenameFromURINoExtension(url);
string[] urlparts = url.Split('/');
string videoName = urlparts[urlparts.Length - 2];
if (!framesPerVideo.ContainsKey(videoName))
{
framesPerVideo.Add(videoName, new List <string>());
}
framesPerVideo[videoName].Add(url);
}
foreach (string video in framesPerVideo.Keys)
{
string outputDirectory = guidFolder + "\\" + video;
Directory.CreateDirectory(outputDirectory);
foreach (string frameURL in framesPerVideo[video])
{
string frameName = URIUtilities.filenameFromURI(frameURL);
client.DownloadFile(frameURL, outputDirectory + "\\" + frameName);
}
GroupFramesIntoChunksAndUploadChunks(video, outputDirectory, noFramePerChunk, noFrameOverlap, satyamContainerName, satyamDirectoryName);
Directory.Delete(outputDirectory, true);
}
}
return(true);
}
