public Logger(SecureString blobConnectionString, string containerName) { this.blobManager = new BlobContainerManager( blobConnectionString, containerName); this.fileName = Logger.GetLogFileName(); this.blobManager.CreateAppendBlob(Path.Combine(FolderName, fileName)); }
public static void SaveByGUIDRequester(string guid) { //get all the results of a GUID SatyamResultsTableAccess resultsDB = new SatyamResultsTableAccess(); List <SatyamResultsTableEntry> results = resultsDB.getEntriesByGUID(guid); if (results.Count == 0) { resultsDB.close(); return; } //get the azure info SatyamResult result = JSonUtils.ConvertJSonToObject <SatyamResult>(results[0].ResultString); SatyamTask task = JSonUtils.ConvertJSonToObject <SatyamTask>(result.TaskParametersString); SatyamJob job = task.jobEntry; string ConnectionString = job.azureInformation.AzureBlobStorageConnectionString; string ContainerName = job.azureInformation.AzureBlobStorageContainerName; string DirectoryName = job.azureInformation.AzureBlobStorageContainerDirectoryName; //create the data to be saved List <SatyamResultSaveDataRequester> savingDataList = new List <SatyamResultSaveDataRequester>(); StringBuilder s = new StringBuilder(); for (int i = 0; i < results.Count; i++) { SatyamResultSaveDataRequester data = new SatyamResultSaveDataRequester(results[i]); String jsonString = JSonUtils.ConvertObjectToJSon(data); s.Append(jsonString); if (i == results.Count - 1) { s.Append("\n"); } } string dataToBeSaved = s.ToString(); BlobContainerManager bcm = new BlobContainerManager(); bcm.Connect(ConnectionString); string FileName = "Results-" + results[0].JobGUID + ".txt"; bcm.SaveATextFile(ContainerName, DirectoryName, FileName, dataToBeSaved); resultsDB.close(); }
public static bool copyDataFromUserBlobToSatyamBlob(SatyamJobSubmissionsTableAccessEntry jobEntry) { string satyamContainerName = SatyamTaskGenerator.JobTemplateToSatyamContainerNameMap[jobEntry.JobTemplateType]; string GUID = jobEntry.JobGUID; string satyamDirectoryName = GUID; SatyamJob job = JSonUtils.ConvertJSonToObject <SatyamJob>(jobEntry.JobParametersString); BlobContainerManager bcm = new BlobContainerManager(); string status = bcm.Connect(job.azureInformation.AzureBlobStorageConnectionString); List <string> FileTypes = SatyamTaskGenerator.ValidFileTypesByTemplate[job.JobTemplateType]; if (status != "SUCCESS") { return(false); } SatyamJobStorageAccountAccess satyamStorage = new SatyamJobStorageAccountAccess(); satyamStorage.copyFilesFromAnotherAzureBlob(bcm, job.azureInformation.AzureBlobStorageContainerName, job.azureInformation.AzureBlobStorageContainerDirectoryName, satyamContainerName, satyamDirectoryName, FileTypes); return(true); }
public int getNoFiles() { int noFiles = -1; BlobContainerManager bcm = new BlobContainerManager(); string status = bcm.Connect(ConnectionString); if (status == "SUCCESS") { List <string> names = new List <string>(); if (DirectoryName != "" && DirectoryName != null) { names = bcm.getURLList(ContainerName, DirectoryName); } else { names = bcm.getURLList(ContainerName); } if (names != null) { noFiles = names.Count; } } return(noFiles); }
public static bool ProcessAndUploadToAzureBlob(SatyamJobSubmissionsTableAccessEntry jobEntry) { // Assumpltion: the input is a folder videos, convert to mp4, and upload to satyam blob SatyamJobStorageAccountAccess satyamStorage = new SatyamJobStorageAccountAccess(); string satyamContainerName = SatyamTaskGenerator.JobTemplateToSatyamContainerNameMap[jobEntry.JobTemplateType]; string GUID = jobEntry.JobGUID; string satyamDirectoryName = GUID; SatyamJob job = JSonUtils.ConvertJSonToObject <SatyamJob>(jobEntry.JobParametersString); SingleObjectLabelingSubmittedJob jobParams = JSonUtils.ConvertJSonToObject <SingleObjectLabelingSubmittedJob>(job.JobParameters); BlobContainerManager bcm = new BlobContainerManager(); string status = bcm.Connect(job.azureInformation.AzureBlobStorageConnectionString); List <string> FileTypes = SatyamTaskGenerator.ValidFileTypesByTemplate[job.JobTemplateType]; if (status != "SUCCESS") { return(false); } string guidFolder = DirectoryConstants.defaultTempDirectory + "\\" + GUID; Directory.CreateDirectory(guidFolder); //var client = new WebClient(); List <string> videoUrls = bcm.getURLList(job.azureInformation.AzureBlobStorageContainerName, job.azureInformation.AzureBlobStorageContainerDirectoryName); foreach (string video in videoUrls) { FFMpegWrappers.ConvertVideoURLToMP4(DirectoryConstants.ffmpeg, video, guidFolder); } satyamStorage.uploadALocalFolder(guidFolder, satyamContainerName, satyamDirectoryName); return(true); }
public static List <SatyamTask> generateDebug(SatyamJobSubmissionsTableAccessEntry jobEntry, string debugString, out string newDebugString) { newDebugString = debugString + "In Generate \n"; BlobContainerManager containerManager = new BlobContainerManager(); newDebugString += "About to connect to Satayam\n"; string statusString = containerManager.ConnectDebug(SatyamJobStorageAccountAccess.connection_string); newDebugString += "The connetion status is : " + statusString; return(null); //SatyamJobStorageAccountAccess satyamStorage = new SatyamJobStorageAccountAccess(); //newDebugString += "Got a new storage account\n"; //return null; //string satyamContainerName = JobTemplateToSatyamContainerNameMap[jobEntry.JobTemplateType]; //string GUID = jobEntry.JobGUID; //string satyamDirectoryName = GUID; //newDebugString += "About to get object from JSon\n"; //return null; //SatyamJob job = JSonUtils.ConvertJSonToObject<SatyamJob>(jobEntry.JobParametersString); //newDebugString += "Got the job from JSon\n"; //return null; //BlobContainerManager bcm = new BlobContainerManager(); //string status = bcm.Connect(job.azureInformation.AzureBlobStorageConnectionString); //if (status != "SUCCESS") //{ // return null; //} ////first copy all the images to satyam location //List<string> FileTypes = ValidFileTypesByTemplate[job.JobTemplateType]; //satyamStorage.copyFilesFromAnotherAzureBlob(bcm, job.azureInformation.AzureBlobStorageContainerName, job.azureInformation.AzureBlobStorageContainerDirectoryName, satyamContainerName, satyamDirectoryName, FileTypes); ////Now get all the URIs at the remote location //List<string> FullURIList = bcm.getURLList(job.azureInformation.AzureBlobStorageContainerName, job.azureInformation.AzureBlobStorageContainerDirectoryName); //List<string> URIList = new List<string>(); //foreach (string uri in FullURIList) //{ // string filename = URIUtilities.filenameFromURI(uri); // if (CheckFileExtensions.IsAllowedType(filename, FileTypes)) // { // URIList.Add(uri); // } //} //List<string> satyamURIList = satyamStorage.getURLList(satyamContainerName, satyamDirectoryName); ////Now create a dictionary mapping filename to URI for both the locations //Dictionary<string, string> fileNameToURI = new Dictionary<string, string>(); //Dictionary<string, string> satyamFileNameToURI = new Dictionary<string, string>(); //for (int i = 0; i < URIList.Count; i++) //{ // string fileName = URIUtilities.filenameFromURI(URIList[i]); // string satyamFileName = URIUtilities.filenameFromURI(satyamURIList[i]); // fileNameToURI.Add(fileName, URIList[i]); // satyamFileNameToURI.Add(satyamFileName, satyamURIList[i]); //} //List<string> fileNameList = fileNameToURI.Keys.ToList(); ////now create a task for each of these one by one //List<SatyamTask> tasks = new List<SatyamTask>(); //foreach (string filename in fileNameList) //{ // SatyamTask task = new SatyamTask(); // task.jobEntry = job; // //task.OriginalURI = fileNameToURI[filename]; // task.SatyamURI = satyamFileNameToURI[filename]; // tasks.Add(task); //} //return tasks; }
/// <summary> /// Input should be a folder of videos and corresponding annotation file with the same name /// </summary> /// <param name="jobEntry"></param> public static bool ProcessAndUploadToAzureBlob(SatyamJobSubmissionsTableAccessEntry jobEntry) { // if the input is a folder of folders of frames, then copy directly SatyamJobStorageAccountAccess satyamStorage = new SatyamJobStorageAccountAccess(); string satyamContainerName = SatyamTaskGenerator.JobTemplateToSatyamContainerNameMap[jobEntry.JobTemplateType]; string GUID = jobEntry.JobGUID; string satyamDirectoryName = GUID; SatyamJob job = JSonUtils.ConvertJSonToObject <SatyamJob>(jobEntry.JobParametersString); MultiObjectTrackingSubmittedJob jobParams = JSonUtils.ConvertJSonToObject <MultiObjectTrackingSubmittedJob>(job.JobParameters); BlobContainerManager bcm = new BlobContainerManager(); string status = bcm.Connect(job.azureInformation.AzureBlobStorageConnectionString); List <string> FileTypes = SatyamTaskGenerator.ValidFileTypesByTemplate[job.JobTemplateType]; if (status != "SUCCESS") { return(false); } string guidFolder = DirectoryConstants.defaultTempDirectory + "\\" + GUID; Directory.CreateDirectory(guidFolder); int chunkLength = jobParams.ChunkDuration; // sec int outputFPS = jobParams.FrameRate; double chunkOverlap = jobParams.ChunkOverlap; // sec var client = new WebClient(); // sample to frames int noFramePerChunk = (int)(chunkLength * outputFPS); int noFrameOverlap = (int)(chunkOverlap * outputFPS); List <string> videoUrls = bcm.getURLList(job.azureInformation.AzureBlobStorageContainerName, job.azureInformation.AzureBlobStorageContainerDirectoryName); Dictionary <string, List <string> > videos = new Dictionary <string, List <string> >(); foreach (string videolink in videoUrls) { string videoName = URIUtilities.filenameFromURINoExtension(videolink); if (!videos.ContainsKey(videoName)) { videos.Add(videoName, new List <string>()); } videos[videoName].Add(videolink); } foreach (string videoName in videos.Keys) { // filter out those that doesn't provide a annotation file with it.... if (videos[videoName].Count != 2) { Console.WriteLine("Warning: Not 2 files provided for {0}.", videoName); //Directory.Delete(guidFolder, true); //return false; continue; } string videoURL = ""; string annotationURL = ""; foreach (string fileLink in videos[videoName]) { string extension = URIUtilities.fileExtensionFromURI(fileLink); if (extension != "txt") { videoURL = fileLink; } else { annotationURL = fileLink; } } string outputDirectory = guidFolder + "\\" + videoName; Directory.CreateDirectory(outputDirectory); string videoNameWithExtension = URIUtilities.filenameFromURI(videoURL); client.DownloadFile(videoURL, outputDirectory + "\\" + videoNameWithExtension); FFMpegWrappers.ExtractFrames(DirectoryConstants.ffmpeg, outputDirectory + "\\" + videoNameWithExtension, outputDirectory, videoName, DateTime.Now, outputFPS); File.Delete(outputDirectory + "\\" + videoNameWithExtension); List <string> chunkFolders = TrackingDataPreprocessor.GroupFramesIntoChunks(outputDirectory, noFramePerChunk); //parse VIRAT annotation file string annotationNameWithExtension = URIUtilities.filenameFromURI(annotationURL); client.DownloadFile(annotationURL, outputDirectory + "\\" + annotationNameWithExtension); parseAnnotationFileIntoChunkFolders(chunkFolders, outputDirectory + "\\" + annotationNameWithExtension, noFramePerChunk, outputFPS); //upload for (int i = 0; i < chunkFolders.Count; i++) { string subDir = chunkFolders[i]; satyamStorage.uploadALocalFolder(subDir, satyamContainerName, satyamDirectoryName + "/Video_" + videoName + "_startingFrame_" + noFramePerChunk * i); } Directory.Delete(outputDirectory, true); } return(true); }
public static bool ProcessAndUploadToAzureBlob(SatyamJobSubmissionsTableAccessEntry jobEntry) { // if the input is a folder of folders of frames, then copy directly SatyamJobStorageAccountAccess satyamStorage = new SatyamJobStorageAccountAccess(); string satyamContainerName = SatyamTaskGenerator.JobTemplateToSatyamContainerNameMap[jobEntry.JobTemplateType]; string GUID = jobEntry.JobGUID; string satyamDirectoryName = GUID; SatyamJob job = JSonUtils.ConvertJSonToObject <SatyamJob>(jobEntry.JobParametersString); MultiObjectTrackingSubmittedJob jobParams = JSonUtils.ConvertJSonToObject <MultiObjectTrackingSubmittedJob>(job.JobParameters); BlobContainerManager bcm = new BlobContainerManager(); string status = bcm.Connect(job.azureInformation.AzureBlobStorageConnectionString); List <string> FileTypes = SatyamTaskGenerator.ValidFileTypesByTemplate[job.JobTemplateType]; if (status != "SUCCESS") { return(false); } string guidFolder = DirectoryConstants.defaultTempDirectory + "\\" + GUID; Directory.CreateDirectory(guidFolder); int chunkLength = jobParams.ChunkDuration; // sec int outputFPS = jobParams.FrameRate; double chunkOverlap = jobParams.ChunkOverlap; // sec var client = new WebClient(); if (jobParams.DataSrcFormat == DataFormat.Video) { // sample to frames int noFramePerChunk = (int)(chunkLength * outputFPS); int noFrameOverlap = (int)(chunkOverlap * outputFPS); List <string> videoUrls = bcm.getURLList(job.azureInformation.AzureBlobStorageContainerName, job.azureInformation.AzureBlobStorageContainerDirectoryName); foreach (string video in videoUrls) { string videoName = URIUtilities.filenameFromURINoExtension(video); string videonameExtension = URIUtilities.filenameFromURI(video); string outputDirectory = guidFolder + "\\" + videoName; Directory.CreateDirectory(outputDirectory); client.DownloadFile(video, outputDirectory + "\\" + videonameExtension); FFMpegWrappers.ExtractFrames(DirectoryConstants.ffmpeg, outputDirectory + "\\" + videonameExtension, outputDirectory, videoName, DateTime.Now, outputFPS); Console.WriteLine("deleting downloaded file..."); File.Delete(outputDirectory + "\\" + videonameExtension); GroupFramesIntoChunksAndUploadChunks(videoName, outputDirectory, noFramePerChunk, noFrameOverlap, satyamContainerName, satyamDirectoryName); Directory.Delete(outputDirectory, true); } } if (jobParams.DataSrcFormat == DataFormat.VideoFrame) { int noFramePerChunk = (int)(chunkLength * outputFPS);//just use one fps for now, assume input should've already downsampled int noFrameOverlap = (int)(chunkOverlap * outputFPS); // chunk according to parameters List <string> frameUrls = bcm.getURLList(job.azureInformation.AzureBlobStorageContainerName, job.azureInformation.AzureBlobStorageContainerDirectoryName); Dictionary <string, List <string> > framesPerVideo = new Dictionary <string, List <string> >(); foreach (string url in frameUrls) { // assumed hierarchy: blob/directory.../videoname/frameid.jpg //string frameName = URIUtilities.filenameFromURINoExtension(url); string[] urlparts = url.Split('/'); string videoName = urlparts[urlparts.Length - 2]; if (!framesPerVideo.ContainsKey(videoName)) { framesPerVideo.Add(videoName, new List <string>()); } framesPerVideo[videoName].Add(url); } foreach (string video in framesPerVideo.Keys) { string outputDirectory = guidFolder + "\\" + video; Directory.CreateDirectory(outputDirectory); foreach (string frameURL in framesPerVideo[video]) { string frameName = URIUtilities.filenameFromURI(frameURL); client.DownloadFile(frameURL, outputDirectory + "\\" + frameName); } GroupFramesIntoChunksAndUploadChunks(video, outputDirectory, noFramePerChunk, noFrameOverlap, satyamContainerName, satyamDirectoryName); Directory.Delete(outputDirectory, true); } } return(true); }
public FileConfigs(SecureString blobConnectionString) { this.blobManager = new BlobContainerManager(blobConnectionString, BlobContainerName); }