// This is the part that is executed by Azure Batch VMs in the cloud. // From Azure Batch PoV, it's just an executable that runs for a while. // Step 1. Call "convert.exe" to process an image. // Step 2. Update result into blob. // Note there is no explicit download step since it's handled by Task scheduler public static void TaskMain(string[] args) { if (args == null || args.Length != 3) { throw new Exception("Usage: ImgProc.exe --Task <outputContainerSAS> <OutputFilenamePrefix>"); } Console.WriteLine("Start ImgProcTask."); string outputContainerSAS = args[1]; string prefix = args[2]; //Convert all the jpg files to thumbnails by launching imagemick process GenerateThumbnailImages(prefix); string[] outputFiles = Directory.GetFiles(Directory.GetCurrentDirectory(), prefix + "*"); if (outputFiles == null || outputFiles.Length == 0) { throw new Exception("Thumbnail files not generated"); } //Store output to blob foreach (string outputFile in outputFiles) { int index = outputFile.LastIndexOf("thumb"); string blobName = outputFile.Substring(index); ImgProcUtils.UploadFileToBlob(blobName, outputContainerSAS); } Console.WriteLine("Done."); }
/// <summary> /// This is the client that creates workitem and submits tasks. /// </summary> /// <param name="args"></param> public static void SubmitTasks(string[] args) { config = Config.ParseConfig(); //Upload resources if specified if (config.UploadResources) { //Upload ImgProc.exe, Batch.dll and the Storage Client ImgProcUtils.UploadFileToBlob(Constants.StorageClientDllName, config.ResourceContainerSAS); ImgProcUtils.UploadFileToBlob(Constants.ImgProcExeName, config.ResourceContainerSAS); ImgProcUtils.UploadFileToBlob(Constants.BatchClientDllName, config.ResourceContainerSAS); Console.WriteLine("Done uploading files to blob"); } try { using (IWorkItemManager wm = config.Client.OpenWorkItemManager()) { IToolbox toolbox = config.Client.OpenToolbox(); //Use the task submission helper to ease creation of workitem and addition of tasks, as well as management of resource file staging. ITaskSubmissionHelper taskSubmissionHelper = toolbox.CreateTaskSubmissionHelper(wm, config.PoolName); taskSubmissionHelper.WorkItemName = config.WorkitemName; //Compute the number of images each task should process int numImgsPerTask = (int)Math.Round(config.NumInputBlobs / (decimal)config.NumTasks); for (int i = 0; i < config.NumTasks; i++) { ICloudTask task = new CloudTask( name: "task_no_" + i, commandline: string.Format("{0} --Task {1} thumb{2}", Constants.ImgProcExeName, config.OutputContainerSAS, i)); Console.WriteLine("Generating task: {0}", task.Name); task.FilesToStage = new List <IFileStagingProvider>(); int start = i * numImgsPerTask; int end; if (i < config.NumTasks - 1) { end = ((i + 1) * numImgsPerTask) - 1; } else { end = config.NumInputBlobs - 1; } //Generate and set up the list of files to be processed by this task for (int j = start; j < end; j++) { string input = GetTempFilePath(j); ImgProcUtils.GenerateImages(input, string.Format("{0}", j)); task.FilesToStage.Add(new FileToStage(input, new StagingStorageAccount(config.StorageAccount, config.StorageKey, config.StorageBlobEndpoint))); } task.ResourceFiles = ImgProcUtils.GetResourceFiles(config.ResourceContainerSAS); taskSubmissionHelper.AddTask(task); } IJobCommitUnboundArtifacts artifacts = null; try { Console.WriteLine("Submitting {0} tasks to the Batch Service", config.NumTasks); //Submit the tasks to the Batch Service artifacts = taskSubmissionHelper.Commit() as IJobCommitUnboundArtifacts; } catch (AggregateException ae) { // Go through all exceptions and dump useful information ae.Handle(x => { if (x is BatchException) { BatchException be = x as BatchException; if (null != be.RequestInformation && null != be.RequestInformation.AzureError) { // Write the server side error information Console.Error.WriteLine(be.RequestInformation.AzureError.Code); Console.Error.WriteLine(be.RequestInformation.AzureError.Message.Value); if (null != be.RequestInformation.AzureError.Values) { foreach (var v in be.RequestInformation.AzureError.Values) { Console.Error.WriteLine(v.Key + " : " + v.Value); } } } } // Indicate that the error has been handled return(true); }); } DateTime starttime = DateTime.Now; //Wait for the job to complete if (config.WaitForCompletion) { ICloudJob job = wm.GetJob(artifacts.WorkItemName, artifacts.JobName); Console.WriteLine("Waiting for tasks to complete..."); // Wait up to 15 minutes for all tasks to reach the completed state config.Client.OpenToolbox().CreateTaskStateMonitor().WaitAll(job.ListTasks(), TaskState.Completed, new TimeSpan(0, 15, 0)); DateTime endtime = DateTime.Now; Console.WriteLine("Time taken for processing the images : {0} sec", endtime.Subtract(starttime).TotalSeconds); } } } finally { //Delete the workitem that we created if (config.DeleteWorkitem && config.WaitForCompletion) { Console.WriteLine("Press any key to delete the workitem . . ."); Console.ReadKey(); config.Client.OpenWorkItemManager().DeleteWorkItem(config.WorkitemName); } } }