// This is the part that is executed by Azure Batch VMs in the cloud.
// From Azure Batch PoV, it's just an executable that runs for a while.
// Step 1. Call "convert.exe" to process an image.
// Step 2. Update result into blob.
// Note there is no explicit download step since it's handled by Task scheduler
public static void TaskMain(string[] args)
{
if (args == null || args.Length != 3)
{
throw new Exception("Usage: ImgProc.exe --Task <outputContainerSAS> <OutputFilenamePrefix>");
}
Console.WriteLine("Start ImgProcTask.");
string outputContainerSAS = args[1];
string prefix = args[2];
//Convert all the jpg files to thumbnails by launching imagemick process
GenerateThumbnailImages(prefix);
string[] outputFiles = Directory.GetFiles(Directory.GetCurrentDirectory(), prefix + "*");
if (outputFiles == null ||
outputFiles.Length == 0)
{
throw new Exception("Thumbnail files not generated");
}
//Store output to blob
foreach (string outputFile in outputFiles)
{
int index = outputFile.LastIndexOf("thumb");
string blobName = outputFile.Substring(index);
ImgProcUtils.UploadFileToBlob(blobName, outputContainerSAS);
}
Console.WriteLine("Done.");
}
/// <summary>
/// Read the configuration object in from the App.Config
/// </summary>
/// <returns></returns>
public static Config ParseConfig()
{
var config = new Config();
config.BatchServiceUrl = GetConfigParam("BatchServiceUrl");
config.AccountName = GetConfigParam("Account");
config.Key = GetConfigParam("Key");
config.Client = BatchClient.Connect(config.BatchServiceUrl, new BatchCredentials(config.AccountName, config.Key));
config.StorageAccount = GetConfigParam("StorageAccount");
config.StorageKey = GetConfigParam("StorageKey");
config.StorageBlobEndpoint = "https://" + config.StorageAccount + ".blob.core.windows.net";
config.NumTasks = Int32.Parse(GetConfigParam("NumTasks"));
config.NumTvms = Int32.Parse(GetConfigParam("NumTvms"));
config.InputDataContainerSAS = GetConfigParam("InputDataContainerSAS");
config.InputBlobPrefix = GetConfigParam("InputBlobPrefix");
config.NumInputBlobs = Int32.Parse(GetConfigParam("NumInputBlobs"));
config.OutputContainerSAS = GetConfigParam("OutputContainerSAS");
config.WorkitemName = "ImgProcWi" + DateTime.Now.ToString("_yyMMdd_HHmmss_") + Guid.NewGuid().ToString("N");
config.ResourceContainerSAS = GetConfigParam("ResourcesSAS");
config.ImageMagickExeSAS = GetConfigParam("ImageMagickExeSAS");
config.WaitForCompletion = bool.Parse(GetConfigParam("WaitForCompletion"));
config.WaitForPool = bool.Parse(GetConfigParam("WaitForPool"));
config.DeleteWorkitem = bool.Parse(GetConfigParam("DeleteWorkitem"));
config.CreatePool = bool.Parse(GetConfigParam("CreatePool"));
config.PoolName = GetConfigParam("PoolName");
if (config.CreatePool)
{
if (String.IsNullOrEmpty(config.PoolName))
{
config.PoolName = "ImgProcPool" + Guid.NewGuid().ToString("N");
}
}
else
{
if (String.IsNullOrEmpty(config.PoolName))
{
throw new Exception("Provide pool name as CreatePool is false");
}
}
config.DeletePool = bool.Parse(GetConfigParam("DeletePool"));
config.InitializeStorageContainerSAS = bool.Parse(GetConfigParam("InitStorage"));
if (config.InitializeStorageContainerSAS)
{
string account = GetConfigParam("StorageAccount");
string key = GetConfigParam("StorageKey");
config.InputDataContainerSAS = ImgProcUtils.CreateContainerWithPolicySASIfNotExist(
account,
key,
"watask-input",
"readandlist",
DateTime.Now,
DateTime.Now.AddMonths(12),
SharedAccessBlobPermissions.Read | SharedAccessBlobPermissions.List);
config.OutputContainerSAS = ImgProcUtils.CreateContainerWithPolicySASIfNotExist(
account,
key,
"watask-output",
"readandwrite",
DateTime.Now,
DateTime.Now.AddMonths(12),
SharedAccessBlobPermissions.Read | SharedAccessBlobPermissions.Write);
config.ResourceContainerSAS = ImgProcUtils.CreateContainerWithPolicySASIfNotExist(
account,
key,
"watask-resource",
"readandwrite",
DateTime.Now,
DateTime.Now.AddMonths(12),
SharedAccessBlobPermissions.Read | SharedAccessBlobPermissions.Write);
}
config.UploadResources = bool.Parse(GetConfigParam("UploadResources"));
return(config);
}
/// <summary>
/// This is the client that creates workitem and submits tasks.
/// </summary>
/// <param name="args"></param>
public static void SubmitTasks(string[] args)
{
config = Config.ParseConfig();
//Upload resources if specified
if (config.UploadResources)
{
//Upload ImgProc.exe, Batch.dll and the Storage Client
ImgProcUtils.UploadFileToBlob(Constants.StorageClientDllName, config.ResourceContainerSAS);
ImgProcUtils.UploadFileToBlob(Constants.ImgProcExeName, config.ResourceContainerSAS);
ImgProcUtils.UploadFileToBlob(Constants.BatchClientDllName, config.ResourceContainerSAS);
Console.WriteLine("Done uploading files to blob");
}
try
{
using (IWorkItemManager wm = config.Client.OpenWorkItemManager())
{
IToolbox toolbox = config.Client.OpenToolbox();
//Use the task submission helper to ease creation of workitem and addition of tasks, as well as management of resource file staging.
ITaskSubmissionHelper taskSubmissionHelper = toolbox.CreateTaskSubmissionHelper(wm, config.PoolName);
taskSubmissionHelper.WorkItemName = config.WorkitemName;
//Compute the number of images each task should process
int numImgsPerTask = (int)Math.Round(config.NumInputBlobs / (decimal)config.NumTasks);
for (int i = 0; i < config.NumTasks; i++)
{
ICloudTask task = new CloudTask(
name: "task_no_" + i,
commandline: string.Format("{0} --Task {1} thumb{2}", Constants.ImgProcExeName, config.OutputContainerSAS, i));
Console.WriteLine("Generating task: {0}", task.Name);
task.FilesToStage = new List <IFileStagingProvider>();
int start = i * numImgsPerTask;
int end;
if (i < config.NumTasks - 1)
{
end = ((i + 1) * numImgsPerTask) - 1;
}
else
{
end = config.NumInputBlobs - 1;
}
//Generate and set up the list of files to be processed by this task
for (int j = start; j < end; j++)
{
string input = GetTempFilePath(j);
ImgProcUtils.GenerateImages(input, string.Format("{0}", j));
task.FilesToStage.Add(new FileToStage(input, new StagingStorageAccount(config.StorageAccount, config.StorageKey, config.StorageBlobEndpoint)));
}
task.ResourceFiles = ImgProcUtils.GetResourceFiles(config.ResourceContainerSAS);
taskSubmissionHelper.AddTask(task);
}
IJobCommitUnboundArtifacts artifacts = null;
try
{
Console.WriteLine("Submitting {0} tasks to the Batch Service", config.NumTasks);
//Submit the tasks to the Batch Service
artifacts = taskSubmissionHelper.Commit() as IJobCommitUnboundArtifacts;
}
catch (AggregateException ae)
{
// Go through all exceptions and dump useful information
ae.Handle(x =>
{
if (x is BatchException)
{
BatchException be = x as BatchException;
if (null != be.RequestInformation && null != be.RequestInformation.AzureError)
{
// Write the server side error information
Console.Error.WriteLine(be.RequestInformation.AzureError.Code);
Console.Error.WriteLine(be.RequestInformation.AzureError.Message.Value);
if (null != be.RequestInformation.AzureError.Values)
{
foreach (var v in be.RequestInformation.AzureError.Values)
{
Console.Error.WriteLine(v.Key + " : " + v.Value);
}
}
}
}
// Indicate that the error has been handled
return(true);
});
}
DateTime starttime = DateTime.Now;
//Wait for the job to complete
if (config.WaitForCompletion)
{
ICloudJob job = wm.GetJob(artifacts.WorkItemName, artifacts.JobName);
Console.WriteLine("Waiting for tasks to complete...");
// Wait up to 15 minutes for all tasks to reach the completed state
config.Client.OpenToolbox().CreateTaskStateMonitor().WaitAll(job.ListTasks(), TaskState.Completed, new TimeSpan(0, 15, 0));
DateTime endtime = DateTime.Now;
Console.WriteLine("Time taken for processing the images : {0} sec", endtime.Subtract(starttime).TotalSeconds);
}
}
}
finally
{
//Delete the workitem that we created
if (config.DeleteWorkitem &&
config.WaitForCompletion)
{
Console.WriteLine("Press any key to delete the workitem . . .");
Console.ReadKey();
config.Client.OpenWorkItemManager().DeleteWorkItem(config.WorkitemName);
}
}
}
