private static void AddWork(IBatchClient client)
{
using (IWorkItemManager wm = client.OpenWorkItemManager())
{
//The toolbox contains some helper mechanisms to ease submission and monitoring of tasks.
IToolbox toolbox = client.OpenToolbox();
// to submit a batch of tasks, the TaskSubmissionHelper is useful.
ITaskSubmissionHelper taskSubmissionHelper = toolbox.CreateTaskSubmissionHelper(wm, Program.PoolName);
// workitem is uniquely identified by its name so we will use a timestamp as suffix
taskSubmissionHelper.WorkItemName = Environment.GetEnvironmentVariable("USERNAME") + DateTime.Now.ToString("yyyyMMdd-HHmmss");
Console.WriteLine("Creating work item: {0}", taskSubmissionHelper.WorkItemName);
// add 2 quick tasks. Tasks within a job must have unique names
taskSubmissionHelper.AddTask(new CloudTask("task1", "hostname"));
taskSubmissionHelper.AddTask(new CloudTask("task2", "cmd /c dir /s"));
//Commit the tasks to the Batch Service
IJobCommitUnboundArtifacts artifacts = taskSubmissionHelper.Commit() as IJobCommitUnboundArtifacts;
// TaskSubmissionHelper commit artifacts returns the workitem and job name
ICloudJob job = wm.GetJob(artifacts.WorkItemName, artifacts.JobName);
Console.WriteLine("Waiting for all tasks to complete on work item: {0}, Job: {1} ...", artifacts.WorkItemName, artifacts.JobName);
//We use the task state monitor to monitor the state of our tasks -- in this case we will wait for them all to complete.
ITaskStateMonitor taskStateMonitor = toolbox.CreateTaskStateMonitor();
// blocking wait on the list of tasks until all tasks reach completed state
bool timedOut = taskStateMonitor.WaitAll(job.ListTasks(), TaskState.Completed, new TimeSpan(0, 20, 0));
if (timedOut)
{
throw new TimeoutException("Timed out waiting for tasks");
}
// dump task output
foreach (var t in job.ListTasks())
{
Console.WriteLine("Task " + t.Name + " says:\n" + t.GetTaskFile(Constants.StandardOutFileName).ReadAsString());
}
// remember to delete the workitem before exiting
Console.WriteLine("Deleting work item: {0}", artifacts.WorkItemName);
wm.DeleteWorkItem(artifacts.WorkItemName);
}
}
/// <summary>
/// Submit a work item with tasks which have dependant files.
/// The files are automatically uploaded to Azure Storage using the FileStaging feature of the Azure.Batch client library.
/// </summary>
/// <param name="client"></param>
private static void AddWorkWithFileStaging(IBatchClient client)
{
using (IWorkItemManager wm = client.OpenWorkItemManager())
{
IToolbox toolbox = client.OpenToolbox();
ITaskSubmissionHelper taskSubmissionHelper = toolbox.CreateTaskSubmissionHelper(wm, Program.PoolName);
taskSubmissionHelper.WorkItemName = Environment.GetEnvironmentVariable("USERNAME") + DateTime.Now.ToString("yyyyMMdd-HHmmss");
Console.WriteLine("Creating work item: {0}", taskSubmissionHelper.WorkItemName);
ICloudTask taskToAdd1 = new CloudTask("task_with_file1", "cmd /c type *.txt");
ICloudTask taskToAdd2 = new CloudTask("task_with_file2", "cmd /c dir /s");
//Set up a collection of files to be staged -- these files will be uploaded to Azure Storage
//when the tasks are submitted to the Azure Batch service.
taskToAdd1.FilesToStage = new List <IFileStagingProvider>();
taskToAdd2.FilesToStage = new List <IFileStagingProvider>();
// generate a local file in temp directory
Process cur = Process.GetCurrentProcess();
string path = Path.Combine(Environment.GetEnvironmentVariable("TEMP"), cur.Id.ToString() + ".txt");
System.IO.File.WriteAllText(path, "hello from " + cur.Id.ToString());
// add file as task dependency so it'll be uploaded to storage before task
// is submitted and download onto the VM before task starts execution
FileToStage file = new FileToStage(path, new StagingStorageAccount(Program.StorageAccount, Program.StorageKey, Program.StorageBlobEndpoint));
taskToAdd1.FilesToStage.Add(file);
taskToAdd2.FilesToStage.Add(file); // filetostage object can be reused
taskSubmissionHelper.AddTask(taskToAdd1);
taskSubmissionHelper.AddTask(taskToAdd2);
IJobCommitUnboundArtifacts artifacts = null;
bool errors = false;
try
{
//Stage the files to Azure Storage and add the tasks to Azure Batch.
artifacts = taskSubmissionHelper.Commit() as IJobCommitUnboundArtifacts;
}
catch (AggregateException ae)
{
errors = true;
// Go through all exceptions and dump useful information
ae.Handle((x) =>
{
if (x is BatchException)
{
BatchException be = x as BatchException;
if (null != be.RequestInformation && null != be.RequestInformation.AzureError)
{
// Write the server side error information
Console.Error.WriteLine(be.RequestInformation.AzureError.Code);
Console.Error.WriteLine(be.RequestInformation.AzureError.Message.Value);
if (null != be.RequestInformation.AzureError.Values)
{
foreach (var v in be.RequestInformation.AzureError.Values)
{
Console.Error.WriteLine(v.Key + " : " + v.Value);
}
}
}
}
else
{
Console.WriteLine(x);
}
// Indicate that the error has been handled
return(true);
});
}
// if there is no exception, wait for job response
if (!errors)
{
List <ICloudTask> tasksToMonitorForCompletion = wm.ListTasks(artifacts.WorkItemName, artifacts.JobName).ToList();
Console.WriteLine("Waiting for all tasks to complete on work item: {0}, Job: {1} ...", artifacts.WorkItemName, artifacts.JobName);
client.OpenToolbox().CreateTaskStateMonitor().WaitAll(tasksToMonitorForCompletion, TaskState.Completed, TimeSpan.FromMinutes(30));
foreach (ICloudTask task in wm.ListTasks(artifacts.WorkItemName, artifacts.JobName))
{
Console.WriteLine("Task " + task.Name + " says:\n" + task.GetTaskFile(Constants.StandardOutFileName).ReadAsString());
Console.WriteLine(task.GetTaskFile(Constants.StandardErrorFileName).ReadAsString());
}
}
Console.WriteLine("Deleting work item: {0}", artifacts.WorkItemName);
wm.DeleteWorkItem(artifacts.WorkItemName); //Don't forget to delete the work item before you exit
}
}
/// <summary>
/// This is the client that creates workitem and submits tasks.
/// </summary>
/// <param name="args"></param>
public static void SubmitTasks(string[] args)
{
config = Config.ParseConfig();
//Upload resources if specified
if (config.UploadResources)
{
//Upload ImgProc.exe, Batch.dll and the Storage Client
ImgProcUtils.UploadFileToBlob(Constants.StorageClientDllName, config.ResourceContainerSAS);
ImgProcUtils.UploadFileToBlob(Constants.ImgProcExeName, config.ResourceContainerSAS);
ImgProcUtils.UploadFileToBlob(Constants.BatchClientDllName, config.ResourceContainerSAS);
Console.WriteLine("Done uploading files to blob");
}
try
{
using (IWorkItemManager wm = config.Client.OpenWorkItemManager())
{
IToolbox toolbox = config.Client.OpenToolbox();
//Use the task submission helper to ease creation of workitem and addition of tasks, as well as management of resource file staging.
ITaskSubmissionHelper taskSubmissionHelper = toolbox.CreateTaskSubmissionHelper(wm, config.PoolName);
taskSubmissionHelper.WorkItemName = config.WorkitemName;
//Compute the number of images each task should process
int numImgsPerTask = (int)Math.Round(config.NumInputBlobs / (decimal)config.NumTasks);
for (int i = 0; i < config.NumTasks; i++)
{
ICloudTask task = new CloudTask(
name: "task_no_" + i,
commandline: string.Format("{0} --Task {1} thumb{2}", Constants.ImgProcExeName, config.OutputContainerSAS, i));
Console.WriteLine("Generating task: {0}", task.Name);
task.FilesToStage = new List <IFileStagingProvider>();
int start = i * numImgsPerTask;
int end;
if (i < config.NumTasks - 1)
{
end = ((i + 1) * numImgsPerTask) - 1;
}
else
{
end = config.NumInputBlobs - 1;
}
//Generate and set up the list of files to be processed by this task
for (int j = start; j < end; j++)
{
string input = GetTempFilePath(j);
ImgProcUtils.GenerateImages(input, string.Format("{0}", j));
task.FilesToStage.Add(new FileToStage(input, new StagingStorageAccount(config.StorageAccount, config.StorageKey, config.StorageBlobEndpoint)));
}
task.ResourceFiles = ImgProcUtils.GetResourceFiles(config.ResourceContainerSAS);
taskSubmissionHelper.AddTask(task);
}
IJobCommitUnboundArtifacts artifacts = null;
try
{
Console.WriteLine("Submitting {0} tasks to the Batch Service", config.NumTasks);
//Submit the tasks to the Batch Service
artifacts = taskSubmissionHelper.Commit() as IJobCommitUnboundArtifacts;
}
catch (AggregateException ae)
{
// Go through all exceptions and dump useful information
ae.Handle(x =>
{
if (x is BatchException)
{
BatchException be = x as BatchException;
if (null != be.RequestInformation && null != be.RequestInformation.AzureError)
{
// Write the server side error information
Console.Error.WriteLine(be.RequestInformation.AzureError.Code);
Console.Error.WriteLine(be.RequestInformation.AzureError.Message.Value);
if (null != be.RequestInformation.AzureError.Values)
{
foreach (var v in be.RequestInformation.AzureError.Values)
{
Console.Error.WriteLine(v.Key + " : " + v.Value);
}
}
}
}
// Indicate that the error has been handled
return(true);
});
}
DateTime starttime = DateTime.Now;
//Wait for the job to complete
if (config.WaitForCompletion)
{
ICloudJob job = wm.GetJob(artifacts.WorkItemName, artifacts.JobName);
Console.WriteLine("Waiting for tasks to complete...");
// Wait up to 15 minutes for all tasks to reach the completed state
config.Client.OpenToolbox().CreateTaskStateMonitor().WaitAll(job.ListTasks(), TaskState.Completed, new TimeSpan(0, 15, 0));
DateTime endtime = DateTime.Now;
Console.WriteLine("Time taken for processing the images : {0} sec", endtime.Subtract(starttime).TotalSeconds);
}
}
}
finally
{
//Delete the workitem that we created
if (config.DeleteWorkitem &&
config.WaitForCompletion)
{
Console.WriteLine("Press any key to delete the workitem . . .");
Console.ReadKey();
config.Client.OpenWorkItemManager().DeleteWorkItem(config.WorkitemName);
}
}
}
public static void JobMain(string[] args)
{
//Load the configuration
TopNWordsConfiguration configuration = TopNWordsConfiguration.LoadConfigurationFromAppConfig();
StagingStorageAccount stagingStorageAccount = new StagingStorageAccount(
configuration.StorageAccountName,
configuration.StorageAccountKey,
configuration.StorageAccountBlobEndpoint);
IBatchClient client = BatchClient.Connect(configuration.BatchServiceUrl, new BatchCredentials(configuration.BatchAccountName, configuration.BatchAccountKey));
string stagingContainer = null;
//Create a pool (if user hasn't provided one)
if (configuration.ShouldCreatePool)
{
using (IPoolManager pm = client.OpenPoolManager())
{
//OSFamily 4 == OS 2012 R2
//You can learn more about os families and versions at:
//http://msdn.microsoft.com/en-us/library/azure/ee924680.aspx
ICloudPool pool = pm.CreatePool(configuration.PoolName, targetDedicated: configuration.PoolSize, osFamily: "4", vmSize: "small");
Console.WriteLine("Adding pool {0}", configuration.PoolName);
pool.Commit();
}
}
try
{
using (IWorkItemManager wm = client.OpenWorkItemManager())
{
IToolbox toolbox = client.OpenToolbox();
//Use the TaskSubmissionHelper to help us create a WorkItem and add tasks to it.
ITaskSubmissionHelper taskSubmissionHelper = toolbox.CreateTaskSubmissionHelper(wm, configuration.PoolName);
taskSubmissionHelper.WorkItemName = configuration.WorkItemName;
FileToStage topNWordExe = new FileToStage(TopNWordsExeName, stagingStorageAccount);
FileToStage storageDll = new FileToStage(StorageClientDllName, stagingStorageAccount);
string bookFileUri = UploadBookFileToCloudBlob(configuration, configuration.BookFileName);
Console.WriteLine("{0} uploaded to cloud", configuration.BookFileName);
for (int i = 1; i <= configuration.NumberOfTasks; i++)
{
ICloudTask task = new CloudTask("task_no_" + i, String.Format("{0} --Task {1} {2} {3} {4}",
TopNWordsExeName,
bookFileUri,
configuration.NumberOfTopWords,
configuration.StorageAccountName,
configuration.StorageAccountKey));
//This is the list of files to stage to a container -- for each TaskSubmissionHelper one container is created and
//files all resolve to Azure Blobs by their name (so two tasks with the same named file will create just 1 blob in
//the TaskSubmissionHelper's container).
task.FilesToStage = new List <IFileStagingProvider>
{
topNWordExe,
storageDll
};
taskSubmissionHelper.AddTask(task);
}
//Commit all the tasks to the Batch Service.
IJobCommitUnboundArtifacts artifacts = taskSubmissionHelper.Commit() as IJobCommitUnboundArtifacts;
foreach (var fileStagingArtifact in artifacts.FileStagingArtifacts)
{
SequentialFileStagingArtifact stagingArtifact = fileStagingArtifact.Value as SequentialFileStagingArtifact;
if (stagingArtifact != null)
{
stagingContainer = stagingArtifact.BlobContainerCreated;
Console.WriteLine("Uploaded files to container: {0} -- you will be charged for their storage unless you delete them.",
stagingArtifact.BlobContainerCreated);
}
}
//Get the job to monitor status.
ICloudJob job = wm.GetJob(artifacts.WorkItemName, artifacts.JobName);
Console.Write("Waiting for tasks to complete ...");
// Wait 1 minute for all tasks to reach the completed state
client.OpenToolbox().CreateTaskStateMonitor().WaitAll(job.ListTasks(), TaskState.Completed, TimeSpan.FromMinutes(20));
Console.WriteLine("Done.");
foreach (ICloudTask task in job.ListTasks())
{
Console.WriteLine("Task " + task.Name + " says:\n" + task.GetTaskFile(Constants.StandardOutFileName).ReadAsString());
Console.WriteLine(task.GetTaskFile(Constants.StandardErrorFileName).ReadAsString());
}
}
}
finally
{
//Delete the pool that we created
if (configuration.ShouldCreatePool)
{
using (IPoolManager pm = client.OpenPoolManager())
{
Console.WriteLine("Deleting pool: {0}", configuration.PoolName);
pm.DeletePool(configuration.PoolName);
}
}
//Delete the workitem that we created
if (configuration.ShouldDeleteWorkItem)
{
using (IWorkItemManager wm = client.OpenWorkItemManager())
{
Console.WriteLine("Deleting work item: {0}", configuration.WorkItemName);
wm.DeleteWorkItem(configuration.WorkItemName);
}
}
//Delete the containers we created
if (configuration.ShouldDeleteContainer)
{
DeleteContainers(configuration, stagingContainer);
}
}
}
