private static void AddWork(IBatchClient client) { using (IWorkItemManager wm = client.OpenWorkItemManager()) { //The toolbox contains some helper mechanisms to ease submission and monitoring of tasks. IToolbox toolbox = client.OpenToolbox(); // to submit a batch of tasks, the TaskSubmissionHelper is useful. ITaskSubmissionHelper taskSubmissionHelper = toolbox.CreateTaskSubmissionHelper(wm, Program.PoolName); // workitem is uniquely identified by its name so we will use a timestamp as suffix taskSubmissionHelper.WorkItemName = Environment.GetEnvironmentVariable("USERNAME") + DateTime.Now.ToString("yyyyMMdd-HHmmss"); Console.WriteLine("Creating work item: {0}", taskSubmissionHelper.WorkItemName); // add 2 quick tasks. Tasks within a job must have unique names taskSubmissionHelper.AddTask(new CloudTask("task1", "hostname")); taskSubmissionHelper.AddTask(new CloudTask("task2", "cmd /c dir /s")); //Commit the tasks to the Batch Service IJobCommitUnboundArtifacts artifacts = taskSubmissionHelper.Commit() as IJobCommitUnboundArtifacts; // TaskSubmissionHelper commit artifacts returns the workitem and job name ICloudJob job = wm.GetJob(artifacts.WorkItemName, artifacts.JobName); Console.WriteLine("Waiting for all tasks to complete on work item: {0}, Job: {1} ...", artifacts.WorkItemName, artifacts.JobName); //We use the task state monitor to monitor the state of our tasks -- in this case we will wait for them all to complete. ITaskStateMonitor taskStateMonitor = toolbox.CreateTaskStateMonitor(); // blocking wait on the list of tasks until all tasks reach completed state bool timedOut = taskStateMonitor.WaitAll(job.ListTasks(), TaskState.Completed, new TimeSpan(0, 20, 0)); if (timedOut) { throw new TimeoutException("Timed out waiting for tasks"); } // dump task output foreach (var t in job.ListTasks()) { Console.WriteLine("Task " + t.Name + " says:\n" + t.GetTaskFile(Constants.StandardOutFileName).ReadAsString()); } // remember to delete the workitem before exiting Console.WriteLine("Deleting work item: {0}", artifacts.WorkItemName); wm.DeleteWorkItem(artifacts.WorkItemName); } }
/// <summary> /// Populates Azure Storage with the required files, and /// submits the work item to the Azure Batch service. /// </summary> public async Task RunAsync() { Console.WriteLine("Running with the following settings: "); Console.WriteLine("----------------------------------------"); Console.WriteLine(this.configurationSettings.ToString()); //Upload resources if required. if (this.configurationSettings.ShouldUploadResources) { Console.WriteLine("Splitting file: {0} into {1} subfiles", Constants.TextFilePath, this.configurationSettings.NumberOfMapperTasks); //Split the text file into the correct number of files for consumption by the mapper tasks. FileSplitter splitter = new FileSplitter(); List <string> mapperTaskFiles = await splitter.SplitAsync( Constants.TextFilePath, this.configurationSettings.NumberOfMapperTasks); await this.UploadResourcesAsync(mapperTaskFiles); } //Generate a SAS for the container. string containerSasUrl = Helpers.ConstructContainerSas( this.configurationSettings.StorageAccountName, this.configurationSettings.StorageAccountKey, this.configurationSettings.StorageServiceUrl, this.configurationSettings.BlobContainer); //Set up the Batch Service credentials used to authenticate with the Batch Service. BatchCredentials batchCredentials = new BatchCredentials( this.configurationSettings.BatchAccountName, this.configurationSettings.BatchAccountKey); using (IBatchClient batchClient = BatchClient.Connect(this.configurationSettings.BatchServiceUrl, batchCredentials)) { using (IWorkItemManager workItemManager = batchClient.OpenWorkItemManager()) { //Create the unbound work item in local memory. An object which exists only in local memory (and not on the Batch Service) is "unbound". string workItemName = Environment.GetEnvironmentVariable("USERNAME") + DateTime.UtcNow.ToString("yyyyMMdd-HHmmss"); ICloudWorkItem unboundWorkItem = workItemManager.CreateWorkItem(workItemName); // // Construct the work item properties in local memory before commiting them to the Batch Service. // //Allow enough VMs in the pool to run each mapper task, and 1 extra to run the job manager. int numberOfPoolVMs = 1 + this.configurationSettings.NumberOfMapperTasks; //Define the pool specification for the pool which the work item will run on. IPoolSpecification poolSpecification = new PoolSpecification() { TargetDedicated = numberOfPoolVMs, VMSize = "small", //You can learn more about os families and versions at: //http://msdn.microsoft.com/en-us/library/azure/ee924680.aspx OSFamily = "4", TargetOSVersion = "*" }; //Use the auto pool feature of the Batch Service to create a pool when the work item is created. //This creates a new pool for each work item which is added. IAutoPoolSpecification autoPoolSpecification = new AutoPoolSpecification() { AutoPoolNamePrefix = "TextSearchPool", KeepAlive = false, PoolLifeTimeOption = PoolLifeTimeOption.WorkItem, PoolSpecification = poolSpecification }; //Define the execution environment for this work item -- it will run on the pool defined by the auto pool specification above. unboundWorkItem.JobExecutionEnvironment = new JobExecutionEnvironment() { AutoPoolSpecification = autoPoolSpecification }; //Define the job manager for this work item. This job manager will run when any job is created and will submit the tasks for //the work item. The job manager is the executable which manages the lifetime of the job //and all tasks which should run for the job. In this case, the job manager submits the mapper and reducer tasks. string jobManagerCommandLine = string.Format("{0} -JobManagerTask", Constants.TextSearchExe); List <IResourceFile> jobManagerResourceFiles = Helpers.GetResourceFiles(containerSasUrl, Constants.RequiredExecutableFiles); const string jobManagerTaskName = "JobManager"; unboundWorkItem.JobSpecification = new JobSpecification() { JobManager = new JobManager() { ResourceFiles = jobManagerResourceFiles, CommandLine = jobManagerCommandLine, //Determines if the job should terminate when the job manager process exits KillJobOnCompletion = false, Name = jobManagerTaskName } }; try { //Commit the unbound work item to the Batch Service. Console.WriteLine("Adding work item: {0} to the Batch Service.", unboundWorkItem.Name); await unboundWorkItem.CommitAsync(); //Issues a request to the Batch Service to add the work item which was defined above. // // Wait for the job manager task to complete. // //An object which is backed by a corresponding Batch Service object is "bound." ICloudWorkItem boundWorkItem = await workItemManager.GetWorkItemAsync(workItemName); //Wait for the job to be created automatically by the Batch Service. string boundJobName = await Helpers.WaitForActiveJobAsync(boundWorkItem); ICloudTask boundJobManagerTask = await workItemManager.GetTaskAsync( workItemName, boundJobName, jobManagerTaskName); TimeSpan maxJobCompletionTimeout = TimeSpan.FromMinutes(30); IToolbox toolbox = batchClient.OpenToolbox(); ITaskStateMonitor monitor = toolbox.CreateTaskStateMonitor(); bool timedOut = await monitor.WaitAllAsync(new List <ICloudTask> { boundJobManagerTask }, TaskState.Completed, maxJobCompletionTimeout); Console.WriteLine("Done waiting for job manager task."); await boundJobManagerTask.RefreshAsync(); //Check to ensure the job manager task exited successfully. await Helpers.CheckForTaskSuccessAsync(boundJobManagerTask, dumpStandardOutOnTaskSuccess : true); if (timedOut) { throw new TimeoutException(string.Format("Timed out waiting for job manager task to complete.")); } } catch (AggregateException e) { e.Handle( (innerE) => { //We print all the inner exceptions for debugging purposes. Console.WriteLine(innerE.ToString()); return(false); }); throw; } catch (Exception e) { Console.WriteLine("Hit unexpected exception: {0}", e.ToString()); throw; } finally { //Delete the work item. //This will delete the auto pool associated with the work item as long as the pool //keep alive property is set to false. if (this.configurationSettings.ShouldDeleteWorkItem) { Console.WriteLine("Deleting work item {0}", workItemName); workItemManager.DeleteWorkItem(workItemName); } //Note that there were files uploaded to a container specified in the //configuration file. This container will not be deleted or cleaned up by this sample. } } } }
/// <summary> /// Runs the job manager task. /// </summary> public async Task RunAsync() { Console.WriteLine("JobManager for account: {0}, work item: {1}, job: {2} has started...", this.accountName, this.workItemName, this.jobName); Console.WriteLine(); Console.WriteLine("JobManager running with the following settings: "); Console.WriteLine("----------------------------------------"); Console.WriteLine(this.configurationSettings.ToString()); //Set up the Batch Service credentials used to authenticate with the Batch Service. BatchCredentials batchCredentials = new BatchCredentials( this.configurationSettings.BatchAccountName, this.configurationSettings.BatchAccountKey); using (IBatchClient batchClient = BatchClient.Connect(this.configurationSettings.BatchServiceUrl, batchCredentials)) { using (IWorkItemManager workItemManager = batchClient.OpenWorkItemManager()) { IToolbox toolbox = batchClient.OpenToolbox(); //Construct a container SAS to provide the Batch Service access to the files required to //run the mapper and reducer tasks. string containerSas = Helpers.ConstructContainerSas( this.configurationSettings.StorageAccountName, this.configurationSettings.StorageAccountKey, this.configurationSettings.StorageServiceUrl, this.configurationSettings.BlobContainer); // // Submit mapper tasks. // Console.WriteLine("Submitting {0} mapper tasks.", this.configurationSettings.NumberOfMapperTasks); //The collection of tasks to add to the Batch Service. List <ICloudTask> tasksToAdd = new List <ICloudTask>(); for (int i = 0; i < this.configurationSettings.NumberOfMapperTasks; i++) { string taskName = Helpers.GetMapperTaskName(i); string fileBlobName = Helpers.GetSplitFileName(i); string fileBlobPath = Helpers.ConstructBlobSource(containerSas, fileBlobName); string commandLine = string.Format("{0} -MapperTask {1}", Constants.TextSearchExe, fileBlobPath); ICloudTask unboundMapperTask = new CloudTask(taskName, commandLine); //The set of files (exe's, dll's and configuration files) required to run the mapper task. IReadOnlyList <string> mapperTaskRequiredFiles = Constants.RequiredExecutableFiles; List <IResourceFile> mapperTaskResourceFiles = Helpers.GetResourceFiles(containerSas, mapperTaskRequiredFiles); unboundMapperTask.ResourceFiles = mapperTaskResourceFiles; tasksToAdd.Add(unboundMapperTask); } //Submit the unbound task collection to the Batch Service. //Use the AddTask method which takes a collection of ICloudTasks for the best performance. await workItemManager.AddTaskAsync(this.workItemName, this.jobName, tasksToAdd); // // Wait for the mapper tasks to complete. // Console.WriteLine("Waiting for the mapper tasks to complete..."); //List all the mapper tasks using a name filter. DetailLevel mapperTaskNameFilter = new ODATADetailLevel() { FilterClause = string.Format("startswith(name, '{0}')", Constants.MapperTaskPrefix) }; List <ICloudTask> tasksToMonitor = workItemManager.ListTasks( this.workItemName, this.jobName, detailLevel: mapperTaskNameFilter).ToList(); //Use the task state monitor to wait for the tasks to complete. ITaskStateMonitor taskStateMonitor = toolbox.CreateTaskStateMonitor(); bool timedOut = await taskStateMonitor.WaitAllAsync(tasksToMonitor, TaskState.Completed, TimeSpan.FromMinutes(5)); //Get the list of mapper tasks in order to analyze their state and ensure they completed successfully. IEnumerableAsyncExtended <ICloudTask> asyncEnumerable = workItemManager.ListTasks( this.workItemName, this.jobName, detailLevel: mapperTaskNameFilter); IAsyncEnumerator <ICloudTask> asyncEnumerator = asyncEnumerable.GetAsyncEnumerator(); //Dump the status of each mapper task. while (await asyncEnumerator.MoveNextAsync()) { ICloudTask cloudTask = asyncEnumerator.Current; Console.WriteLine("Task {0} is in state: {1}", cloudTask.Name, cloudTask.State); await Helpers.CheckForTaskSuccessAsync(cloudTask, dumpStandardOutOnTaskSuccess : false); Console.WriteLine(); } //If not all the tasks reached the desired state within the timeout then the job manager //cannot continue. if (timedOut) { const string errorMessage = "Mapper tasks did not complete within expected timeout."; Console.WriteLine(errorMessage); throw new TimeoutException(errorMessage); } // // Create the reducer task. // string reducerTaskCommandLine = string.Format("{0} -ReducerTask", Constants.TextSearchExe); Console.WriteLine("Adding the reducer task: {0}", Constants.ReducerTaskName); ICloudTask unboundReducerTask = new CloudTask(Constants.ReducerTaskName, reducerTaskCommandLine); //The set of files (exe's, dll's and configuration files) required to run the reducer task. List <IResourceFile> reducerTaskResourceFiles = Helpers.GetResourceFiles(containerSas, Constants.RequiredExecutableFiles); unboundReducerTask.ResourceFiles = reducerTaskResourceFiles; //Send the request to the Batch Service to add the reducer task. await workItemManager.AddTaskAsync(this.workItemName, this.jobName, unboundReducerTask); // //Wait for the reducer task to complete. // //Get the bound reducer task and monitor it for completion. ICloudTask boundReducerTask = await workItemManager.GetTaskAsync(this.workItemName, this.jobName, Constants.ReducerTaskName); timedOut = await taskStateMonitor.WaitAllAsync(new List <ICloudTask> { boundReducerTask }, TaskState.Completed, TimeSpan.FromMinutes(2)); //Refresh the reducer task to get the most recent information about it from the Batch Service. await boundReducerTask.RefreshAsync(); //Dump the reducer tasks exit code and scheduling error for debugging purposes. await Helpers.CheckForTaskSuccessAsync(boundReducerTask, dumpStandardOutOnTaskSuccess : true); //Handle the possibilty that the reducer task did not complete in the expected timeout. if (timedOut) { const string errorMessage = "Reducer task did not complete within expected timeout."; Console.WriteLine("Task {0} is in state: {1}", boundReducerTask.Name, boundReducerTask.State); Console.WriteLine(errorMessage); throw new TimeoutException(errorMessage); } //The job manager has completed. Console.WriteLine("JobManager completed successfully."); } } }
private static void AddWork(IBatchClient client) { using (var wm = client.OpenWorkItemManager()) { //The toolbox contains some helper mechanisms to ease submission and monitoring of tasks. var toolbox = client.OpenToolbox(); //Create a work item var workItemName = Environment.GetEnvironmentVariable("USERNAME") + DateTime.Now.ToString("yyyyMMdd-HHmmss"); Console.WriteLine("Creating work item {0}", workItemName); var cloudWorkItem = wm.CreateWorkItem(workItemName); cloudWorkItem.JobExecutionEnvironment = new JobExecutionEnvironment { PoolName = PoolName }; cloudWorkItem.JobSpecification = new JobSpecification() { JobPreparationTask = new JobPreparationTask() { Name = "jobprep", CommandLine = "cmd /c ping 127.0.0.1" } }; cloudWorkItem.Commit(); //Wait for an active job var maxJobCreationTimeout = TimeSpan.FromSeconds(90); var jobCreationStartTime = DateTime.Now; var jobCreationTimeoutTime = jobCreationStartTime.Add(maxJobCreationTimeout); cloudWorkItem = wm.GetWorkItem(workItemName); Console.WriteLine("Waiting for a job to become active..."); while (cloudWorkItem.ExecutionInformation?.RecentJob == null) { cloudWorkItem.Refresh(); if (DateTime.Now > jobCreationTimeoutTime) { throw new TimeoutException("Timed out waiting for job."); } Thread.Sleep(TimeSpan.FromSeconds(5)); } var jobName = cloudWorkItem.ExecutionInformation.RecentJob.Name; Console.WriteLine("Found job {0}. Adding task objects...", jobName); var job = wm.GetJob(workItemName, jobName); job.AddTask(new CloudTask("task1", "hostname")); job.AddTask(new CloudTask("task2", "cmd /c dir /s ..\\..")); var haveEnoughSuccessJobPrepTask = WaitAllJobPrepTask( client, PoolName, workItemName, jobName, (succ, fail) => succ >= 2 || fail > 0, // stop indicator (succ, fail) => succ >= 2, // success indicator TimeSpan.FromMinutes(5), TimeSpan.FromSeconds(5), 10 ); if (haveEnoughSuccessJobPrepTask) // got enough nodes, continue { //We use the task state monitor to monitor the state of our tasks -- in this case we will wait for them all to complete. var taskStateMonitor = toolbox.CreateTaskStateMonitor(); // blocking wait on the list of tasks until all tasks reach completed state var timedOut = taskStateMonitor.WaitAll(job.ListTasks(), TaskState.Completed, TimeSpan.FromMinutes(20)); if (timedOut) { throw new TimeoutException("Timed out waiting for tasks"); } // dump task output foreach (var t in job.ListTasks()) { Console.WriteLine("Task " + t.Name + " output:\n" + t.GetTaskFile(Constants.StandardOutFileName).ReadAsString()); } } else // too much failure, abort { Console.WriteLine("too much failure, abort."); } // remember to delete the workitem before exiting Console.WriteLine("Deleting work item: {0}", workItemName); wm.DeleteWorkItem(workItemName); } }
/// <summary> /// Submit a work item with tasks which have dependant files. /// The files are automatically uploaded to Azure Storage using the FileStaging feature of the Azure.Batch client library. /// </summary> /// <param name="client"></param> private static void AddWorkWithFileStaging(IBatchClient client) { using (IWorkItemManager wm = client.OpenWorkItemManager()) { IToolbox toolbox = client.OpenToolbox(); ITaskSubmissionHelper taskSubmissionHelper = toolbox.CreateTaskSubmissionHelper(wm, Program.PoolName); taskSubmissionHelper.WorkItemName = Environment.GetEnvironmentVariable("USERNAME") + DateTime.Now.ToString("yyyyMMdd-HHmmss"); Console.WriteLine("Creating work item: {0}", taskSubmissionHelper.WorkItemName); ICloudTask taskToAdd1 = new CloudTask("task_with_file1", "cmd /c type *.txt"); ICloudTask taskToAdd2 = new CloudTask("task_with_file2", "cmd /c dir /s"); //Set up a collection of files to be staged -- these files will be uploaded to Azure Storage //when the tasks are submitted to the Azure Batch service. taskToAdd1.FilesToStage = new List <IFileStagingProvider>(); taskToAdd2.FilesToStage = new List <IFileStagingProvider>(); // generate a local file in temp directory Process cur = Process.GetCurrentProcess(); string path = Path.Combine(Environment.GetEnvironmentVariable("TEMP"), cur.Id.ToString() + ".txt"); System.IO.File.WriteAllText(path, "hello from " + cur.Id.ToString()); // add file as task dependency so it'll be uploaded to storage before task // is submitted and download onto the VM before task starts execution FileToStage file = new FileToStage(path, new StagingStorageAccount(Program.StorageAccount, Program.StorageKey, Program.StorageBlobEndpoint)); taskToAdd1.FilesToStage.Add(file); taskToAdd2.FilesToStage.Add(file); // filetostage object can be reused taskSubmissionHelper.AddTask(taskToAdd1); taskSubmissionHelper.AddTask(taskToAdd2); IJobCommitUnboundArtifacts artifacts = null; bool errors = false; try { //Stage the files to Azure Storage and add the tasks to Azure Batch. artifacts = taskSubmissionHelper.Commit() as IJobCommitUnboundArtifacts; } catch (AggregateException ae) { errors = true; // Go through all exceptions and dump useful information ae.Handle((x) => { if (x is BatchException) { BatchException be = x as BatchException; if (null != be.RequestInformation && null != be.RequestInformation.AzureError) { // Write the server side error information Console.Error.WriteLine(be.RequestInformation.AzureError.Code); Console.Error.WriteLine(be.RequestInformation.AzureError.Message.Value); if (null != be.RequestInformation.AzureError.Values) { foreach (var v in be.RequestInformation.AzureError.Values) { Console.Error.WriteLine(v.Key + " : " + v.Value); } } } } else { Console.WriteLine(x); } // Indicate that the error has been handled return(true); }); } // if there is no exception, wait for job response if (!errors) { List <ICloudTask> tasksToMonitorForCompletion = wm.ListTasks(artifacts.WorkItemName, artifacts.JobName).ToList(); Console.WriteLine("Waiting for all tasks to complete on work item: {0}, Job: {1} ...", artifacts.WorkItemName, artifacts.JobName); client.OpenToolbox().CreateTaskStateMonitor().WaitAll(tasksToMonitorForCompletion, TaskState.Completed, TimeSpan.FromMinutes(30)); foreach (ICloudTask task in wm.ListTasks(artifacts.WorkItemName, artifacts.JobName)) { Console.WriteLine("Task " + task.Name + " says:\n" + task.GetTaskFile(Constants.StandardOutFileName).ReadAsString()); Console.WriteLine(task.GetTaskFile(Constants.StandardErrorFileName).ReadAsString()); } } Console.WriteLine("Deleting work item: {0}", artifacts.WorkItemName); wm.DeleteWorkItem(artifacts.WorkItemName); //Don't forget to delete the work item before you exit } }
public static void JobMain(string[] args) { //Load the configuration TopNWordsConfiguration configuration = TopNWordsConfiguration.LoadConfigurationFromAppConfig(); StagingStorageAccount stagingStorageAccount = new StagingStorageAccount( configuration.StorageAccountName, configuration.StorageAccountKey, configuration.StorageAccountBlobEndpoint); IBatchClient client = BatchClient.Connect(configuration.BatchServiceUrl, new BatchCredentials(configuration.BatchAccountName, configuration.BatchAccountKey)); string stagingContainer = null; //Create a pool (if user hasn't provided one) if (configuration.ShouldCreatePool) { using (IPoolManager pm = client.OpenPoolManager()) { //OSFamily 4 == OS 2012 R2 //You can learn more about os families and versions at: //http://msdn.microsoft.com/en-us/library/azure/ee924680.aspx ICloudPool pool = pm.CreatePool(configuration.PoolName, targetDedicated: configuration.PoolSize, osFamily: "4", vmSize: "small"); Console.WriteLine("Adding pool {0}", configuration.PoolName); pool.Commit(); } } try { using (IWorkItemManager wm = client.OpenWorkItemManager()) { IToolbox toolbox = client.OpenToolbox(); //Use the TaskSubmissionHelper to help us create a WorkItem and add tasks to it. ITaskSubmissionHelper taskSubmissionHelper = toolbox.CreateTaskSubmissionHelper(wm, configuration.PoolName); taskSubmissionHelper.WorkItemName = configuration.WorkItemName; FileToStage topNWordExe = new FileToStage(TopNWordsExeName, stagingStorageAccount); FileToStage storageDll = new FileToStage(StorageClientDllName, stagingStorageAccount); string bookFileUri = UploadBookFileToCloudBlob(configuration, configuration.BookFileName); Console.WriteLine("{0} uploaded to cloud", configuration.BookFileName); for (int i = 1; i <= configuration.NumberOfTasks; i++) { ICloudTask task = new CloudTask("task_no_" + i, String.Format("{0} --Task {1} {2} {3} {4}", TopNWordsExeName, bookFileUri, configuration.NumberOfTopWords, configuration.StorageAccountName, configuration.StorageAccountKey)); //This is the list of files to stage to a container -- for each TaskSubmissionHelper one container is created and //files all resolve to Azure Blobs by their name (so two tasks with the same named file will create just 1 blob in //the TaskSubmissionHelper's container). task.FilesToStage = new List <IFileStagingProvider> { topNWordExe, storageDll }; taskSubmissionHelper.AddTask(task); } //Commit all the tasks to the Batch Service. IJobCommitUnboundArtifacts artifacts = taskSubmissionHelper.Commit() as IJobCommitUnboundArtifacts; foreach (var fileStagingArtifact in artifacts.FileStagingArtifacts) { SequentialFileStagingArtifact stagingArtifact = fileStagingArtifact.Value as SequentialFileStagingArtifact; if (stagingArtifact != null) { stagingContainer = stagingArtifact.BlobContainerCreated; Console.WriteLine("Uploaded files to container: {0} -- you will be charged for their storage unless you delete them.", stagingArtifact.BlobContainerCreated); } } //Get the job to monitor status. ICloudJob job = wm.GetJob(artifacts.WorkItemName, artifacts.JobName); Console.Write("Waiting for tasks to complete ..."); // Wait 1 minute for all tasks to reach the completed state client.OpenToolbox().CreateTaskStateMonitor().WaitAll(job.ListTasks(), TaskState.Completed, TimeSpan.FromMinutes(20)); Console.WriteLine("Done."); foreach (ICloudTask task in job.ListTasks()) { Console.WriteLine("Task " + task.Name + " says:\n" + task.GetTaskFile(Constants.StandardOutFileName).ReadAsString()); Console.WriteLine(task.GetTaskFile(Constants.StandardErrorFileName).ReadAsString()); } } } finally { //Delete the pool that we created if (configuration.ShouldCreatePool) { using (IPoolManager pm = client.OpenPoolManager()) { Console.WriteLine("Deleting pool: {0}", configuration.PoolName); pm.DeletePool(configuration.PoolName); } } //Delete the workitem that we created if (configuration.ShouldDeleteWorkItem) { using (IWorkItemManager wm = client.OpenWorkItemManager()) { Console.WriteLine("Deleting work item: {0}", configuration.WorkItemName); wm.DeleteWorkItem(configuration.WorkItemName); } } //Delete the containers we created if (configuration.ShouldDeleteContainer) { DeleteContainers(configuration, stagingContainer); } } }
/// <summary> /// Submit a work item with tasks which have dependant files. /// The files are automatically uploaded to Azure Storage using the FileStaging feature of the Azure.Batch client library. /// </summary> /// <param name="client"></param> private static void AddWorkWithFileStaging(IBatchClient client) { using (IWorkItemManager wm = client.OpenWorkItemManager()) { IToolbox toolbox = client.OpenToolbox(); ITaskSubmissionHelper taskSubmissionHelper = toolbox.CreateTaskSubmissionHelper(wm, PoolName); taskSubmissionHelper.WorkItemName = Environment.GetEnvironmentVariable("USERNAME") + DateTime.Now.ToString("yyyyMMdd-HHmmss"); Console.WriteLine("Creating work item: {0}", taskSubmissionHelper.WorkItemName); ICloudTask taskToAdd1 = new CloudTask("task_with_file1", "cmd /c type *.txt"); ICloudTask taskToAdd2 = new CloudTask("task_with_file2", "cmd /c dir /s"); //Set up a collection of files to be staged -- these files will be uploaded to Azure Storage //when the tasks are submitted to the Azure Batch service. taskToAdd1.FilesToStage = new List<IFileStagingProvider>(); taskToAdd2.FilesToStage = new List<IFileStagingProvider>(); // generate a local file in temp directory Process cur = Process.GetCurrentProcess(); string path = Path.Combine(Environment.GetEnvironmentVariable("TEMP"), cur.Id + ".txt"); File.WriteAllText(path, "hello from " + cur.Id); // add file as task dependency so it'll be uploaded to storage before task // is submitted and download onto the VM before task starts execution FileToStage file = new FileToStage(path, new StagingStorageAccount(StorageAccount, StorageKey, StorageBlobEndpoint)); taskToAdd1.FilesToStage.Add(file); taskToAdd2.FilesToStage.Add(file); // filetostage object can be reused taskSubmissionHelper.AddTask(taskToAdd1); taskSubmissionHelper.AddTask(taskToAdd2); IJobCommitUnboundArtifacts artifacts = null; bool errors = false; try { //Stage the files to Azure Storage and add the tasks to Azure Batch. artifacts = taskSubmissionHelper.Commit() as IJobCommitUnboundArtifacts; } catch (AggregateException ae) { errors = true; // Go through all exceptions and dump useful information ae.Handle(x => { if (x is BatchException) { BatchException be = x as BatchException; if (null != be.RequestInformation && null != be.RequestInformation.AzureError) { // Write the server side error information Console.Error.WriteLine(be.RequestInformation.AzureError.Code); Console.Error.WriteLine(be.RequestInformation.AzureError.Message.Value); if (null != be.RequestInformation.AzureError.Values) { foreach (var v in be.RequestInformation.AzureError.Values) { Console.Error.WriteLine(v.Key + " : " + v.Value); } } } } else { Console.WriteLine(x); } // Indicate that the error has been handled return true; }); } // if there is no exception, wait for job response if (!errors) { List<ICloudTask> tasksToMonitorForCompletion = wm.ListTasks(artifacts.WorkItemName, artifacts.JobName).ToList(); Console.WriteLine("Waiting for all tasks to complete on work item: {0}, Job: {1} ...", artifacts.WorkItemName, artifacts.JobName); client.OpenToolbox().CreateTaskStateMonitor().WaitAll(tasksToMonitorForCompletion, TaskState.Completed, TimeSpan.FromMinutes(30)); foreach (ICloudTask task in wm.ListTasks(artifacts.WorkItemName, artifacts.JobName)) { Console.WriteLine("Task " + task.Name + " says:\n" + task.GetTaskFile(Constants.StandardOutFileName).ReadAsString()); Console.WriteLine(task.GetTaskFile(Constants.StandardErrorFileName).ReadAsString()); } } Console.WriteLine("Deleting work item: {0}", artifacts.WorkItemName); wm.DeleteWorkItem(artifacts.WorkItemName); //Don't forget to delete the work item before you exit } }
private static void AddWork(IBatchClient client) { using (IWorkItemManager wm = client.OpenWorkItemManager()) { //The toolbox contains some helper mechanisms to ease submission and monitoring of tasks. IToolbox toolbox = client.OpenToolbox(); // to submit a batch of tasks, the TaskSubmissionHelper is useful. ITaskSubmissionHelper taskSubmissionHelper = toolbox.CreateTaskSubmissionHelper(wm, PoolName); // workitem is uniquely identified by its name so we will use a timestamp as suffix taskSubmissionHelper.WorkItemName = Environment.GetEnvironmentVariable("USERNAME") + DateTime.Now.ToString("yyyyMMdd-HHmmss"); Console.WriteLine("Creating work item: {0}", taskSubmissionHelper.WorkItemName); // add 2 quick tasks. Tasks within a job must have unique names taskSubmissionHelper.AddTask(new CloudTask("task1", "hostname")); taskSubmissionHelper.AddTask(new CloudTask("task2", "cmd /c dir /s")); //Commit the tasks to the Batch Service IJobCommitUnboundArtifacts artifacts = taskSubmissionHelper.Commit() as IJobCommitUnboundArtifacts; // TaskSubmissionHelper commit artifacts returns the workitem and job name if (artifacts != null) { ICloudJob job = wm.GetJob(artifacts.WorkItemName, artifacts.JobName); Console.WriteLine("Waiting for all tasks to complete on work item: {0}, Job: {1} ...", artifacts.WorkItemName, artifacts.JobName); //We use the task state monitor to monitor the state of our tasks -- in this case we will wait for them all to complete. ITaskStateMonitor taskStateMonitor = toolbox.CreateTaskStateMonitor(); // blocking wait on the list of tasks until all tasks reach completed state bool timedOut = taskStateMonitor.WaitAll(job.ListTasks(), TaskState.Completed, new TimeSpan(0, 20, 0)); if (timedOut) { throw new TimeoutException("Timed out waiting for tasks"); } // dump task output foreach (var t in job.ListTasks()) { Console.WriteLine("Task " + t.Name + " says:\n" + t.GetTaskFile(Constants.StandardOutFileName).ReadAsString()); } // remember to delete the workitem before exiting Console.WriteLine("Deleting work item: {0}", artifacts.WorkItemName); wm.DeleteWorkItem(artifacts.WorkItemName); } } }