/// <summary> /// Creates a job and adds a task to it. The task is a /// custom executable which has a resource file associated with it. /// </summary> /// <param name="batchClient">The BatchClient to use when interacting with the Batch service.</param> /// <param name="cloudStorageAccount">The storage account to upload the files to.</param> /// <param name="jobId">The ID of the job.</param> /// <returns>The set of container names containing the jobs input files.</returns> private async Task <HashSet <string> > SubmitJobAsync(BatchClient batchClient, CloudStorageAccount cloudStorageAccount, string jobId) { // create an empty unbound Job CloudJob unboundJob = batchClient.JobOperations.CreateJob(); unboundJob.Id = jobId; unboundJob.PoolInformation = new PoolInformation() { PoolId = this.poolsAndResourceFileSettings.PoolId }; // Commit Job to create it in the service await unboundJob.CommitAsync(); List <CloudTask> tasksToRun = new List <CloudTask>(); // Create a task which requires some resource files CloudTask taskWithFiles = new CloudTask("task_with_file1", SimpleTaskExe); // Set up a collection of files to be staged -- these files will be uploaded to Azure Storage // when the tasks are submitted to the Azure Batch service. taskWithFiles.FilesToStage = new List <IFileStagingProvider>(); // generate a local file in temp directory string localSampleFile = Path.Combine(Environment.GetEnvironmentVariable("TEMP"), "HelloWorld.txt"); File.WriteAllText(localSampleFile, "hello from Batch PoolsAndResourceFiles sample!"); StagingStorageAccount fileStagingStorageAccount = new StagingStorageAccount( storageAccount: this.accountSettings.StorageAccountName, storageAccountKey: this.accountSettings.StorageAccountKey, blobEndpoint: cloudStorageAccount.BlobEndpoint.ToString()); // add the files as a task dependency so they will be uploaded to storage before the task // is submitted and downloaded to the node before the task starts execution. FileToStage helloWorldFile = new FileToStage(localSampleFile, fileStagingStorageAccount); FileToStage simpleTaskFile = new FileToStage(SimpleTaskExe, fileStagingStorageAccount); // When this task is added via JobOperations.AddTaskAsync below, the FilesToStage are uploaded to storage once. // The Batch service does not automatically delete content from your storage account, so files added in this // way must be manually removed when they are no longer used. taskWithFiles.FilesToStage.Add(helloWorldFile); taskWithFiles.FilesToStage.Add(simpleTaskFile); tasksToRun.Add(taskWithFiles); var fileStagingArtifacts = new ConcurrentBag <ConcurrentDictionary <Type, IFileStagingArtifact> >(); // Use the AddTask method which takes an enumerable of tasks for best performance, as it submits up to 100 // tasks at once in a single request. If the list of tasks is N where N > 100, this will correctly parallelize // the requests and return when all N tasks have been added. await batchClient.JobOperations.AddTaskAsync(jobId, tasksToRun, fileStagingArtifacts : fileStagingArtifacts); // Extract the names of the blob containers from the file staging artifacts HashSet <string> blobContainerNames = GettingStartedCommon.ExtractBlobContainerNames(fileStagingArtifacts); return(blobContainerNames); }
/// <summary> /// Submits a set of tasks to the job /// </summary> /// <param name="batchClient">The batch client to use.</param> /// <returns>The set of blob artifacts created by file staging.</returns> private async Task <HashSet <string> > SubmitTasks(BatchClient batchClient) { List <CloudTask> tasksToRun = new List <CloudTask>(); // Create a task which requires some resource files CloudTask taskWithFiles = new CloudTask("task_with_file1", SimpleTaskExe); // Set up a collection of files to be staged -- these files will be uploaded to Azure Storage // when the tasks are submitted to the Azure Batch service. taskWithFiles.FilesToStage = new List <IFileStagingProvider>(); // generate a local file in temp directory string localSampleFilePath = GettingStartedCommon.GenerateTemporaryFile("HelloWorld.txt", "hello from Batch JobManager sample!"); StagingStorageAccount fileStagingStorageAccount = new StagingStorageAccount( storageAccount: this.configurationSettings.StorageAccountName, storageAccountKey: this.configurationSettings.StorageAccountKey, blobEndpoint: this.configurationSettings.StorageBlobEndpoint); // add the files as a task dependency so they will be uploaded to storage before the task // is submitted and downloaded to the node before the task starts execution. FileToStage helloWorldFile = new FileToStage(localSampleFilePath, fileStagingStorageAccount); FileToStage simpleTaskFile = new FileToStage(SimpleTaskExe, fileStagingStorageAccount); // When this task is added via JobOperations.AddTaskAsync below, the FilesToStage are uploaded to storage once. // The Batch service does not automatically delete content from your storage account, so files added in this // way must be manually removed when they are no longer used. taskWithFiles.FilesToStage.Add(helloWorldFile); taskWithFiles.FilesToStage.Add(simpleTaskFile); tasksToRun.Add(taskWithFiles); var fileStagingArtifacts = new ConcurrentBag <ConcurrentDictionary <Type, IFileStagingArtifact> >(); // Use the AddTask method which takes an enumerable of tasks for best performance, as it submits up to 100 // tasks at once in a single request. If the list of tasks is N where N > 100, this will correctly parallelize // the requests and return when all N tasks have been added. await batchClient.JobOperations.AddTaskAsync(jobId, tasksToRun, fileStagingArtifacts : fileStagingArtifacts); // Extract the names of the blob containers from the file staging artifacts HashSet <string> blobContainerNames = GettingStartedCommon.ExtractBlobContainerNames(fileStagingArtifacts); return(blobContainerNames); }