/// <summary> /// Runs a series of tasks, using the OutputFiles feature in conjunction with the file conventions library to upload the tasks to a container. /// Then downloads the files from the container to the local machine. /// </summary> public static async Task <CloudBlobContainer> RunWithConventions( BatchClient batchClient, CloudStorageAccount linkedStorageAccount, string poolId, int nodeCount, string jobId) { await CreatePoolAsync(batchClient, poolId, nodeCount); CloudJob job = batchClient.JobOperations.CreateJob(jobId, new PoolInformation { PoolId = poolId }); // Get the container URL to use string containerName = job.OutputStorageContainerName(); CloudBlobContainer container = linkedStorageAccount.CreateCloudBlobClient().GetContainerReference(containerName); await container.CreateIfNotExistsAsync(); string containerUrl = job.GetOutputStorageContainerUrl(linkedStorageAccount); // Commit the job to the Batch service await job.CommitAsync(); Console.WriteLine($"Created job {jobId}"); // Obtain the bound job from the Batch service await job.RefreshAsync(); // Create a series of simple tasks which dump the task environment to a file and then write random values to a text file IEnumerable <CloudTask> tasksToAdd = Enumerable.Range(1, 20).Select(i => { var taskId = i.ToString().PadLeft(3, '0'); var task = new CloudTask(taskId, "cmd /v:ON /c \"echo off && set && (FOR /L %i IN (1,1,100000) DO (ECHO !RANDOM!)) > output.txt\""); task.WithOutputFile(@"..\std*.txt", containerUrl, TaskOutputKind.TaskLog, OutputFileUploadCondition.TaskCompletion) .WithOutputFile(@"output.txt", containerUrl, TaskOutputKind.TaskOutput, OutputFileUploadCondition.TaskSuccess); return(task); } ); // Add the tasks to the job; the tasks are automatically // scheduled for execution on the nodes by the Batch service. await job.AddTaskAsync(tasksToAdd); Console.WriteLine($"All tasks added to job {job.Id}"); Console.WriteLine(); Console.WriteLine($"Downloading outputs to {Directory.GetCurrentDirectory()}"); foreach (CloudTask task in job.CompletedTasks()) { if (task.ExecutionInformation.Result != TaskExecutionResult.Success) { Console.WriteLine($"Task {task.Id} failed"); Console.WriteLine(SampleHelpers.GetFailureInfoDetails(task.ExecutionInformation.FailureInformation)); } else { Console.WriteLine($"Task {task.Id} completed successfully"); } foreach (OutputFileReference output in task.OutputStorage(linkedStorageAccount).ListOutputs(TaskOutputKind.TaskOutput)) { Console.WriteLine($"output file: {output.FilePath}"); await output.DownloadToFileAsync($"{jobId}-{output.FilePath}", System.IO.FileMode.Create); } } return(container); }
/// <summary> /// Runs a series of tasks, using the OutputFiles feature to upload the tasks files to a container. /// Then downloads the files from the container to the local machine. /// </summary> public static async Task <CloudBlobContainer> Run( BatchClient batchClient, CloudStorageAccount storageAccount, string poolId, int nodeCount, string jobId) { const string containerName = "outputfilescontainer"; await CreatePoolAsync(batchClient, poolId, nodeCount); CloudJob job = batchClient.JobOperations.CreateJob(jobId, new PoolInformation { PoolId = poolId }); CloudBlobContainer container = storageAccount.CreateCloudBlobClient().GetContainerReference(containerName); await container.CreateIfNotExistsAsync(); string containerSas = container.GetSharedAccessSignature(new SharedAccessBlobPolicy() { Permissions = SharedAccessBlobPermissions.Write, SharedAccessExpiryTime = DateTimeOffset.UtcNow.AddDays(1) }); string containerUrl = container.Uri.AbsoluteUri + containerSas; // Commit the job to the Batch service await job.CommitAsync(); Console.WriteLine($"Created job {jobId}"); // Obtain the bound job from the Batch service await job.RefreshAsync(); // Create a series of simple tasks which dump the task environment to a file and then write random values to a text file IEnumerable <CloudTask> tasksToAdd = Enumerable.Range(1, 20).Select(i => { var taskId = i.ToString().PadLeft(3, '0'); return(new CloudTask(taskId, "cmd /v:ON /c \"echo off && set && (FOR /L %i IN (1,1,100000) DO (ECHO !RANDOM!)) > output.txt\"") { OutputFiles = new List <OutputFile> { new OutputFile( filePattern: @"..\std*.txt", destination: new OutputFileDestination(new OutputFileBlobContainerDestination( containerUrl: containerUrl, path: taskId)), uploadOptions: new OutputFileUploadOptions( uploadCondition: OutputFileUploadCondition.TaskCompletion)), new OutputFile( filePattern: @"output.txt", destination: new OutputFileDestination(new OutputFileBlobContainerDestination( containerUrl: containerUrl, path: taskId + @"\output.txt")), uploadOptions: new OutputFileUploadOptions( uploadCondition: OutputFileUploadCondition.TaskCompletion)), } }); } ); // Add the tasks to the job; the tasks are automatically // scheduled for execution on the nodes by the Batch service. await job.AddTaskAsync(tasksToAdd); Console.WriteLine($"All tasks added to job {job.Id}"); Console.WriteLine(); Console.WriteLine($"Downloading outputs to {Directory.GetCurrentDirectory()}"); foreach (CloudTask task in job.CompletedTasks()) { if (task.ExecutionInformation.Result != TaskExecutionResult.Success) { Console.WriteLine($"Task {task.Id} failed"); Console.WriteLine(SampleHelpers.GetFailureInfoDetails(task.ExecutionInformation.FailureInformation)); } else { Console.WriteLine($"Task {task.Id} completed successfully"); } CloudBlobDirectory directory = container.GetDirectoryReference(task.Id); Directory.CreateDirectory(task.Id); foreach (var blobInDirectory in directory.ListBlobs()) { CloudBlockBlob blockBlob = blobInDirectory as CloudBlockBlob; Console.WriteLine($" {blockBlob.Name}"); await blockBlob.DownloadToFileAsync(blockBlob.Name, FileMode.Create); } } return(container); }
public static async Task <CloudBlobContainer> Run( BatchClient batchClient, CloudStorageAccount linkedStorageAccount, string poolId, int nodeCount, string jobId) { const string appPackageId = "PersistOutputsTask"; const string appPackageVersion = "1.0"; // Create and configure an unbound pool. CloudPool pool = batchClient.PoolOperations.CreatePool( poolId: poolId, virtualMachineSize: "standard_d1_v2", targetDedicatedComputeNodes: nodeCount, cloudServiceConfiguration: new CloudServiceConfiguration(osFamily: "5")); // Specify the application and version to deploy to the compute nodes. You must // first build PersistOutputsTask, then upload it as an application package. // See https://azure.microsoft.com/documentation/articles/batch-application-packages/ pool.ApplicationPackageReferences = new List <ApplicationPackageReference> { new ApplicationPackageReference { ApplicationId = appPackageId, Version = appPackageVersion } }; // Commit the pool to the Batch service await GettingStartedCommon.CreatePoolIfNotExistAsync(batchClient, pool); CloudJob job = batchClient.JobOperations.CreateJob(jobId, new PoolInformation { PoolId = poolId }); // Create the blob storage container for the outputs. await job.PrepareOutputStorageAsync(linkedStorageAccount); // Create an environment variable on the compute nodes that the // task application can reference when persisting its outputs. string containerName = job.OutputStorageContainerName(); CloudBlobContainer container = linkedStorageAccount.CreateCloudBlobClient().GetContainerReference(containerName); string containerUrl = job.GetOutputStorageContainerUrl(linkedStorageAccount); job.CommonEnvironmentSettings = new[] { new EnvironmentSetting("JOB_CONTAINER_URL", containerUrl) }; // Commit the job to the Batch service await job.CommitAsync(); Console.WriteLine($"Created job {jobId}"); // Obtain the bound job from the Batch service await job.RefreshAsync(); IEnumerable <CloudTask> tasks = Enumerable.Range(1, 20).Select(i => new CloudTask(i.ToString().PadLeft(3, '0'), $"cmd /c %AZ_BATCH_APP_PACKAGE_{appPackageId.ToUpper()}#{appPackageVersion}%\\PersistOutputsTask.exe") ); // Add the tasks to the job; the tasks are automatically // scheduled for execution on the nodes by the Batch service. await job.AddTaskAsync(tasks); Console.WriteLine($"All tasks added to job {job.Id}"); Console.WriteLine(); Console.WriteLine($"Downloading outputs to {Directory.GetCurrentDirectory()}"); foreach (CloudTask task in job.CompletedTasks()) { if (task.ExecutionInformation.Result != TaskExecutionResult.Success) { Console.WriteLine($"Task {task.Id} failed"); Console.WriteLine(SampleHelpers.GetFailureInfoDetails(task.ExecutionInformation.FailureInformation)); } else { Console.WriteLine($"Task {task.Id} completed successfully"); } foreach (OutputFileReference output in task.OutputStorage(linkedStorageAccount).ListOutputs(TaskOutputKind.TaskOutput)) { Console.WriteLine($"output file: {output.FilePath}"); await output.DownloadToFileAsync($"{jobId}-{output.FilePath}", System.IO.FileMode.Create); } } return(container); }