public void CannotGetOutputStorageUrlForNullJob() { CloudJob job = null; CloudStorageAccount storageAccount = new CloudStorageAccount(new StorageCredentials("fake", "ZmFrZQ=="), true); var ex = Assert.Throws <ArgumentNullException>(() => job.GetOutputStorageContainerUrl(storageAccount, TimeSpan.FromMinutes(5))); Assert.Equal("job", ex.ParamName); }
public void CannotGetOutputStorageUrlForNullStorageAccount() { using (var batchClient = BatchClient.Open(new FakeBatchServiceClient())) { CloudJob job = batchClient.JobOperations.CreateJob(); job.Id = "fakejob"; CloudStorageAccount storageAccount = null; var ex = Assert.Throws <ArgumentNullException>(() => job.GetOutputStorageContainerUrl(storageAccount, TimeSpan.FromMinutes(5))); Assert.Equal("storageAccount", ex.ParamName); } }
public void CannotGetOutputStorageUrlWithNegativeExpiryTime() { using (var batchClient = BatchClient.Open(new FakeBatchServiceClient())) { CloudJob job = batchClient.JobOperations.CreateJob(); job.Id = "fakejob"; CloudStorageAccount storageAccount = new CloudStorageAccount(new StorageCredentials("fake", "ZmFrZQ=="), true); var ex = Assert.Throws <ArgumentException>(() => job.GetOutputStorageContainerUrl(storageAccount, TimeSpan.FromMinutes(-5))); Assert.Equal("expiryTime", ex.ParamName); } }
public async Task CannotGetOutputStorageUrlWithZeroExpiryTime() { using (var batchClient = await BatchClient.OpenAsync(new FakeBatchServiceClient())) { CloudJob job = batchClient.JobOperations.CreateJob(); job.Id = "fakejob"; CloudStorageAccount storageAccount = new CloudStorageAccount(new StorageCredentials("fake", new byte[] { 65, 66, 67, 68 }), true); var ex = Assert.Throws <ArgumentException>(() => job.GetOutputStorageContainerUrl(storageAccount, TimeSpan.FromMinutes(0))); Assert.Equal("expiryTime", ex.ParamName); } }
/// <summary> /// Runs a series of tasks, using the OutputFiles feature in conjunction with the file conventions library to upload the tasks to a container. /// Then downloads the files from the container to the local machine. /// </summary> public static async Task <CloudBlobContainer> RunWithConventions( BatchClient batchClient, CloudStorageAccount linkedStorageAccount, string poolId, int nodeCount, string jobId) { await CreatePoolAsync(batchClient, poolId, nodeCount); CloudJob job = batchClient.JobOperations.CreateJob(jobId, new PoolInformation { PoolId = poolId }); // Get the container URL to use string containerName = job.OutputStorageContainerName(); CloudBlobContainer container = linkedStorageAccount.CreateCloudBlobClient().GetContainerReference(containerName); await container.CreateIfNotExistsAsync(); string containerUrl = job.GetOutputStorageContainerUrl(linkedStorageAccount); // Commit the job to the Batch service await job.CommitAsync(); Console.WriteLine($"Created job {jobId}"); // Obtain the bound job from the Batch service await job.RefreshAsync(); // Create a series of simple tasks which dump the task environment to a file and then write random values to a text file IEnumerable <CloudTask> tasksToAdd = Enumerable.Range(1, 20).Select(i => { var taskId = i.ToString().PadLeft(3, '0'); var task = new CloudTask(taskId, "cmd /v:ON /c \"echo off && set && (FOR /L %i IN (1,1,100000) DO (ECHO !RANDOM!)) > output.txt\""); task.WithOutputFile(@"..\std*.txt", containerUrl, TaskOutputKind.TaskLog, OutputFileUploadCondition.TaskCompletion) .WithOutputFile(@"output.txt", containerUrl, TaskOutputKind.TaskOutput, OutputFileUploadCondition.TaskSuccess); return(task); } ); // Add the tasks to the job; the tasks are automatically // scheduled for execution on the nodes by the Batch service. await job.AddTaskAsync(tasksToAdd); Console.WriteLine($"All tasks added to job {job.Id}"); Console.WriteLine(); Console.WriteLine($"Downloading outputs to {Directory.GetCurrentDirectory()}"); foreach (CloudTask task in job.CompletedTasks()) { if (task.ExecutionInformation.Result != TaskExecutionResult.Success) { Console.WriteLine($"Task {task.Id} failed"); Console.WriteLine(SampleHelpers.GetFailureInfoDetails(task.ExecutionInformation.FailureInformation)); } else { Console.WriteLine($"Task {task.Id} completed successfully"); } foreach (OutputFileReference output in task.OutputStorage(linkedStorageAccount).ListOutputs(TaskOutputKind.TaskOutput)) { Console.WriteLine($"output file: {output.FilePath}"); await output.DownloadToFileAsync($"{jobId}-{output.FilePath}", System.IO.FileMode.Create); } } return(container); }
static void Main(string[] args) { if (String.IsNullOrEmpty(BatchAccountName) || String.IsNullOrEmpty(BatchAccountKey) || String.IsNullOrEmpty(BatchAccountUrl) || String.IsNullOrEmpty(StorageAccountName) || String.IsNullOrEmpty(StorageAccountKey)) { throw new InvalidOperationException("One or more account credential strings have not been populated. Please ensure that your Batch and Storage account credentials have been specified."); } try { Console.WriteLine("Sample start: {0}", DateTime.Now); Console.WriteLine(); Stopwatch timer = new Stopwatch(); timer.Start(); // Create the blob client, for use in obtaining references to blob storage containers (CloudBlobClient, CloudStorageAccount)cc = CreateCloudBlobClient(StorageAccountName, StorageAccountKey); CloudBlobClient blobClient = cc.Item1; CloudStorageAccount linkedStorageAccount = cc.Item2; // Use the blob client to create the input container in Azure Storage const string inputContainerName = "input"; CloudBlobContainer container = blobClient.GetContainerReference(inputContainerName); container.CreateIfNotExistsAsync().Wait(); // The collection of data files that are to be processed by the tasks List <string> inputFilePaths = new List <string> { "input.txt" }; // Upload the data files to Azure Storage. This is the data that will be processed by each of the tasks that are // executed on the compute nodes within the pool. UploadFileToContainer(blobClient, inputContainerName, "input.txt"); // Get a Batch client using account creds BatchSharedKeyCredentials cred = new BatchSharedKeyCredentials(BatchAccountUrl, BatchAccountName, BatchAccountKey); using (BatchClient batchClient = BatchClient.Open(cred)) { Console.WriteLine("Creating pool [{0}]...", PoolId); // Create a Windows Server image, VM configuration, Batch pool ImageReference imageReference = CreateImageReference(); VirtualMachineConfiguration vmConfiguration = CreateVirtualMachineConfiguration(imageReference); CreateBatchPool(batchClient, vmConfiguration); // Create a Batch job Console.WriteLine("Creating job [{0}]...", JobId); try { CloudJob job = batchClient.JobOperations.CreateJob(); job.Id = JobId; job.PoolInformation = new PoolInformation { PoolId = PoolId }; job.UsesTaskDependencies = true; // Create the blob storage container for the outputs. Task t1 = job.PrepareOutputStorageAsync(linkedStorageAccount); Task.WaitAll(t1); string containerName = job.OutputStorageContainerName(); string containerUrl = job.GetOutputStorageContainerUrl(linkedStorageAccount); job.CommonEnvironmentSettings = new[] { new EnvironmentSetting("JOB_CONTAINER_URL", containerUrl) }; job.Commit(); } catch (BatchException be) { // Accept the specific error code JobExists as that is expected if the job already exists if (be.RequestInformation?.BatchError?.Code == BatchErrorCodeStrings.JobExists) { Console.WriteLine("The job {0} already existed when we tried to create it", JobId); } else { throw; // Any other exception is unexpected } } // Create a collection to hold the tasks that we'll be adding to the job List <CloudTask> tasks = new List <CloudTask>(); // Create each of the tasks to process one of the input files. string lastTaskId = null; for (int i = 0; i < 3; i++) { string taskId = String.Format("Task{0}", i); CloudTask task; // first task if (lastTaskId == null) { //task = new CloudTask(taskId, $"cmd /c %AZ_BATCH_APP_PACKAGE_TESTBATCHAPP%\\testbatchapp.exe {i}") task = new CloudTask(taskId, $"cmd /c %AZ_BATCH_APP_PACKAGE_HPCAPP%\\testbatchapp.exe {i}") { ApplicationPackageReferences = new List <ApplicationPackageReference> { new ApplicationPackageReference { //ApplicationId = "testbatchapp", ApplicationId = "hpcapp" //, //Version = "1.1" } } }; } else { // task = new CloudTask(taskId, $"cmd /c %AZ_BATCH_APP_PACKAGE_TESTBATCHAPP%\\testbatchapp.exe {i}") task = new CloudTask(taskId, $"cmd /c %AZ_BATCH_APP_PACKAGE_HPCAPP%\\testbatchapp.exe {i}") { ApplicationPackageReferences = new List <ApplicationPackageReference> { new ApplicationPackageReference { // ApplicationId = "testbatchapp", ApplicationId = "hpcapp" //, //Version = "1.1" } } }; task.DependsOn = TaskDependencies.OnId(lastTaskId); } lastTaskId = taskId; tasks.Add(task); } // Add all tasks to the job. batchClient.JobOperations.AddTask(JobId, tasks); // Monitor task success/failure, specifying a maximum amount of time to wait for the tasks to complete. TimeSpan timeout = TimeSpan.FromMinutes(30); Console.WriteLine("Monitoring all tasks for 'Completed' state, timeout in {0}...", timeout); IEnumerable <CloudTask> addedTasks = batchClient.JobOperations.ListTasks(JobId); batchClient.Utilities.CreateTaskStateMonitor().WaitAll(addedTasks, TaskState.Completed, timeout); Console.WriteLine("All tasks reached state Completed."); // Print task output Console.WriteLine(); Console.WriteLine("Printing task output..."); IEnumerable <CloudTask> completedtasks = batchClient.JobOperations.ListTasks(JobId); foreach (CloudTask task in completedtasks) { List <Task> readTasks = new List <Task>(); foreach (OutputFileReference output in task.OutputStorage(linkedStorageAccount).ListOutputs(TaskOutputKind.TaskOutput)) { // Console.WriteLine($"output file: {output.FilePath}"); readTasks.Add(output.DownloadToFileAsync($"{JobId}-{output.FilePath}", System.IO.FileMode.Create)); } Task.WaitAll(readTasks.ToArray()); } // Print out some timing info timer.Stop(); Console.WriteLine(); Console.WriteLine("Sample end: {0}", DateTime.Now); Console.WriteLine("Elapsed time: {0}", timer.Elapsed); // Clean up Storage resources container.DeleteIfExistsAsync().Wait(); Console.WriteLine("Container [{0}] deleted.", inputContainerName); // Clean up Batch resources (if the user so chooses) Console.WriteLine(); Console.Write("Delete job? [yes] no: "); string response = Console.ReadLine().ToLower(); if (response != "n" && response != "no") { batchClient.JobOperations.DeleteJob(JobId); } Console.Write("Delete pool? [yes] no: "); response = Console.ReadLine().ToLower(); if (response != "n" && response != "no") { batchClient.PoolOperations.DeletePool(PoolId); } } } finally { Console.WriteLine(); Console.WriteLine("Sample complete, hit ENTER to exit..."); Console.ReadLine(); } }
public static async Task <CloudBlobContainer> Run( BatchClient batchClient, CloudStorageAccount linkedStorageAccount, string poolId, int nodeCount, string jobId) { const string appPackageId = "PersistOutputsTask"; const string appPackageVersion = "1.0"; // Create and configure an unbound pool. CloudPool pool = batchClient.PoolOperations.CreatePool( poolId: poolId, virtualMachineSize: "standard_d1_v2", targetDedicatedComputeNodes: nodeCount, cloudServiceConfiguration: new CloudServiceConfiguration(osFamily: "5")); // Specify the application and version to deploy to the compute nodes. You must // first build PersistOutputsTask, then upload it as an application package. // See https://azure.microsoft.com/documentation/articles/batch-application-packages/ pool.ApplicationPackageReferences = new List <ApplicationPackageReference> { new ApplicationPackageReference { ApplicationId = appPackageId, Version = appPackageVersion } }; // Commit the pool to the Batch service await GettingStartedCommon.CreatePoolIfNotExistAsync(batchClient, pool); CloudJob job = batchClient.JobOperations.CreateJob(jobId, new PoolInformation { PoolId = poolId }); // Create the blob storage container for the outputs. await job.PrepareOutputStorageAsync(linkedStorageAccount); // Create an environment variable on the compute nodes that the // task application can reference when persisting its outputs. string containerName = job.OutputStorageContainerName(); CloudBlobContainer container = linkedStorageAccount.CreateCloudBlobClient().GetContainerReference(containerName); string containerUrl = job.GetOutputStorageContainerUrl(linkedStorageAccount); job.CommonEnvironmentSettings = new[] { new EnvironmentSetting("JOB_CONTAINER_URL", containerUrl) }; // Commit the job to the Batch service await job.CommitAsync(); Console.WriteLine($"Created job {jobId}"); // Obtain the bound job from the Batch service await job.RefreshAsync(); IEnumerable <CloudTask> tasks = Enumerable.Range(1, 20).Select(i => new CloudTask(i.ToString().PadLeft(3, '0'), $"cmd /c %AZ_BATCH_APP_PACKAGE_{appPackageId.ToUpper()}#{appPackageVersion}%\\PersistOutputsTask.exe") ); // Add the tasks to the job; the tasks are automatically // scheduled for execution on the nodes by the Batch service. await job.AddTaskAsync(tasks); Console.WriteLine($"All tasks added to job {job.Id}"); Console.WriteLine(); Console.WriteLine($"Downloading outputs to {Directory.GetCurrentDirectory()}"); foreach (CloudTask task in job.CompletedTasks()) { if (task.ExecutionInformation.Result != TaskExecutionResult.Success) { Console.WriteLine($"Task {task.Id} failed"); Console.WriteLine(SampleHelpers.GetFailureInfoDetails(task.ExecutionInformation.FailureInformation)); } else { Console.WriteLine($"Task {task.Id} completed successfully"); } foreach (OutputFileReference output in task.OutputStorage(linkedStorageAccount).ListOutputs(TaskOutputKind.TaskOutput)) { Console.WriteLine($"output file: {output.FilePath}"); await output.DownloadToFileAsync($"{jobId}-{output.FilePath}", System.IO.FileMode.Create); } } return(container); }