/// <inheritdoc/> public async Task StartJobWithTask(string taskId, string commandLineText) { await this.TryInit(); try { using (var batchClient = BatchClient.Open(this.credentials)) { CloudJob job = await batchClient.JobOperations.GetJobAsync(this.jobId); var task = new CloudTask(taskId, commandLineText); task.UserIdentity = new UserIdentity(new AutoUserSpecification(AutoUserScope.Task, ElevationLevel.Admin)); await job.AddTaskAsync(task); } } catch (BatchException ex) { // Accept the specific error code JobExists as that is expected if the job already exists if (ex.RequestInformation?.BatchError?.Code == BatchErrorCodeStrings.TaskExists) { throw new BrowseCloudServiceException($"The batch task {taskId} already existed when we tried to create it", ex); } else { throw new BrowseCloudServiceException($"An unexpected error occurred while scheduling batch task {taskId}", ex); } } }
public async Task Job_GetTaskCounts_ReturnsCorrectCountNonZeroTaskSlots() { Func <Task> test = async() => { using (BatchClient batchCli = await TestUtilities.OpenBatchClientFromEnvironmentAsync().ConfigureAwait(false)) { string jobId = "NonZeroTaskSlots-" + TestUtilities.GetMyName(); try { PoolInformation poolInfo = new PoolInformation() { PoolId = "Fake" }; CloudJob unboundJob = batchCli.JobOperations.CreateJob(jobId, poolInfo); await unboundJob.CommitAsync().ConfigureAwait(false); await unboundJob.RefreshAsync().ConfigureAwait(false); CloudTask t1 = new CloudTask("t1", "cmd /c dir"); t1.RequiredSlots = 2; CloudTask t2 = new CloudTask("t2", "cmd /c ping 127.0.0.1 -n 4"); t2.RequiredSlots = 3; await unboundJob.AddTaskAsync(new[] { t1, t2 }).ConfigureAwait(false); await Task.Delay(TimeSpan.FromSeconds(5)).ConfigureAwait(false); // Give the service some time to get the counts var counts = await unboundJob.GetTaskCountsAsync().ConfigureAwait(false); var retTask1 = await unboundJob.GetTaskAsync(t1.Id); Assert.Equal(t1.RequiredSlots, retTask1.RequiredSlots); var retTask2 = await unboundJob.GetTaskAsync(t1.Id); Assert.Equal(t1.RequiredSlots, retTask2.RequiredSlots); Assert.Equal(2, counts.TaskCounts.Active); // Task slots counts is currently broken // Assert.Equal(5, counts.TaskSlotCounts.Active); } finally { await TestUtilities.DeleteJobIfExistsAsync(batchCli, jobId); } } }; await SynchronizationContextHelper.RunTestAsync(test, TestTimeout); }
public async Task CreateTask(Guid guid, List <BatchBlobUrlInput> inputFiles, List <BatchBlobUrlInput> outputFiles) { CloudJob job = await batchClient.JobOperations.GetJobAsync("NlpBatchProc"); CloudTask task = new CloudTask("task_" + guid.ToString(), "cp /tmp/inputBlob.txt /tmp/outputBlob.txt"); task.ResourceFiles = new List <ResourceFile>(); foreach (BatchBlobUrlInput x in inputFiles) { ResourceFile file = new ResourceFile(x.Url, "/tmp/" + x.Part, "0644"); task.ResourceFiles.Add(file); } task.OutputFiles = new List <OutputFile>(); foreach (BatchBlobUrlInput x in outputFiles) { string outputContainerUrl = x.Url; OutputFileDestination fileDest = new OutputFileDestination(new OutputFileBlobContainerDestination(outputContainerUrl)); OutputFile file = new OutputFile("/tmp/" + x.Part, fileDest, new OutputFileUploadOptions(OutputFileUploadCondition.TaskCompletion)); } await job.AddTaskAsync(task); }
/// <summary> /// Performs a simple AddTask test, adding the specified task count using the specified parallelOptions and resultHandlerFunc /// </summary> /// <returns></returns> private async System.Threading.Tasks.Task AddTasksSimpleTestAsync( BatchClient batchCli, string testName, int taskCount, BatchClientParallelOptions parallelOptions, Func <AddTaskResult, CancellationToken, AddTaskResultStatus> resultHandlerFunc, StagingStorageAccount storageCredentials, IEnumerable <string> localFilesToStage, ConcurrentBag <ConcurrentDictionary <Type, IFileStagingArtifact> > fileStagingArtifacts = null, TimeSpan?timeout = null, bool useJobOperations = true) { JobOperations jobOperations = batchCli.JobOperations; string jobId = "Bulk-" + TestUtilities.GetMyName() + "-" + testName + "-" + useJobOperations; try { CloudJob unboundJob = jobOperations.CreateJob(); this.testOutputHelper.WriteLine("Initial job commit for job: {0}", jobId); unboundJob.PoolInformation = new PoolInformation() { PoolId = "DummyPool" }; unboundJob.Id = jobId; await unboundJob.CommitAsync().ConfigureAwait(false); CloudJob boundJob = await jobOperations.GetJobAsync(jobId).ConfigureAwait(false); // // Add a simple set of tasks // IEnumerable <string> taskNames = GenerateTaskIds(taskCount); List <CloudTask> tasksToAdd = new List <CloudTask>(); List <CloudTask> tasksToValidateWith = new List <CloudTask>(); IList <IFileStagingProvider> lastFilesToStageList = null; foreach (string taskName in taskNames) { CloudTask myTask = new CloudTask(taskName, "cmd /c echo hello world"); CloudTask duplicateReadableTask = new CloudTask(taskName, "cmd /c echo hello world"); if (localFilesToStage != null && storageCredentials != null) { myTask.FilesToStage = new List <IFileStagingProvider>(); lastFilesToStageList = myTask.FilesToStage; duplicateReadableTask.FilesToStage = new List <IFileStagingProvider>(); foreach (string fileToStage in localFilesToStage) { duplicateReadableTask.FilesToStage.Add(new FileToStage(fileToStage, storageCredentials)); myTask.FilesToStage.Add(new FileToStage(fileToStage, storageCredentials)); } } tasksToAdd.Add(myTask); tasksToValidateWith.Add(duplicateReadableTask); } List <BatchClientBehavior> behaviors = new List <BatchClientBehavior>(); if (resultHandlerFunc != null) { behaviors.Add(new AddTaskCollectionResultHandler(resultHandlerFunc)); } //Add the tasks Stopwatch stopwatch = new Stopwatch(); this.testOutputHelper.WriteLine("Starting task add"); stopwatch.Start(); if (useJobOperations) { await jobOperations.AddTaskAsync( jobId, tasksToAdd, parallelOptions : parallelOptions, fileStagingArtifacts : fileStagingArtifacts, timeout : timeout, additionalBehaviors : behaviors).ConfigureAwait(continueOnCapturedContext: false); } else { await boundJob.AddTaskAsync( tasksToAdd, parallelOptions : parallelOptions, fileStagingArtifacts : fileStagingArtifacts, timeout : timeout, additionalBehaviors : behaviors).ConfigureAwait(continueOnCapturedContext: false); } stopwatch.Stop(); this.testOutputHelper.WriteLine("Task add finished, took: {0}", stopwatch.Elapsed); if (lastFilesToStageList != null) { Assert.Throws <InvalidOperationException>(() => lastFilesToStageList.Add(new FileToStage("test", null))); } //Ensure the task lists match List <CloudTask> tasksFromService = await jobOperations.ListTasks(jobId).ToListAsync().ConfigureAwait(false); EnsureTasksListsMatch(tasksToValidateWith, tasksFromService); } catch (Exception e) { this.testOutputHelper.WriteLine("Exception: {0}", e.ToString()); throw; } finally { TestUtilities.DeleteJobIfExistsAsync(batchCli, jobId).Wait(); } }
/// <summary> /// Runs a series of tasks, using the OutputFiles feature to upload the tasks files to a container. /// Then downloads the files from the container to the local machine. /// </summary> public static async Task <CloudBlobContainer> Run( BatchClient batchClient, CloudStorageAccount storageAccount, string poolId, int nodeCount, string jobId) { const string containerName = "outputfilescontainer"; await CreatePoolAsync(batchClient, poolId, nodeCount); CloudJob job = batchClient.JobOperations.CreateJob(jobId, new PoolInformation { PoolId = poolId }); CloudBlobContainer container = storageAccount.CreateCloudBlobClient().GetContainerReference(containerName); await container.CreateIfNotExistsAsync(); string containerSas = container.GetSharedAccessSignature(new SharedAccessBlobPolicy() { Permissions = SharedAccessBlobPermissions.Write, SharedAccessExpiryTime = DateTimeOffset.UtcNow.AddDays(1) }); string containerUrl = container.Uri.AbsoluteUri + containerSas; // Commit the job to the Batch service await job.CommitAsync(); Console.WriteLine($"Created job {jobId}"); // Obtain the bound job from the Batch service await job.RefreshAsync(); // Create a series of simple tasks which dump the task environment to a file and then write random values to a text file IEnumerable <CloudTask> tasksToAdd = Enumerable.Range(1, 20).Select(i => { var taskId = i.ToString().PadLeft(3, '0'); return(new CloudTask(taskId, "cmd /v:ON /c \"echo off && set && (FOR /L %i IN (1,1,100000) DO (ECHO !RANDOM!)) > output.txt\"") { OutputFiles = new List <OutputFile> { new OutputFile( filePattern: @"..\std*.txt", destination: new OutputFileDestination(new OutputFileBlobContainerDestination( containerUrl: containerUrl, path: taskId)), uploadOptions: new OutputFileUploadOptions( uploadCondition: OutputFileUploadCondition.TaskCompletion)), new OutputFile( filePattern: @"output.txt", destination: new OutputFileDestination(new OutputFileBlobContainerDestination( containerUrl: containerUrl, path: taskId + @"\output.txt")), uploadOptions: new OutputFileUploadOptions( uploadCondition: OutputFileUploadCondition.TaskCompletion)), } }); } ); // Add the tasks to the job; the tasks are automatically // scheduled for execution on the nodes by the Batch service. await job.AddTaskAsync(tasksToAdd); Console.WriteLine($"All tasks added to job {job.Id}"); Console.WriteLine(); Console.WriteLine($"Downloading outputs to {Directory.GetCurrentDirectory()}"); foreach (CloudTask task in job.CompletedTasks()) { if (task.ExecutionInformation.Result != TaskExecutionResult.Success) { Console.WriteLine($"Task {task.Id} failed"); Console.WriteLine(SampleHelpers.GetFailureInfoDetails(task.ExecutionInformation.FailureInformation)); } else { Console.WriteLine($"Task {task.Id} completed successfully"); } CloudBlobDirectory directory = container.GetDirectoryReference(task.Id); Directory.CreateDirectory(task.Id); foreach (var blobInDirectory in directory.ListBlobs()) { CloudBlockBlob blockBlob = blobInDirectory as CloudBlockBlob; Console.WriteLine($" {blockBlob.Name}"); await blockBlob.DownloadToFileAsync(blockBlob.Name, FileMode.Create); } } return(container); }
/// <summary> /// Runs a series of tasks, using the OutputFiles feature in conjunction with the file conventions library to upload the tasks to a container. /// Then downloads the files from the container to the local machine. /// </summary> public static async Task <CloudBlobContainer> RunWithConventions( BatchClient batchClient, CloudStorageAccount linkedStorageAccount, string poolId, int nodeCount, string jobId) { await CreatePoolAsync(batchClient, poolId, nodeCount); CloudJob job = batchClient.JobOperations.CreateJob(jobId, new PoolInformation { PoolId = poolId }); // Get the container URL to use string containerName = job.OutputStorageContainerName(); CloudBlobContainer container = linkedStorageAccount.CreateCloudBlobClient().GetContainerReference(containerName); await container.CreateIfNotExistsAsync(); string containerUrl = job.GetOutputStorageContainerUrl(linkedStorageAccount); // Commit the job to the Batch service await job.CommitAsync(); Console.WriteLine($"Created job {jobId}"); // Obtain the bound job from the Batch service await job.RefreshAsync(); // Create a series of simple tasks which dump the task environment to a file and then write random values to a text file IEnumerable <CloudTask> tasksToAdd = Enumerable.Range(1, 20).Select(i => { var taskId = i.ToString().PadLeft(3, '0'); var task = new CloudTask(taskId, "cmd /v:ON /c \"echo off && set && (FOR /L %i IN (1,1,100000) DO (ECHO !RANDOM!)) > output.txt\""); task.WithOutputFile(@"..\std*.txt", containerUrl, TaskOutputKind.TaskLog, OutputFileUploadCondition.TaskCompletion) .WithOutputFile(@"output.txt", containerUrl, TaskOutputKind.TaskOutput, OutputFileUploadCondition.TaskSuccess); return(task); } ); // Add the tasks to the job; the tasks are automatically // scheduled for execution on the nodes by the Batch service. await job.AddTaskAsync(tasksToAdd); Console.WriteLine($"All tasks added to job {job.Id}"); Console.WriteLine(); Console.WriteLine($"Downloading outputs to {Directory.GetCurrentDirectory()}"); foreach (CloudTask task in job.CompletedTasks()) { if (task.ExecutionInformation.Result != TaskExecutionResult.Success) { Console.WriteLine($"Task {task.Id} failed"); Console.WriteLine(SampleHelpers.GetFailureInfoDetails(task.ExecutionInformation.FailureInformation)); } else { Console.WriteLine($"Task {task.Id} completed successfully"); } foreach (OutputFileReference output in task.OutputStorage(linkedStorageAccount).ListOutputs(TaskOutputKind.TaskOutput)) { Console.WriteLine($"output file: {output.FilePath}"); await output.DownloadToFileAsync($"{jobId}-{output.FilePath}", System.IO.FileMode.Create); } } return(container); }
public static async Task <CloudBlobContainer> Run( BatchClient batchClient, CloudStorageAccount linkedStorageAccount, string poolId, int nodeCount, string jobId) { const string appPackageId = "PersistOutputsTask"; const string appPackageVersion = "1.0"; // Create and configure an unbound pool. CloudPool pool = batchClient.PoolOperations.CreatePool( poolId: poolId, virtualMachineSize: "standard_d1_v2", targetDedicatedComputeNodes: nodeCount, cloudServiceConfiguration: new CloudServiceConfiguration(osFamily: "5")); // Specify the application and version to deploy to the compute nodes. You must // first build PersistOutputsTask, then upload it as an application package. // See https://azure.microsoft.com/documentation/articles/batch-application-packages/ pool.ApplicationPackageReferences = new List <ApplicationPackageReference> { new ApplicationPackageReference { ApplicationId = appPackageId, Version = appPackageVersion } }; // Commit the pool to the Batch service await GettingStartedCommon.CreatePoolIfNotExistAsync(batchClient, pool); CloudJob job = batchClient.JobOperations.CreateJob(jobId, new PoolInformation { PoolId = poolId }); // Create the blob storage container for the outputs. await job.PrepareOutputStorageAsync(linkedStorageAccount); // Create an environment variable on the compute nodes that the // task application can reference when persisting its outputs. string containerName = job.OutputStorageContainerName(); CloudBlobContainer container = linkedStorageAccount.CreateCloudBlobClient().GetContainerReference(containerName); string containerUrl = job.GetOutputStorageContainerUrl(linkedStorageAccount); job.CommonEnvironmentSettings = new[] { new EnvironmentSetting("JOB_CONTAINER_URL", containerUrl) }; // Commit the job to the Batch service await job.CommitAsync(); Console.WriteLine($"Created job {jobId}"); // Obtain the bound job from the Batch service await job.RefreshAsync(); IEnumerable <CloudTask> tasks = Enumerable.Range(1, 20).Select(i => new CloudTask(i.ToString().PadLeft(3, '0'), $"cmd /c %AZ_BATCH_APP_PACKAGE_{appPackageId.ToUpper()}#{appPackageVersion}%\\PersistOutputsTask.exe") ); // Add the tasks to the job; the tasks are automatically // scheduled for execution on the nodes by the Batch service. await job.AddTaskAsync(tasks); Console.WriteLine($"All tasks added to job {job.Id}"); Console.WriteLine(); Console.WriteLine($"Downloading outputs to {Directory.GetCurrentDirectory()}"); foreach (CloudTask task in job.CompletedTasks()) { if (task.ExecutionInformation.Result != TaskExecutionResult.Success) { Console.WriteLine($"Task {task.Id} failed"); Console.WriteLine(SampleHelpers.GetFailureInfoDetails(task.ExecutionInformation.FailureInformation)); } else { Console.WriteLine($"Task {task.Id} completed successfully"); } foreach (OutputFileReference output in task.OutputStorage(linkedStorageAccount).ListOutputs(TaskOutputKind.TaskOutput)) { Console.WriteLine($"output file: {output.FilePath}"); await output.DownloadToFileAsync($"{jobId}-{output.FilePath}", System.IO.FileMode.Create); } } return(container); }