Пример #1
0
        /// <inheritdoc/>
        public async Task StartJobWithTask(string taskId, string commandLineText)
        {
            await this.TryInit();

            try
            {
                using (var batchClient = BatchClient.Open(this.credentials))
                {
                    CloudJob job = await batchClient.JobOperations.GetJobAsync(this.jobId);

                    var task = new CloudTask(taskId, commandLineText);
                    task.UserIdentity = new UserIdentity(new AutoUserSpecification(AutoUserScope.Task, ElevationLevel.Admin));

                    await job.AddTaskAsync(task);
                }
            }
            catch (BatchException ex)
            {
                // Accept the specific error code JobExists as that is expected if the job already exists
                if (ex.RequestInformation?.BatchError?.Code == BatchErrorCodeStrings.TaskExists)
                {
                    throw new BrowseCloudServiceException($"The batch task {taskId} already existed when we tried to create it", ex);
                }
                else
                {
                    throw new BrowseCloudServiceException($"An unexpected error occurred while scheduling batch task {taskId}", ex);
                }
            }
        }
        public async Task Job_GetTaskCounts_ReturnsCorrectCountNonZeroTaskSlots()
        {
            Func <Task> test = async() =>
            {
                using (BatchClient batchCli = await TestUtilities.OpenBatchClientFromEnvironmentAsync().ConfigureAwait(false))
                {
                    string jobId = "NonZeroTaskSlots-" + TestUtilities.GetMyName();
                    try
                    {
                        PoolInformation poolInfo = new PoolInformation()
                        {
                            PoolId = "Fake"
                        };

                        CloudJob unboundJob = batchCli.JobOperations.CreateJob(jobId, poolInfo);
                        await unboundJob.CommitAsync().ConfigureAwait(false);

                        await unboundJob.RefreshAsync().ConfigureAwait(false);

                        CloudTask t1 = new CloudTask("t1", "cmd /c dir");
                        t1.RequiredSlots = 2;
                        CloudTask t2 = new CloudTask("t2", "cmd /c ping 127.0.0.1 -n 4");
                        t2.RequiredSlots = 3;

                        await unboundJob.AddTaskAsync(new[] { t1, t2 }).ConfigureAwait(false);

                        await Task.Delay(TimeSpan.FromSeconds(5)).ConfigureAwait(false); // Give the service some time to get the counts

                        var counts = await unboundJob.GetTaskCountsAsync().ConfigureAwait(false);

                        var retTask1 = await unboundJob.GetTaskAsync(t1.Id);

                        Assert.Equal(t1.RequiredSlots, retTask1.RequiredSlots);
                        var retTask2 = await unboundJob.GetTaskAsync(t1.Id);

                        Assert.Equal(t1.RequiredSlots, retTask2.RequiredSlots);

                        Assert.Equal(2, counts.TaskCounts.Active);
                        // Task slots counts is currently broken
                        // Assert.Equal(5, counts.TaskSlotCounts.Active);
                    }
                    finally
                    {
                        await TestUtilities.DeleteJobIfExistsAsync(batchCli, jobId);
                    }
                }
            };

            await SynchronizationContextHelper.RunTestAsync(test, TestTimeout);
        }
Пример #3
0
        public async Task CreateTask(Guid guid, List <BatchBlobUrlInput> inputFiles, List <BatchBlobUrlInput> outputFiles)
        {
            CloudJob job = await batchClient.JobOperations.GetJobAsync("NlpBatchProc");

            CloudTask task = new CloudTask("task_" + guid.ToString(), "cp /tmp/inputBlob.txt /tmp/outputBlob.txt");

            task.ResourceFiles = new List <ResourceFile>();
            foreach (BatchBlobUrlInput x in inputFiles)
            {
                ResourceFile file = new ResourceFile(x.Url, "/tmp/" + x.Part, "0644");
                task.ResourceFiles.Add(file);
            }
            task.OutputFiles = new List <OutputFile>();
            foreach (BatchBlobUrlInput x in outputFiles)
            {
                string outputContainerUrl      = x.Url;
                OutputFileDestination fileDest = new OutputFileDestination(new OutputFileBlobContainerDestination(outputContainerUrl));
                OutputFile            file     = new OutputFile("/tmp/" + x.Part, fileDest, new OutputFileUploadOptions(OutputFileUploadCondition.TaskCompletion));
            }

            await job.AddTaskAsync(task);
        }
        /// <summary>
        /// Performs a simple AddTask test, adding the specified task count using the specified parallelOptions and resultHandlerFunc
        /// </summary>
        /// <returns></returns>
        private async System.Threading.Tasks.Task AddTasksSimpleTestAsync(
            BatchClient batchCli,
            string testName,
            int taskCount,
            BatchClientParallelOptions parallelOptions,
            Func <AddTaskResult, CancellationToken, AddTaskResultStatus> resultHandlerFunc,
            StagingStorageAccount storageCredentials,
            IEnumerable <string> localFilesToStage,
            ConcurrentBag <ConcurrentDictionary <Type, IFileStagingArtifact> > fileStagingArtifacts = null,
            TimeSpan?timeout      = null,
            bool useJobOperations = true)
        {
            JobOperations jobOperations = batchCli.JobOperations;

            string jobId = "Bulk-" + TestUtilities.GetMyName() + "-" + testName + "-" + useJobOperations;

            try
            {
                CloudJob unboundJob = jobOperations.CreateJob();

                this.testOutputHelper.WriteLine("Initial job commit for job: {0}", jobId);
                unboundJob.PoolInformation = new PoolInformation()
                {
                    PoolId = "DummyPool"
                };
                unboundJob.Id = jobId;
                await unboundJob.CommitAsync().ConfigureAwait(false);

                CloudJob boundJob = await jobOperations.GetJobAsync(jobId).ConfigureAwait(false);

                //
                // Add a simple set of tasks
                //
                IEnumerable <string>         taskNames            = GenerateTaskIds(taskCount);
                List <CloudTask>             tasksToAdd           = new List <CloudTask>();
                List <CloudTask>             tasksToValidateWith  = new List <CloudTask>();
                IList <IFileStagingProvider> lastFilesToStageList = null;
                foreach (string taskName in taskNames)
                {
                    CloudTask myTask = new CloudTask(taskName, "cmd /c echo hello world");
                    CloudTask duplicateReadableTask = new CloudTask(taskName, "cmd /c echo hello world");

                    if (localFilesToStage != null && storageCredentials != null)
                    {
                        myTask.FilesToStage = new List <IFileStagingProvider>();

                        lastFilesToStageList = myTask.FilesToStage;

                        duplicateReadableTask.FilesToStage = new List <IFileStagingProvider>();
                        foreach (string fileToStage in localFilesToStage)
                        {
                            duplicateReadableTask.FilesToStage.Add(new FileToStage(fileToStage, storageCredentials));
                            myTask.FilesToStage.Add(new FileToStage(fileToStage, storageCredentials));
                        }
                    }

                    tasksToAdd.Add(myTask);
                    tasksToValidateWith.Add(duplicateReadableTask);
                }

                List <BatchClientBehavior> behaviors = new List <BatchClientBehavior>();
                if (resultHandlerFunc != null)
                {
                    behaviors.Add(new AddTaskCollectionResultHandler(resultHandlerFunc));
                }

                //Add the tasks
                Stopwatch stopwatch = new Stopwatch();
                this.testOutputHelper.WriteLine("Starting task add");
                stopwatch.Start();

                if (useJobOperations)
                {
                    await jobOperations.AddTaskAsync(
                        jobId,
                        tasksToAdd,
                        parallelOptions : parallelOptions,
                        fileStagingArtifacts : fileStagingArtifacts,
                        timeout : timeout,
                        additionalBehaviors : behaviors).ConfigureAwait(continueOnCapturedContext: false);
                }
                else
                {
                    await boundJob.AddTaskAsync(
                        tasksToAdd,
                        parallelOptions : parallelOptions,
                        fileStagingArtifacts : fileStagingArtifacts,
                        timeout : timeout,
                        additionalBehaviors : behaviors).ConfigureAwait(continueOnCapturedContext: false);
                }

                stopwatch.Stop();
                this.testOutputHelper.WriteLine("Task add finished, took: {0}", stopwatch.Elapsed);

                if (lastFilesToStageList != null)
                {
                    Assert.Throws <InvalidOperationException>(() => lastFilesToStageList.Add(new FileToStage("test", null)));
                }

                //Ensure the task lists match
                List <CloudTask> tasksFromService = await jobOperations.ListTasks(jobId).ToListAsync().ConfigureAwait(false);

                EnsureTasksListsMatch(tasksToValidateWith, tasksFromService);
            }
            catch (Exception e)
            {
                this.testOutputHelper.WriteLine("Exception: {0}", e.ToString());
                throw;
            }
            finally
            {
                TestUtilities.DeleteJobIfExistsAsync(batchCli, jobId).Wait();
            }
        }
Пример #5
0
        /// <summary>
        /// Runs a series of tasks, using the OutputFiles feature to upload the tasks files to a container.
        /// Then downloads the files from the container to the local machine.
        /// </summary>
        public static async Task <CloudBlobContainer> Run(
            BatchClient batchClient,
            CloudStorageAccount storageAccount,
            string poolId,
            int nodeCount,
            string jobId)
        {
            const string containerName = "outputfilescontainer";

            await CreatePoolAsync(batchClient, poolId, nodeCount);

            CloudJob job = batchClient.JobOperations.CreateJob(jobId, new PoolInformation {
                PoolId = poolId
            });

            CloudBlobContainer container = storageAccount.CreateCloudBlobClient().GetContainerReference(containerName);
            await container.CreateIfNotExistsAsync();

            string containerSas = container.GetSharedAccessSignature(new SharedAccessBlobPolicy()
            {
                Permissions            = SharedAccessBlobPermissions.Write,
                SharedAccessExpiryTime = DateTimeOffset.UtcNow.AddDays(1)
            });
            string containerUrl = container.Uri.AbsoluteUri + containerSas;

            // Commit the job to the Batch service
            await job.CommitAsync();

            Console.WriteLine($"Created job {jobId}");

            // Obtain the bound job from the Batch service
            await job.RefreshAsync();

            // Create a series of simple tasks which dump the task environment to a file and then write random values to a text file
            IEnumerable <CloudTask> tasksToAdd = Enumerable.Range(1, 20).Select(i =>
            {
                var taskId = i.ToString().PadLeft(3, '0');
                return(new CloudTask(taskId, "cmd /v:ON /c \"echo off && set && (FOR /L %i IN (1,1,100000) DO (ECHO !RANDOM!)) > output.txt\"")
                {
                    OutputFiles = new List <OutputFile>
                    {
                        new OutputFile(
                            filePattern: @"..\std*.txt",
                            destination: new OutputFileDestination(new OutputFileBlobContainerDestination(
                                                                       containerUrl: containerUrl,
                                                                       path: taskId)),
                            uploadOptions: new OutputFileUploadOptions(
                                uploadCondition: OutputFileUploadCondition.TaskCompletion)),
                        new OutputFile(
                            filePattern: @"output.txt",
                            destination: new OutputFileDestination(new OutputFileBlobContainerDestination(
                                                                       containerUrl: containerUrl,
                                                                       path: taskId + @"\output.txt")),
                            uploadOptions: new OutputFileUploadOptions(
                                uploadCondition: OutputFileUploadCondition.TaskCompletion)),
                    }
                });
            }
                                                                                );

            // Add the tasks to the job; the tasks are automatically
            // scheduled for execution on the nodes by the Batch service.
            await job.AddTaskAsync(tasksToAdd);

            Console.WriteLine($"All tasks added to job {job.Id}");
            Console.WriteLine();

            Console.WriteLine($"Downloading outputs to {Directory.GetCurrentDirectory()}");

            foreach (CloudTask task in job.CompletedTasks())
            {
                if (task.ExecutionInformation.Result != TaskExecutionResult.Success)
                {
                    Console.WriteLine($"Task {task.Id} failed");
                    Console.WriteLine(SampleHelpers.GetFailureInfoDetails(task.ExecutionInformation.FailureInformation));
                }
                else
                {
                    Console.WriteLine($"Task {task.Id} completed successfully");
                }

                CloudBlobDirectory directory = container.GetDirectoryReference(task.Id);
                Directory.CreateDirectory(task.Id);
                foreach (var blobInDirectory in directory.ListBlobs())
                {
                    CloudBlockBlob blockBlob = blobInDirectory as CloudBlockBlob;
                    Console.WriteLine($"  {blockBlob.Name}");
                    await blockBlob.DownloadToFileAsync(blockBlob.Name, FileMode.Create);
                }
            }

            return(container);
        }
Пример #6
0
        /// <summary>
        /// Runs a series of tasks, using the OutputFiles feature in conjunction with the file conventions library to upload the tasks to a container.
        /// Then downloads the files from the container to the local machine.
        /// </summary>
        public static async Task <CloudBlobContainer> RunWithConventions(
            BatchClient batchClient,
            CloudStorageAccount linkedStorageAccount,
            string poolId,
            int nodeCount,
            string jobId)
        {
            await CreatePoolAsync(batchClient, poolId, nodeCount);

            CloudJob job = batchClient.JobOperations.CreateJob(jobId, new PoolInformation {
                PoolId = poolId
            });

            // Get the container URL to use
            string             containerName = job.OutputStorageContainerName();
            CloudBlobContainer container     = linkedStorageAccount.CreateCloudBlobClient().GetContainerReference(containerName);
            await container.CreateIfNotExistsAsync();

            string containerUrl = job.GetOutputStorageContainerUrl(linkedStorageAccount);

            // Commit the job to the Batch service
            await job.CommitAsync();

            Console.WriteLine($"Created job {jobId}");

            // Obtain the bound job from the Batch service
            await job.RefreshAsync();

            // Create a series of simple tasks which dump the task environment to a file and then write random values to a text file
            IEnumerable <CloudTask> tasksToAdd = Enumerable.Range(1, 20).Select(i =>
            {
                var taskId = i.ToString().PadLeft(3, '0');
                var task   = new CloudTask(taskId, "cmd /v:ON /c \"echo off && set && (FOR /L %i IN (1,1,100000) DO (ECHO !RANDOM!)) > output.txt\"");

                task.WithOutputFile(@"..\std*.txt", containerUrl, TaskOutputKind.TaskLog, OutputFileUploadCondition.TaskCompletion)
                .WithOutputFile(@"output.txt", containerUrl, TaskOutputKind.TaskOutput, OutputFileUploadCondition.TaskSuccess);

                return(task);
            }
                                                                                );

            // Add the tasks to the job; the tasks are automatically
            // scheduled for execution on the nodes by the Batch service.
            await job.AddTaskAsync(tasksToAdd);

            Console.WriteLine($"All tasks added to job {job.Id}");
            Console.WriteLine();

            Console.WriteLine($"Downloading outputs to {Directory.GetCurrentDirectory()}");

            foreach (CloudTask task in job.CompletedTasks())
            {
                if (task.ExecutionInformation.Result != TaskExecutionResult.Success)
                {
                    Console.WriteLine($"Task {task.Id} failed");
                    Console.WriteLine(SampleHelpers.GetFailureInfoDetails(task.ExecutionInformation.FailureInformation));
                }
                else
                {
                    Console.WriteLine($"Task {task.Id} completed successfully");
                }

                foreach (OutputFileReference output in task.OutputStorage(linkedStorageAccount).ListOutputs(TaskOutputKind.TaskOutput))
                {
                    Console.WriteLine($"output file: {output.FilePath}");
                    await output.DownloadToFileAsync($"{jobId}-{output.FilePath}", System.IO.FileMode.Create);
                }
            }

            return(container);
        }
Пример #7
0
        public static async Task <CloudBlobContainer> Run(
            BatchClient batchClient,
            CloudStorageAccount linkedStorageAccount,
            string poolId,
            int nodeCount,
            string jobId)
        {
            const string appPackageId      = "PersistOutputsTask";
            const string appPackageVersion = "1.0";

            // Create and configure an unbound pool.
            CloudPool pool = batchClient.PoolOperations.CreatePool(
                poolId: poolId,
                virtualMachineSize: "standard_d1_v2",
                targetDedicatedComputeNodes: nodeCount,
                cloudServiceConfiguration: new CloudServiceConfiguration(osFamily: "5"));

            // Specify the application and version to deploy to the compute nodes. You must
            // first build PersistOutputsTask, then upload it as an application package.
            // See https://azure.microsoft.com/documentation/articles/batch-application-packages/
            pool.ApplicationPackageReferences = new List <ApplicationPackageReference>
            {
                new ApplicationPackageReference
                {
                    ApplicationId = appPackageId,
                    Version       = appPackageVersion
                }
            };

            // Commit the pool to the Batch service
            await GettingStartedCommon.CreatePoolIfNotExistAsync(batchClient, pool);

            CloudJob job = batchClient.JobOperations.CreateJob(jobId, new PoolInformation {
                PoolId = poolId
            });

            // Create the blob storage container for the outputs.
            await job.PrepareOutputStorageAsync(linkedStorageAccount);

            // Create an environment variable on the compute nodes that the
            // task application can reference when persisting its outputs.
            string             containerName = job.OutputStorageContainerName();
            CloudBlobContainer container     = linkedStorageAccount.CreateCloudBlobClient().GetContainerReference(containerName);
            string             containerUrl  = job.GetOutputStorageContainerUrl(linkedStorageAccount);

            job.CommonEnvironmentSettings = new[] { new EnvironmentSetting("JOB_CONTAINER_URL", containerUrl) };

            // Commit the job to the Batch service
            await job.CommitAsync();

            Console.WriteLine($"Created job {jobId}");

            // Obtain the bound job from the Batch service
            await job.RefreshAsync();

            IEnumerable <CloudTask> tasks = Enumerable.Range(1, 20).Select(i =>
                                                                           new CloudTask(i.ToString().PadLeft(3, '0'), $"cmd /c %AZ_BATCH_APP_PACKAGE_{appPackageId.ToUpper()}#{appPackageVersion}%\\PersistOutputsTask.exe")
                                                                           );

            // Add the tasks to the job; the tasks are automatically
            // scheduled for execution on the nodes by the Batch service.
            await job.AddTaskAsync(tasks);

            Console.WriteLine($"All tasks added to job {job.Id}");
            Console.WriteLine();

            Console.WriteLine($"Downloading outputs to {Directory.GetCurrentDirectory()}");

            foreach (CloudTask task in job.CompletedTasks())
            {
                if (task.ExecutionInformation.Result != TaskExecutionResult.Success)
                {
                    Console.WriteLine($"Task {task.Id} failed");
                    Console.WriteLine(SampleHelpers.GetFailureInfoDetails(task.ExecutionInformation.FailureInformation));
                }
                else
                {
                    Console.WriteLine($"Task {task.Id} completed successfully");
                }

                foreach (OutputFileReference output in task.OutputStorage(linkedStorageAccount).ListOutputs(TaskOutputKind.TaskOutput))
                {
                    Console.WriteLine($"output file: {output.FilePath}");
                    await output.DownloadToFileAsync($"{jobId}-{output.FilePath}", System.IO.FileMode.Create);
                }
            }

            return(container);
        }