/// <summary> /// Submit a large number of tasks to the Batch Service. /// </summary> /// <param name="client">The batch client.</param> /// <param name="sharedPoolId">The ID of the pool to use for the job</param> private static void SubmitLargeNumberOfTasks(BatchClient client, string sharedPoolId) { const int taskCountToCreate = 5000; // In order to simulate a "large" task object which has many properties set (such as resource files, environment variables, etc) // we create a big environment variable so we have a big task object. char[] env = new char[2048]; for (int i = 0; i < env.Length; i++) { env[i] = 'a'; } string envStr = new string(env); string jobId = CreateJobId("HelloWorldLargeTaskCountJob"); Console.WriteLine("Creating job: " + jobId); CloudJob boundJob = CreateBoundJob(client.JobOperations, sharedPoolId, jobId); //Generate a large number of tasks to submit List <CloudTask> tasksToSubmit = new List <CloudTask>(taskCountToCreate); for (int i = 0; i < taskCountToCreate; i++) { CloudTask task = new CloudTask("echo" + i.ToString("D5"), "echo"); List <EnvironmentSetting> environmentSettings = new List <EnvironmentSetting>(); environmentSettings.Add(new EnvironmentSetting("envone", envStr)); task.EnvironmentSettings = environmentSettings; tasksToSubmit.Add(task); } BatchClientParallelOptions parallelOptions = new BatchClientParallelOptions() { //This will result in at most 10 simultaneous Bulk Add requests to the Batch Service. MaxDegreeOfParallelism = 10 }; Console.WriteLine("Submitting {0} tasks to job: {1}, on pool: {2}", taskCountToCreate, boundJob.Id, boundJob.ExecutionInformation.PoolId); Stopwatch stopwatch = new Stopwatch(); stopwatch.Start(); // Use the AddTask overload which supports a list of tasks for best AddTask performence - internally this method performs a // submission of multiple tasks in one REST API request in order to limit the number of calls made to the Batch Service. client.JobOperations.AddTask(boundJob.Id, tasksToSubmit, parallelOptions); stopwatch.Stop(); Console.WriteLine("Submitted {0} tasks in {1}", taskCountToCreate, stopwatch.Elapsed); //Delete the job to ensure the tasks are cleaned up Console.WriteLine("Deleting job: {0}", boundJob.Id); client.JobOperations.DeleteJob(boundJob.Id); }
public async Task Bug1360227_AddTasksBatchFailure(bool useJobOperations) { const string testName = "Bug1360227_AddTasksBatchFailure"; int count = 0; const int countToFailAt = 102; const int taskCount = 407; HashSet <string> taskIdsExpectedToFail = new HashSet <string>(); Func <AddTaskResult, CancellationToken, AddTaskResultStatus> resultHandlerFunc = (result, token) => { this.testOutputHelper.WriteLine("Task: {0} got status code: {1}", result.TaskId, result.Status); ++count; if (taskIdsExpectedToFail.Contains(result.TaskId)) { return(AddTaskResultStatus.Retry); } else { if (count >= countToFailAt) { taskIdsExpectedToFail.Add(result.TaskId); this.testOutputHelper.WriteLine("Forcing a failure"); //Throw an exception to cause a failure from the customers result handler -- this is a supported scenario which will //terminate the add task operation throw new HttpRequestException("Test"); } else { return(AddTaskResultStatus.Success); } } }; await SynchronizationContextHelper.RunTestAsync(async() => { using (BatchClient batchCli = await TestUtilities.OpenBatchClientFromEnvironmentAsync()) { BatchClientParallelOptions parallelOptions = new BatchClientParallelOptions() { MaxDegreeOfParallelism = 2 }; var exception = await TestUtilities.AssertThrowsAsync <ParallelOperationsException>( async() => await this.AddTasksSimpleTestAsync( batchCli, testName, taskCount, parallelOptions, resultHandlerFunc, useJobOperations: useJobOperations).ConfigureAwait(false)).ConfigureAwait(false); Assert.IsType <HttpRequestException>(exception.InnerException); } }, TestTimeout); }
public async Task Bug1360227_AddTasksBatchRequestFailure(bool useJobOperations) { const string testName = "Bug1360227_AddTasksBatchRequestFailure"; Random rand = new Random(); object randLock = new object(); BatchClientBehavior customBehavior = new Protocol.RequestInterceptor(request => { var typedRequest = request as Protocol.BatchRequests.TaskAddCollectionBatchRequest; if (typedRequest != null) { var originalServiceRequestFunction = typedRequest.ServiceRequestFunc; typedRequest.ServiceRequestFunc = token => { lock (randLock) { double d = rand.NextDouble(); if (d > 0.3) { throw new HttpRequestException("Simulating a network problem"); } else { return(originalServiceRequestFunction(token)); } } }; } }); await SynchronizationContextHelper.RunTestAsync(async() => { using (BatchClient batchCli = await TestUtilities.OpenBatchClientAsync(TestUtilities.GetCredentialsFromEnvironment(), addDefaultRetryPolicy: false)) { batchCli.JobOperations.CustomBehaviors.Add(customBehavior); BatchClientParallelOptions parallelOptions = new BatchClientParallelOptions() { MaxDegreeOfParallelism = 2 }; var exception = await TestUtilities.AssertThrowsAsync <ParallelOperationsException>(async() => await this.AddTasksSimpleTestAsync(batchCli, testName, 397, parallelOptions, useJobOperations: useJobOperations).ConfigureAwait(false) ).ConfigureAwait(false); Assert.IsType <HttpRequestException>(exception.InnerException); } }, TestTimeout); }
public async Task Bug1360227_AddTasksBatchConfirmResultHandlerTaskReadOnly() { const string testName = "Bug1360227_ConfirmResultHandlerTaskReadOnly"; Func <AddTaskResult, CancellationToken, AddTaskResultStatus> resultHandlerFunc = (result, token) => { //Count everything as a success AddTaskResultStatus resultAction = AddTaskResultStatus.Success; //Try to set a property of the cloud task InvalidOperationException e = TestUtilities.AssertThrows <InvalidOperationException>(() => result.Task.Constraints = new TaskConstraints(TimeSpan.FromSeconds(5), null, null)); Assert.Contains("Write access is not allowed.", e.Message); //Try to call a method of a CloudTask //TODO: This should be blocked but isn't right now... //try //{ // result.Task.Terminate(); // Debug.Fail("Should not have gotten here"); //} //catch (Exception e) //{ // Console.WriteLine(e); // //Swallow this exception as it is expected //} return(resultAction); }; await SynchronizationContextHelper.RunTestAsync(async() => { using (BatchClient batchCli = await TestUtilities.OpenBatchClientFromEnvironmentAsync()) { BatchClientParallelOptions parallelOptions = new BatchClientParallelOptions() { MaxDegreeOfParallelism = 2 }; await this.AddTasksSimpleTestAsync( batchCli, testName, 55, parallelOptions, resultHandlerFunc).ConfigureAwait(false); } }, TestTimeout); }
public async Task Bug1360227_AddTasksBatchHugeTaskCount() { const string testName = "Bug1360227_AddTasksBatchHugeTaskCount"; await SynchronizationContextHelper.RunTestAsync(async() => { using BatchClient batchCli = await TestUtilities.OpenBatchClientFromEnvironmentAsync(); BatchClientParallelOptions parallelOptions = new BatchClientParallelOptions() { MaxDegreeOfParallelism = 25 }; await AddTasksSimpleTestAsync(batchCli, testName, 5025, parallelOptions).ConfigureAwait(false); }, LongTestTimeout); }
public async Task Bug1360227_AddTasksBatchCancelation(bool useJobOperations) { const string testName = "Bug1360227_AddTasksBatchCancelation"; const int taskCount = 322; await SynchronizationContextHelper.RunTestAsync(async() => { using (BatchClient batchCli = await TestUtilities.OpenBatchClientFromEnvironmentAsync()) { using (CancellationTokenSource source = new CancellationTokenSource()) { BatchClientParallelOptions parallelOptions = new BatchClientParallelOptions() { MaxDegreeOfParallelism = 2, CancellationToken = source.Token }; System.Threading.Tasks.Task t = this.AddTasksSimpleTestAsync( batchCli, testName, taskCount, parallelOptions, useJobOperations: useJobOperations); Thread.Sleep(TimeSpan.FromSeconds(.3)); //Wait till we get into the workflow this.testOutputHelper.WriteLine("Canceling the work flow"); source.Cancel(); try { await t.ConfigureAwait(false); } catch (Exception e) { //This is expected to throw one of two possible exception types... if (!(e is TaskCanceledException) && !(e is OperationCanceledException)) { throw new ThrowsException(typeof(TaskCanceledException), e); } } } } }, TestTimeout); }
public async Task AddTasksRequestEntityTooLarge_ReduceChunkSize() { const string testName = "AddTasksRequestEntityTooLarge_ReduceChunkSize"; List <ResourceFile> resourceFiles = new List <ResourceFile>(); ResourceFile resourceFile; int countChunksOf100 = 0; int numTasks = 176; int degreesOfParallelism = 2; BatchClientBehavior customBehavior = new Protocol.RequestInterceptor(request => { var typedRequest = request as Protocol.BatchRequests.TaskAddCollectionBatchRequest; if (typedRequest != null) { if (typedRequest.Parameters.Count > 50) { Interlocked.Increment(ref countChunksOf100); } } }); // If this test fails try increasing the size of the Task in case maximum size increase for (int i = 0; i < 100; i++) { resourceFile = ResourceFile.FromUrl("https://mystorageaccount.blob.core.windows.net/files/resourceFile" + i, "resourceFile" + i); resourceFiles.Add(resourceFile); } await SynchronizationContextHelper.RunTestAsync(async() => { using (BatchClient batchCli = TestUtilities.OpenBatchClient(TestUtilities.GetCredentialsFromEnvironment(), addDefaultRetryPolicy: false)) { batchCli.JobOperations.CustomBehaviors.Add(customBehavior); BatchClientParallelOptions parallelOptions = new BatchClientParallelOptions() { MaxDegreeOfParallelism = degreesOfParallelism }; await AddTasksSimpleTestAsync(batchCli, testName, numTasks, parallelOptions, resourceFiles: resourceFiles).ConfigureAwait(false); } }, TestTimeout); Assert.True(countChunksOf100 <= Math.Min(Math.Ceiling(numTasks / 100.0), degreesOfParallelism)); }
private void SubmitTasks(Guid jobId, CancellationToken ct) { summary += $"Starting to submit tasks - {DateTime.UtcNow}\n"; var attempts = 0; //while (attempts++ < 20) //{ if (ct.IsCancellationRequested) { return; } //try //{ BatchClientParallelOptions parallelOptions = new BatchClientParallelOptions() { CancellationToken = ct, MaxDegreeOfParallelism = 4, }; summary += "Fetching tasks to submit...\n"; _batchClient.JobOperations.AddTaskAsync(jobId.ToString(), GetTasksToSubmit(jobId), parallelOptions).Wait(ct); summary += _taskProvider.Output; summary += $"Starting to submit tasks - {DateTime.UtcNow}\n"; // break; //} //catch (AggregateException e) //{ // Console.WriteLine("An error occurred submitting tasks: {0}", e.InnerException); //} //catch (Exception e) //{ // Console.WriteLine("An error occurred submitting tasks: {0}", e); //} ct.WaitHandle.WaitOne(TimeSpan.FromSeconds(5)); //} }
private void SubmitTasks(Guid jobId, CancellationToken ct) { Console.WriteLine("Starting to submit tasks - {0}", DateTime.UtcNow); var attempts = 0; while (attempts++ < 20) { if (ct.IsCancellationRequested) { return; } try { BatchClientParallelOptions parallelOptions = new BatchClientParallelOptions() { CancellationToken = ct, MaxDegreeOfParallelism = 4, }; _batchClient.JobOperations.AddTaskAsync(jobId.ToString(), GetTasksToSubmit(jobId), parallelOptions).Wait(ct); Console.WriteLine("Starting to submit tasks - {0}", DateTime.UtcNow); break; } catch (AggregateException e) { Console.WriteLine("An error occurred submitting tasks: {0}", e.InnerException); } catch (Exception e) { Console.WriteLine("An error occurred submitting tasks: {0}", e); } ct.WaitHandle.WaitOne(TimeSpan.FromSeconds(5)); } }
/// <summary> /// Performs a simple AddTask test, adding the specified task count using the specified parallelOptions and resultHandlerFunc /// </summary> /// <returns></returns> private async System.Threading.Tasks.Task AddTasksSimpleTestAsync( BatchClient batchCli, string testName, int taskCount, BatchClientParallelOptions parallelOptions, Func <AddTaskResult, CancellationToken, AddTaskResultStatus> resultHandlerFunc, StagingStorageAccount storageCredentials, IEnumerable <string> localFilesToStage, ConcurrentBag <ConcurrentDictionary <Type, IFileStagingArtifact> > fileStagingArtifacts = null, TimeSpan?timeout = null, bool useJobOperations = true) { JobOperations jobOperations = batchCli.JobOperations; string jobId = "Bulk-" + TestUtilities.GetMyName() + "-" + testName + "-" + useJobOperations; try { CloudJob unboundJob = jobOperations.CreateJob(); this.testOutputHelper.WriteLine("Initial job commit for job: {0}", jobId); unboundJob.PoolInformation = new PoolInformation() { PoolId = "DummyPool" }; unboundJob.Id = jobId; await unboundJob.CommitAsync().ConfigureAwait(false); CloudJob boundJob = await jobOperations.GetJobAsync(jobId).ConfigureAwait(false); // // Add a simple set of tasks // IEnumerable <string> taskNames = GenerateTaskIds(taskCount); List <CloudTask> tasksToAdd = new List <CloudTask>(); List <CloudTask> tasksToValidateWith = new List <CloudTask>(); IList <IFileStagingProvider> lastFilesToStageList = null; foreach (string taskName in taskNames) { CloudTask myTask = new CloudTask(taskName, "cmd /c echo hello world"); CloudTask duplicateReadableTask = new CloudTask(taskName, "cmd /c echo hello world"); if (localFilesToStage != null && storageCredentials != null) { myTask.FilesToStage = new List <IFileStagingProvider>(); lastFilesToStageList = myTask.FilesToStage; duplicateReadableTask.FilesToStage = new List <IFileStagingProvider>(); foreach (string fileToStage in localFilesToStage) { duplicateReadableTask.FilesToStage.Add(new FileToStage(fileToStage, storageCredentials)); myTask.FilesToStage.Add(new FileToStage(fileToStage, storageCredentials)); } } tasksToAdd.Add(myTask); tasksToValidateWith.Add(duplicateReadableTask); } List <BatchClientBehavior> behaviors = new List <BatchClientBehavior>(); if (resultHandlerFunc != null) { behaviors.Add(new AddTaskCollectionResultHandler(resultHandlerFunc)); } //Add the tasks Stopwatch stopwatch = new Stopwatch(); this.testOutputHelper.WriteLine("Starting task add"); stopwatch.Start(); if (useJobOperations) { await jobOperations.AddTaskAsync( jobId, tasksToAdd, parallelOptions : parallelOptions, fileStagingArtifacts : fileStagingArtifacts, timeout : timeout, additionalBehaviors : behaviors).ConfigureAwait(continueOnCapturedContext: false); } else { await boundJob.AddTaskAsync( tasksToAdd, parallelOptions : parallelOptions, fileStagingArtifacts : fileStagingArtifacts, timeout : timeout, additionalBehaviors : behaviors).ConfigureAwait(continueOnCapturedContext: false); } stopwatch.Stop(); this.testOutputHelper.WriteLine("Task add finished, took: {0}", stopwatch.Elapsed); if (lastFilesToStageList != null) { Assert.Throws <InvalidOperationException>(() => lastFilesToStageList.Add(new FileToStage("test", null))); } //Ensure the task lists match List <CloudTask> tasksFromService = await jobOperations.ListTasks(jobId).ToListAsync().ConfigureAwait(false); EnsureTasksListsMatch(tasksToValidateWith, tasksFromService); } catch (Exception e) { this.testOutputHelper.WriteLine("Exception: {0}", e.ToString()); throw; } finally { TestUtilities.DeleteJobIfExistsAsync(batchCli, jobId).Wait(); } }
public async Task Bug1360227_AddTasksBatchRetry(bool useJobOperations) { const string testName = "Bug1360227_AddTasksBatchRetry"; Random rand = new Random(); object randLock = new object(); int numberOfTasksWhichHitClientError = 0; int numberOfTasksWhichWereForcedToRetry = 0; Func <AddTaskResult, CancellationToken, AddTaskResultStatus> resultHandlerFunc = (result, token) => { this.testOutputHelper.WriteLine("Task: {0} got status code: {1}", result.TaskId, result.Status); AddTaskResultStatus resultAction; if (result.Status == AddTaskStatus.ClientError) { ++numberOfTasksWhichHitClientError; return(AddTaskResultStatus.Success); //Have to count client error as success } lock (randLock) { double d = rand.NextDouble(); if (d > 0.8) { this.testOutputHelper.WriteLine("Forcing retry for task: {0}", result.TaskId); resultAction = AddTaskResultStatus.Retry; ++numberOfTasksWhichWereForcedToRetry; } else { resultAction = AddTaskResultStatus.Success; } } return(resultAction); }; await SynchronizationContextHelper.RunTestAsync(async() => { StagingStorageAccount storageCredentials = TestUtilities.GetStorageCredentialsFromEnvironment(); using (BatchClient batchCli = await TestUtilities.OpenBatchClientFromEnvironmentAsync()) { BatchClientParallelOptions parallelOptions = new BatchClientParallelOptions() { MaxDegreeOfParallelism = 2 }; await this.AddTasksSimpleTestAsync( batchCli, testName, 1281, parallelOptions, resultHandlerFunc, storageCredentials, new List <string> { "TestResources\\Data.txt" }, useJobOperations: useJobOperations).ConfigureAwait(false); } }, LongTestTimeout); //Ensure that we forced some tasks to retry this.testOutputHelper.WriteLine("Forced a total of {0} tasks to retry", numberOfTasksWhichWereForcedToRetry); Assert.True(numberOfTasksWhichWereForcedToRetry > 0); Assert.Equal(numberOfTasksWhichWereForcedToRetry, numberOfTasksWhichHitClientError); }
/// <summary> /// Submit a large number of tasks to the Batch Service. /// </summary> /// <param name="client">The batch client.</param> /// <param name="sharedPoolId">The ID of the pool to use for the job</param> private static void SubmitLargeNumberOfTasks(BatchClient client, string sharedPoolId) { const int taskCountToCreate = 5000; // In order to simulate a "large" task object which has many properties set (such as resource files, environment variables, etc) // we create a big environment variable so we have a big task object. char[] env = new char[2048]; for (int i = 0; i < env.Length; i++) { env[i] = 'a'; } string envStr = new string(env); string jobId = CreateJobId("HelloWorldLargeTaskCountJob"); Console.WriteLine("Creating job: " + jobId); CloudJob boundJob = CreateBoundJob(client.JobOperations, sharedPoolId, jobId); //Generate a large number of tasks to submit List<CloudTask> tasksToSubmit = new List<CloudTask>(taskCountToCreate); for (int i = 0; i < taskCountToCreate; i++) { CloudTask task = new CloudTask("echo" + i.ToString("D5"), "echo"); List<EnvironmentSetting> environmentSettings = new List<EnvironmentSetting>(); environmentSettings.Add(new EnvironmentSetting("envone", envStr)); task.EnvironmentSettings = environmentSettings; tasksToSubmit.Add(task); } BatchClientParallelOptions parallelOptions = new BatchClientParallelOptions() { //This will result in at most 10 simultaneous Bulk Add requests to the Batch Service. MaxDegreeOfParallelism = 10 }; Console.WriteLine("Submitting {0} tasks to job: {1}, on pool: {2}", taskCountToCreate, boundJob.Id, boundJob.ExecutionInformation.PoolId); Stopwatch stopwatch = new Stopwatch(); stopwatch.Start(); // Use the AddTask overload which supports a list of tasks for best AddTask performence - internally this method performs a // submission of multiple tasks in one REST API request in order to limit the number of calls made to the Batch Service. client.JobOperations.AddTask(boundJob.Id, tasksToSubmit, parallelOptions); stopwatch.Stop(); Console.WriteLine("Submitted {0} tasks in {1}", taskCountToCreate, stopwatch.Elapsed); //Delete the job to ensure the tasks are cleaned up Console.WriteLine("Deleting job: {0}", boundJob.Id); client.JobOperations.DeleteJob(boundJob.Id); }
/// <summary> /// Creates a job and adds a task to it. /// </summary> /// <param name="batchClient">The BatchClient to use when interacting with the Batch service.</param> /// <param name="configurationSettings">The configuration settings</param> /// <param name="jobId">The ID of the job.</param> /// <returns>An asynchronous <see cref="Task"/> representing the operation.</returns> private static async Task SubmitJobAsync(BatchClient batchClient, Settings configurationSettings, AccountSettings accountSettings, string jobId) { // create an empty unbound Job CloudJob unboundJob = batchClient.JobOperations.CreateJob(); unboundJob.Id = jobId; unboundJob.CommonEnvironmentSettings = new List <EnvironmentSetting> { new EnvironmentSetting("AZURE_STORAGE_CONNECTION_STRING", accountSettings.StorageConnectionString) }; // For this job, ask the Batch service to automatically create a pool of VMs when the job is submitted. unboundJob.PoolInformation = new PoolInformation() { AutoPoolSpecification = new AutoPoolSpecification() { AutoPoolIdPrefix = "HelloWorld", PoolSpecification = new PoolSpecification() { TargetDedicatedComputeNodes = configurationSettings.PoolTargetNodeCount, VirtualMachineConfiguration = new VirtualMachineConfiguration( new ImageReference( "UbuntuServer", "Canonical", "18.04-LTS", "latest" ), "batch.node.ubuntu 18.04" ), TaskSlotsPerNode = configurationSettings.TaskSlotsPerNode, TaskSchedulingPolicy = new TaskSchedulingPolicy(ComputeNodeFillType.Spread), VirtualMachineSize = configurationSettings.PoolNodeVirtualMachineSize, ApplicationPackageReferences = new List <ApplicationPackageReference> { new ApplicationPackageReference { ApplicationId = configurationSettings.ApplicationId, Version = configurationSettings.ApplicationVersion } }, }, KeepAlive = configurationSettings.PoolKeepAlive, PoolLifetimeOption = PoolLifetimeOption.Job } }; // Commit Job to create it in the service await unboundJob.CommitAsync(); // create a simple task. Each task within a job must have a unique ID //await batchClient.JobOperations.AddTaskAsync(jobId, new CloudTask("task-env", "env")); //await batchClient.JobOperations.AddTaskAsync(jobId, new CloudTask("task-hostname", "/bin/sh -c \"hostname\"")); // Task could should equal poolTargetNodeCount * # of cores in VM SKU //TODO https://docs.microsoft.com/en-us/azure/batch/large-number-tasks#example-batch-net // Create list of CloudTasks and addTaskAysnc in single operation. int i; List <CloudTask> tasksToAdd = new List <CloudTask>(); // Populate with your tasks //string cmd = "/bin/sh -c \"$AZ_BATCH_APP_PACKAGE_sleeper_1_2/sleeper-queue consume\""; for (i = 0; i < configurationSettings.TaskCount; i++) { tasksToAdd.Add(new CloudTask($"{configurationSettings.ApplicationId}-{i}", configurationSettings.TaskCommand)); } // https://docs.microsoft.com/en-us/azure/batch/large-number-tasks#example-batch-net BatchClientParallelOptions parallelOptions = new BatchClientParallelOptions() { MaxDegreeOfParallelism = configurationSettings.MaxDegreeOfParallelism }; await batchClient.JobOperations.AddTaskAsync(jobId, tasksToAdd, parallelOptions); }
/// <summary> /// Submit a large number of tasks to the Batch Service. /// </summary> /// <param name="client">The batch client.</param> private static void SubmitLargeNumberOfTasks(IBatchClient client) { const int taskCountToCreate = 5000; // In order to simulate a "large" task object which has many properties set (such as resource files, environment variables, etc) // we create a big environment variable so we have a big task object. char[] env = new char[2048]; for (int i = 0; i < env.Length; i++) { env[i] = 'a'; } string envStr = new string(env); using (IWorkItemManager wm = client.OpenWorkItemManager()) { //Create a work item string workItemName = Environment.GetEnvironmentVariable("USERNAME") + DateTime.Now.ToString("yyyyMMdd-HHmmss"); Console.WriteLine("Creating work item {0}", workItemName); ICloudWorkItem cloudWorkItem = wm.CreateWorkItem(workItemName); cloudWorkItem.JobExecutionEnvironment = new JobExecutionEnvironment() { PoolName = PoolName }; //Specify the pool to run on cloudWorkItem.Commit(); //Wait for an active job TimeSpan maxJobCreationTimeout = TimeSpan.FromSeconds(90); DateTime jobCreationStartTime = DateTime.Now; DateTime jobCreationTimeoutTime = jobCreationStartTime.Add(maxJobCreationTimeout); cloudWorkItem = wm.GetWorkItem(workItemName); Console.WriteLine("Waiting for a job to become active..."); while (cloudWorkItem.ExecutionInformation == null || cloudWorkItem.ExecutionInformation.RecentJob == null) { cloudWorkItem.Refresh(); if (DateTime.Now > jobCreationTimeoutTime) { throw new Exception("Timed out waiting for job."); } Thread.Sleep(TimeSpan.FromSeconds(5)); } string jobName = cloudWorkItem.ExecutionInformation.RecentJob.Name; Console.WriteLine("Found job {0}. Adding task objects...", jobName); //Generate a large number of tasks to submit List <ICloudTask> tasksToSubmit = new List <ICloudTask>(); for (int i = 0; i < taskCountToCreate; i++) { ICloudTask task = new CloudTask("echo" + i.ToString("D5"), "echo"); List <IEnvironmentSetting> environmentSettings = new List <IEnvironmentSetting>(); environmentSettings.Add(new EnvironmentSetting("envone", envStr)); task.EnvironmentSettings = environmentSettings; tasksToSubmit.Add(task); } BatchClientParallelOptions parallelOptions = new BatchClientParallelOptions() { //This will result in at most 10 simultaneous Bulk Add requests to the Batch Service. MaxDegreeOfParallelism = 10 }; Console.WriteLine("Submitting {0} tasks to work item: {1}, job: {2}, on pool: {3}", taskCountToCreate, cloudWorkItem.Name, jobName, cloudWorkItem.JobExecutionEnvironment.PoolName); Stopwatch stopwatch = new Stopwatch(); stopwatch.Start(); //Use the AddTask overload which supports a list of tasks for best AddTask performence - internally this method performs an //intelligent submission of tasks in batches in order to limit the number of REST API calls made to the Batch Service. wm.AddTask(cloudWorkItem.Name, jobName, tasksToSubmit, parallelOptions); stopwatch.Stop(); Console.WriteLine("Submitted {0} tasks in {1}", taskCountToCreate, stopwatch.Elapsed); //Delete the work item to ensure the tasks are cleaned up wm.DeleteWorkItem(workItemName); } }
/// <summary> /// Submit a large number of tasks to the Batch Service. /// </summary> /// <param name="client">The batch client.</param> private static void SubmitLargeNumberOfTasks(IBatchClient client) { const int taskCountToCreate = 5000; // In order to simulate a "large" task object which has many properties set (such as resource files, environment variables, etc) // we create a big environment variable so we have a big task object. char[] env = new char[2048]; for (int i = 0; i < env.Length; i++) { env[i] = 'a'; } string envStr = new string(env); using (IWorkItemManager wm = client.OpenWorkItemManager()) { //Create a work item string workItemName = Environment.GetEnvironmentVariable("USERNAME") + DateTime.Now.ToString("yyyyMMdd-HHmmss"); Console.WriteLine("Creating work item {0}", workItemName); ICloudWorkItem cloudWorkItem = wm.CreateWorkItem(workItemName); cloudWorkItem.JobExecutionEnvironment = new JobExecutionEnvironment() {PoolName = PoolName}; //Specify the pool to run on cloudWorkItem.Commit(); //Wait for an active job TimeSpan maxJobCreationTimeout = TimeSpan.FromSeconds(90); DateTime jobCreationStartTime = DateTime.Now; DateTime jobCreationTimeoutTime = jobCreationStartTime.Add(maxJobCreationTimeout); cloudWorkItem = wm.GetWorkItem(workItemName); Console.WriteLine("Waiting for a job to become active..."); while (cloudWorkItem.ExecutionInformation == null || cloudWorkItem.ExecutionInformation.RecentJob == null) { cloudWorkItem.Refresh(); if (DateTime.Now > jobCreationTimeoutTime) { throw new Exception("Timed out waiting for job."); } Thread.Sleep(TimeSpan.FromSeconds(5)); } string jobName = cloudWorkItem.ExecutionInformation.RecentJob.Name; Console.WriteLine("Found job {0}. Adding task objects...", jobName); //Generate a large number of tasks to submit List<ICloudTask> tasksToSubmit = new List<ICloudTask>(); for (int i = 0; i < taskCountToCreate; i++) { ICloudTask task = new CloudTask("echo" + i.ToString("D5"), "echo"); List<IEnvironmentSetting> environmentSettings = new List<IEnvironmentSetting>(); environmentSettings.Add(new EnvironmentSetting("envone", envStr)); task.EnvironmentSettings = environmentSettings; tasksToSubmit.Add(task); } BatchClientParallelOptions parallelOptions = new BatchClientParallelOptions() { //This will result in at most 10 simultaneous Bulk Add requests to the Batch Service. MaxDegreeOfParallelism = 10 }; Console.WriteLine("Submitting {0} tasks to work item: {1}, job: {2}, on pool: {3}", taskCountToCreate, cloudWorkItem.Name, jobName, cloudWorkItem.JobExecutionEnvironment.PoolName); Stopwatch stopwatch = new Stopwatch(); stopwatch.Start(); //Use the AddTask overload which supports a list of tasks for best AddTask performence - internally this method performs an //intelligent submission of tasks in batches in order to limit the number of REST API calls made to the Batch Service. wm.AddTask(cloudWorkItem.Name, jobName, tasksToSubmit, parallelOptions); stopwatch.Stop(); Console.WriteLine("Submitted {0} tasks in {1}", taskCountToCreate, stopwatch.Elapsed); //Delete the work item to ensure the tasks are cleaned up wm.DeleteWorkItem(workItemName); } }