/// <summary>
/// Submit a large number of tasks to the Batch Service.
/// </summary>
/// <param name="client">The batch client.</param>
/// <param name="sharedPoolId">The ID of the pool to use for the job</param>
private static void SubmitLargeNumberOfTasks(BatchClient client, string sharedPoolId)
{
const int taskCountToCreate = 5000;
// In order to simulate a "large" task object which has many properties set (such as resource files, environment variables, etc)
// we create a big environment variable so we have a big task object.
char[] env = new char[2048];
for (int i = 0; i < env.Length; i++)
{
env[i] = 'a';
}
string envStr = new string(env);
string jobId = CreateJobId("HelloWorldLargeTaskCountJob");
Console.WriteLine("Creating job: " + jobId);
CloudJob boundJob = CreateBoundJob(client.JobOperations, sharedPoolId, jobId);
//Generate a large number of tasks to submit
List <CloudTask> tasksToSubmit = new List <CloudTask>(taskCountToCreate);
for (int i = 0; i < taskCountToCreate; i++)
{
CloudTask task = new CloudTask("echo" + i.ToString("D5"), "echo");
List <EnvironmentSetting> environmentSettings = new List <EnvironmentSetting>();
environmentSettings.Add(new EnvironmentSetting("envone", envStr));
task.EnvironmentSettings = environmentSettings;
tasksToSubmit.Add(task);
}
BatchClientParallelOptions parallelOptions = new BatchClientParallelOptions()
{
//This will result in at most 10 simultaneous Bulk Add requests to the Batch Service.
MaxDegreeOfParallelism = 10
};
Console.WriteLine("Submitting {0} tasks to job: {1}, on pool: {2}",
taskCountToCreate,
boundJob.Id,
boundJob.ExecutionInformation.PoolId);
Stopwatch stopwatch = new Stopwatch();
stopwatch.Start();
// Use the AddTask overload which supports a list of tasks for best AddTask performence - internally this method performs a
// submission of multiple tasks in one REST API request in order to limit the number of calls made to the Batch Service.
client.JobOperations.AddTask(boundJob.Id, tasksToSubmit, parallelOptions);
stopwatch.Stop();
Console.WriteLine("Submitted {0} tasks in {1}", taskCountToCreate, stopwatch.Elapsed);
//Delete the job to ensure the tasks are cleaned up
Console.WriteLine("Deleting job: {0}", boundJob.Id);
client.JobOperations.DeleteJob(boundJob.Id);
}
public async Task Bug1360227_AddTasksBatchFailure(bool useJobOperations)
{
const string testName = "Bug1360227_AddTasksBatchFailure";
int count = 0;
const int countToFailAt = 102;
const int taskCount = 407;
HashSet <string> taskIdsExpectedToFail = new HashSet <string>();
Func <AddTaskResult, CancellationToken, AddTaskResultStatus> resultHandlerFunc = (result, token) =>
{
this.testOutputHelper.WriteLine("Task: {0} got status code: {1}", result.TaskId, result.Status);
++count;
if (taskIdsExpectedToFail.Contains(result.TaskId))
{
return(AddTaskResultStatus.Retry);
}
else
{
if (count >= countToFailAt)
{
taskIdsExpectedToFail.Add(result.TaskId);
this.testOutputHelper.WriteLine("Forcing a failure");
//Throw an exception to cause a failure from the customers result handler -- this is a supported scenario which will
//terminate the add task operation
throw new HttpRequestException("Test");
}
else
{
return(AddTaskResultStatus.Success);
}
}
};
await SynchronizationContextHelper.RunTestAsync(async() =>
{
using (BatchClient batchCli = await TestUtilities.OpenBatchClientFromEnvironmentAsync())
{
BatchClientParallelOptions parallelOptions = new BatchClientParallelOptions()
{
MaxDegreeOfParallelism = 2
};
var exception = await TestUtilities.AssertThrowsAsync <ParallelOperationsException>(
async() => await this.AddTasksSimpleTestAsync(
batchCli,
testName,
taskCount,
parallelOptions,
resultHandlerFunc,
useJobOperations: useJobOperations).ConfigureAwait(false)).ConfigureAwait(false);
Assert.IsType <HttpRequestException>(exception.InnerException);
}
},
TestTimeout);
}
public async Task Bug1360227_AddTasksBatchRequestFailure(bool useJobOperations)
{
const string testName = "Bug1360227_AddTasksBatchRequestFailure";
Random rand = new Random();
object randLock = new object();
BatchClientBehavior customBehavior = new Protocol.RequestInterceptor(request =>
{
var typedRequest = request as Protocol.BatchRequests.TaskAddCollectionBatchRequest;
if (typedRequest != null)
{
var originalServiceRequestFunction = typedRequest.ServiceRequestFunc;
typedRequest.ServiceRequestFunc = token =>
{
lock (randLock)
{
double d = rand.NextDouble();
if (d > 0.3)
{
throw new HttpRequestException("Simulating a network problem");
}
else
{
return(originalServiceRequestFunction(token));
}
}
};
}
});
await SynchronizationContextHelper.RunTestAsync(async() =>
{
using (BatchClient batchCli = await TestUtilities.OpenBatchClientAsync(TestUtilities.GetCredentialsFromEnvironment(), addDefaultRetryPolicy: false))
{
batchCli.JobOperations.CustomBehaviors.Add(customBehavior);
BatchClientParallelOptions parallelOptions = new BatchClientParallelOptions()
{
MaxDegreeOfParallelism = 2
};
var exception = await TestUtilities.AssertThrowsAsync <ParallelOperationsException>(async() =>
await this.AddTasksSimpleTestAsync(batchCli, testName, 397, parallelOptions, useJobOperations: useJobOperations).ConfigureAwait(false)
).ConfigureAwait(false);
Assert.IsType <HttpRequestException>(exception.InnerException);
}
},
TestTimeout);
}
public async Task Bug1360227_AddTasksBatchConfirmResultHandlerTaskReadOnly()
{
const string testName = "Bug1360227_ConfirmResultHandlerTaskReadOnly";
Func <AddTaskResult, CancellationToken, AddTaskResultStatus> resultHandlerFunc = (result, token) =>
{
//Count everything as a success
AddTaskResultStatus resultAction = AddTaskResultStatus.Success;
//Try to set a property of the cloud task
InvalidOperationException e = TestUtilities.AssertThrows <InvalidOperationException>(() =>
result.Task.Constraints = new TaskConstraints(TimeSpan.FromSeconds(5), null, null));
Assert.Contains("Write access is not allowed.", e.Message);
//Try to call a method of a CloudTask
//TODO: This should be blocked but isn't right now...
//try
//{
// result.Task.Terminate();
// Debug.Fail("Should not have gotten here");
//}
//catch (Exception e)
//{
// Console.WriteLine(e);
// //Swallow this exception as it is expected
//}
return(resultAction);
};
await SynchronizationContextHelper.RunTestAsync(async() =>
{
using (BatchClient batchCli = await TestUtilities.OpenBatchClientFromEnvironmentAsync())
{
BatchClientParallelOptions parallelOptions = new BatchClientParallelOptions()
{
MaxDegreeOfParallelism = 2
};
await this.AddTasksSimpleTestAsync(
batchCli,
testName,
55,
parallelOptions,
resultHandlerFunc).ConfigureAwait(false);
}
},
TestTimeout);
}
public async Task Bug1360227_AddTasksBatchHugeTaskCount()
{
const string testName = "Bug1360227_AddTasksBatchHugeTaskCount";
await SynchronizationContextHelper.RunTestAsync(async() =>
{
using BatchClient batchCli = await TestUtilities.OpenBatchClientFromEnvironmentAsync();
BatchClientParallelOptions parallelOptions = new BatchClientParallelOptions()
{
MaxDegreeOfParallelism = 25
};
await AddTasksSimpleTestAsync(batchCli, testName, 5025, parallelOptions).ConfigureAwait(false);
},
LongTestTimeout);
}
public async Task Bug1360227_AddTasksBatchCancelation(bool useJobOperations)
{
const string testName = "Bug1360227_AddTasksBatchCancelation";
const int taskCount = 322;
await SynchronizationContextHelper.RunTestAsync(async() =>
{
using (BatchClient batchCli = await TestUtilities.OpenBatchClientFromEnvironmentAsync())
{
using (CancellationTokenSource source = new CancellationTokenSource())
{
BatchClientParallelOptions parallelOptions = new BatchClientParallelOptions()
{
MaxDegreeOfParallelism = 2,
CancellationToken = source.Token
};
System.Threading.Tasks.Task t = this.AddTasksSimpleTestAsync(
batchCli,
testName,
taskCount,
parallelOptions,
useJobOperations: useJobOperations);
Thread.Sleep(TimeSpan.FromSeconds(.3)); //Wait till we get into the workflow
this.testOutputHelper.WriteLine("Canceling the work flow");
source.Cancel();
try
{
await t.ConfigureAwait(false);
}
catch (Exception e)
{
//This is expected to throw one of two possible exception types...
if (!(e is TaskCanceledException) && !(e is OperationCanceledException))
{
throw new ThrowsException(typeof(TaskCanceledException), e);
}
}
}
}
},
TestTimeout);
}
public async Task AddTasksRequestEntityTooLarge_ReduceChunkSize()
{
const string testName = "AddTasksRequestEntityTooLarge_ReduceChunkSize";
List <ResourceFile> resourceFiles = new List <ResourceFile>();
ResourceFile resourceFile;
int countChunksOf100 = 0;
int numTasks = 176;
int degreesOfParallelism = 2;
BatchClientBehavior customBehavior = new Protocol.RequestInterceptor(request =>
{
var typedRequest = request as Protocol.BatchRequests.TaskAddCollectionBatchRequest;
if (typedRequest != null)
{
if (typedRequest.Parameters.Count > 50)
{
Interlocked.Increment(ref countChunksOf100);
}
}
});
// If this test fails try increasing the size of the Task in case maximum size increase
for (int i = 0; i < 100; i++)
{
resourceFile = ResourceFile.FromUrl("https://mystorageaccount.blob.core.windows.net/files/resourceFile" + i, "resourceFile" + i);
resourceFiles.Add(resourceFile);
}
await SynchronizationContextHelper.RunTestAsync(async() =>
{
using (BatchClient batchCli = TestUtilities.OpenBatchClient(TestUtilities.GetCredentialsFromEnvironment(), addDefaultRetryPolicy: false))
{
batchCli.JobOperations.CustomBehaviors.Add(customBehavior);
BatchClientParallelOptions parallelOptions = new BatchClientParallelOptions()
{
MaxDegreeOfParallelism = degreesOfParallelism
};
await AddTasksSimpleTestAsync(batchCli, testName, numTasks, parallelOptions, resourceFiles: resourceFiles).ConfigureAwait(false);
}
},
TestTimeout);
Assert.True(countChunksOf100 <= Math.Min(Math.Ceiling(numTasks / 100.0), degreesOfParallelism));
}
private void SubmitTasks(Guid jobId, CancellationToken ct)
{
summary += $"Starting to submit tasks - {DateTime.UtcNow}\n";
var attempts = 0;
//while (attempts++ < 20)
//{
if (ct.IsCancellationRequested)
{
return;
}
//try
//{
BatchClientParallelOptions parallelOptions = new BatchClientParallelOptions()
{
CancellationToken = ct,
MaxDegreeOfParallelism = 4,
};
summary += "Fetching tasks to submit...\n";
_batchClient.JobOperations.AddTaskAsync(jobId.ToString(), GetTasksToSubmit(jobId), parallelOptions).Wait(ct);
summary += _taskProvider.Output;
summary += $"Starting to submit tasks - {DateTime.UtcNow}\n";
// break;
//}
//catch (AggregateException e)
//{
// Console.WriteLine("An error occurred submitting tasks: {0}", e.InnerException);
//}
//catch (Exception e)
//{
// Console.WriteLine("An error occurred submitting tasks: {0}", e);
//}
ct.WaitHandle.WaitOne(TimeSpan.FromSeconds(5));
//}
}
private void SubmitTasks(Guid jobId, CancellationToken ct)
{
Console.WriteLine("Starting to submit tasks - {0}", DateTime.UtcNow);
var attempts = 0;
while (attempts++ < 20)
{
if (ct.IsCancellationRequested)
{
return;
}
try
{
BatchClientParallelOptions parallelOptions = new BatchClientParallelOptions()
{
CancellationToken = ct,
MaxDegreeOfParallelism = 4,
};
_batchClient.JobOperations.AddTaskAsync(jobId.ToString(), GetTasksToSubmit(jobId), parallelOptions).Wait(ct);
Console.WriteLine("Starting to submit tasks - {0}", DateTime.UtcNow);
break;
}
catch (AggregateException e)
{
Console.WriteLine("An error occurred submitting tasks: {0}", e.InnerException);
}
catch (Exception e)
{
Console.WriteLine("An error occurred submitting tasks: {0}", e);
}
ct.WaitHandle.WaitOne(TimeSpan.FromSeconds(5));
}
}
/// <summary>
/// Performs a simple AddTask test, adding the specified task count using the specified parallelOptions and resultHandlerFunc
/// </summary>
/// <returns></returns>
private async System.Threading.Tasks.Task AddTasksSimpleTestAsync(
BatchClient batchCli,
string testName,
int taskCount,
BatchClientParallelOptions parallelOptions,
Func <AddTaskResult, CancellationToken, AddTaskResultStatus> resultHandlerFunc,
StagingStorageAccount storageCredentials,
IEnumerable <string> localFilesToStage,
ConcurrentBag <ConcurrentDictionary <Type, IFileStagingArtifact> > fileStagingArtifacts = null,
TimeSpan?timeout = null,
bool useJobOperations = true)
{
JobOperations jobOperations = batchCli.JobOperations;
string jobId = "Bulk-" + TestUtilities.GetMyName() + "-" + testName + "-" + useJobOperations;
try
{
CloudJob unboundJob = jobOperations.CreateJob();
this.testOutputHelper.WriteLine("Initial job commit for job: {0}", jobId);
unboundJob.PoolInformation = new PoolInformation()
{
PoolId = "DummyPool"
};
unboundJob.Id = jobId;
await unboundJob.CommitAsync().ConfigureAwait(false);
CloudJob boundJob = await jobOperations.GetJobAsync(jobId).ConfigureAwait(false);
//
// Add a simple set of tasks
//
IEnumerable <string> taskNames = GenerateTaskIds(taskCount);
List <CloudTask> tasksToAdd = new List <CloudTask>();
List <CloudTask> tasksToValidateWith = new List <CloudTask>();
IList <IFileStagingProvider> lastFilesToStageList = null;
foreach (string taskName in taskNames)
{
CloudTask myTask = new CloudTask(taskName, "cmd /c echo hello world");
CloudTask duplicateReadableTask = new CloudTask(taskName, "cmd /c echo hello world");
if (localFilesToStage != null && storageCredentials != null)
{
myTask.FilesToStage = new List <IFileStagingProvider>();
lastFilesToStageList = myTask.FilesToStage;
duplicateReadableTask.FilesToStage = new List <IFileStagingProvider>();
foreach (string fileToStage in localFilesToStage)
{
duplicateReadableTask.FilesToStage.Add(new FileToStage(fileToStage, storageCredentials));
myTask.FilesToStage.Add(new FileToStage(fileToStage, storageCredentials));
}
}
tasksToAdd.Add(myTask);
tasksToValidateWith.Add(duplicateReadableTask);
}
List <BatchClientBehavior> behaviors = new List <BatchClientBehavior>();
if (resultHandlerFunc != null)
{
behaviors.Add(new AddTaskCollectionResultHandler(resultHandlerFunc));
}
//Add the tasks
Stopwatch stopwatch = new Stopwatch();
this.testOutputHelper.WriteLine("Starting task add");
stopwatch.Start();
if (useJobOperations)
{
await jobOperations.AddTaskAsync(
jobId,
tasksToAdd,
parallelOptions : parallelOptions,
fileStagingArtifacts : fileStagingArtifacts,
timeout : timeout,
additionalBehaviors : behaviors).ConfigureAwait(continueOnCapturedContext: false);
}
else
{
await boundJob.AddTaskAsync(
tasksToAdd,
parallelOptions : parallelOptions,
fileStagingArtifacts : fileStagingArtifacts,
timeout : timeout,
additionalBehaviors : behaviors).ConfigureAwait(continueOnCapturedContext: false);
}
stopwatch.Stop();
this.testOutputHelper.WriteLine("Task add finished, took: {0}", stopwatch.Elapsed);
if (lastFilesToStageList != null)
{
Assert.Throws <InvalidOperationException>(() => lastFilesToStageList.Add(new FileToStage("test", null)));
}
//Ensure the task lists match
List <CloudTask> tasksFromService = await jobOperations.ListTasks(jobId).ToListAsync().ConfigureAwait(false);
EnsureTasksListsMatch(tasksToValidateWith, tasksFromService);
}
catch (Exception e)
{
this.testOutputHelper.WriteLine("Exception: {0}", e.ToString());
throw;
}
finally
{
TestUtilities.DeleteJobIfExistsAsync(batchCli, jobId).Wait();
}
}
public async Task Bug1360227_AddTasksBatchRetry(bool useJobOperations)
{
const string testName = "Bug1360227_AddTasksBatchRetry";
Random rand = new Random();
object randLock = new object();
int numberOfTasksWhichHitClientError = 0;
int numberOfTasksWhichWereForcedToRetry = 0;
Func <AddTaskResult, CancellationToken, AddTaskResultStatus> resultHandlerFunc = (result, token) =>
{
this.testOutputHelper.WriteLine("Task: {0} got status code: {1}", result.TaskId, result.Status);
AddTaskResultStatus resultAction;
if (result.Status == AddTaskStatus.ClientError)
{
++numberOfTasksWhichHitClientError;
return(AddTaskResultStatus.Success); //Have to count client error as success
}
lock (randLock)
{
double d = rand.NextDouble();
if (d > 0.8)
{
this.testOutputHelper.WriteLine("Forcing retry for task: {0}", result.TaskId);
resultAction = AddTaskResultStatus.Retry;
++numberOfTasksWhichWereForcedToRetry;
}
else
{
resultAction = AddTaskResultStatus.Success;
}
}
return(resultAction);
};
await SynchronizationContextHelper.RunTestAsync(async() =>
{
StagingStorageAccount storageCredentials = TestUtilities.GetStorageCredentialsFromEnvironment();
using (BatchClient batchCli = await TestUtilities.OpenBatchClientFromEnvironmentAsync())
{
BatchClientParallelOptions parallelOptions = new BatchClientParallelOptions()
{
MaxDegreeOfParallelism = 2
};
await this.AddTasksSimpleTestAsync(
batchCli,
testName,
1281,
parallelOptions,
resultHandlerFunc,
storageCredentials,
new List <string> {
"TestResources\\Data.txt"
},
useJobOperations: useJobOperations).ConfigureAwait(false);
}
},
LongTestTimeout);
//Ensure that we forced some tasks to retry
this.testOutputHelper.WriteLine("Forced a total of {0} tasks to retry", numberOfTasksWhichWereForcedToRetry);
Assert.True(numberOfTasksWhichWereForcedToRetry > 0);
Assert.Equal(numberOfTasksWhichWereForcedToRetry, numberOfTasksWhichHitClientError);
}
/// <summary>
/// Submit a large number of tasks to the Batch Service.
/// </summary>
/// <param name="client">The batch client.</param>
/// <param name="sharedPoolId">The ID of the pool to use for the job</param>
private static void SubmitLargeNumberOfTasks(BatchClient client, string sharedPoolId)
{
const int taskCountToCreate = 5000;
// In order to simulate a "large" task object which has many properties set (such as resource files, environment variables, etc)
// we create a big environment variable so we have a big task object.
char[] env = new char[2048];
for (int i = 0; i < env.Length; i++)
{
env[i] = 'a';
}
string envStr = new string(env);
string jobId = CreateJobId("HelloWorldLargeTaskCountJob");
Console.WriteLine("Creating job: " + jobId);
CloudJob boundJob = CreateBoundJob(client.JobOperations, sharedPoolId, jobId);
//Generate a large number of tasks to submit
List<CloudTask> tasksToSubmit = new List<CloudTask>(taskCountToCreate);
for (int i = 0; i < taskCountToCreate; i++)
{
CloudTask task = new CloudTask("echo" + i.ToString("D5"), "echo");
List<EnvironmentSetting> environmentSettings = new List<EnvironmentSetting>();
environmentSettings.Add(new EnvironmentSetting("envone", envStr));
task.EnvironmentSettings = environmentSettings;
tasksToSubmit.Add(task);
}
BatchClientParallelOptions parallelOptions = new BatchClientParallelOptions()
{
//This will result in at most 10 simultaneous Bulk Add requests to the Batch Service.
MaxDegreeOfParallelism = 10
};
Console.WriteLine("Submitting {0} tasks to job: {1}, on pool: {2}",
taskCountToCreate,
boundJob.Id,
boundJob.ExecutionInformation.PoolId);
Stopwatch stopwatch = new Stopwatch();
stopwatch.Start();
// Use the AddTask overload which supports a list of tasks for best AddTask performence - internally this method performs a
// submission of multiple tasks in one REST API request in order to limit the number of calls made to the Batch Service.
client.JobOperations.AddTask(boundJob.Id, tasksToSubmit, parallelOptions);
stopwatch.Stop();
Console.WriteLine("Submitted {0} tasks in {1}", taskCountToCreate, stopwatch.Elapsed);
//Delete the job to ensure the tasks are cleaned up
Console.WriteLine("Deleting job: {0}", boundJob.Id);
client.JobOperations.DeleteJob(boundJob.Id);
}
/// <summary>
/// Creates a job and adds a task to it.
/// </summary>
/// <param name="batchClient">The BatchClient to use when interacting with the Batch service.</param>
/// <param name="configurationSettings">The configuration settings</param>
/// <param name="jobId">The ID of the job.</param>
/// <returns>An asynchronous <see cref="Task"/> representing the operation.</returns>
private static async Task SubmitJobAsync(BatchClient batchClient, Settings configurationSettings, AccountSettings accountSettings, string jobId)
{
// create an empty unbound Job
CloudJob unboundJob = batchClient.JobOperations.CreateJob();
unboundJob.Id = jobId;
unboundJob.CommonEnvironmentSettings = new List <EnvironmentSetting>
{
new EnvironmentSetting("AZURE_STORAGE_CONNECTION_STRING", accountSettings.StorageConnectionString)
};
// For this job, ask the Batch service to automatically create a pool of VMs when the job is submitted.
unboundJob.PoolInformation = new PoolInformation()
{
AutoPoolSpecification = new AutoPoolSpecification()
{
AutoPoolIdPrefix = "HelloWorld",
PoolSpecification = new PoolSpecification()
{
TargetDedicatedComputeNodes = configurationSettings.PoolTargetNodeCount,
VirtualMachineConfiguration = new VirtualMachineConfiguration(
new ImageReference(
"UbuntuServer", "Canonical", "18.04-LTS", "latest"
),
"batch.node.ubuntu 18.04"
),
TaskSlotsPerNode = configurationSettings.TaskSlotsPerNode,
TaskSchedulingPolicy = new TaskSchedulingPolicy(ComputeNodeFillType.Spread),
VirtualMachineSize = configurationSettings.PoolNodeVirtualMachineSize,
ApplicationPackageReferences = new List <ApplicationPackageReference>
{
new ApplicationPackageReference {
ApplicationId = configurationSettings.ApplicationId,
Version = configurationSettings.ApplicationVersion
}
},
},
KeepAlive = configurationSettings.PoolKeepAlive,
PoolLifetimeOption = PoolLifetimeOption.Job
}
};
// Commit Job to create it in the service
await unboundJob.CommitAsync();
// create a simple task. Each task within a job must have a unique ID
//await batchClient.JobOperations.AddTaskAsync(jobId, new CloudTask("task-env", "env"));
//await batchClient.JobOperations.AddTaskAsync(jobId, new CloudTask("task-hostname", "/bin/sh -c \"hostname\""));
// Task could should equal poolTargetNodeCount * # of cores in VM SKU
//TODO https://docs.microsoft.com/en-us/azure/batch/large-number-tasks#example-batch-net
// Create list of CloudTasks and addTaskAysnc in single operation.
int i;
List <CloudTask> tasksToAdd = new List <CloudTask>(); // Populate with your tasks
//string cmd = "/bin/sh -c \"$AZ_BATCH_APP_PACKAGE_sleeper_1_2/sleeper-queue consume\"";
for (i = 0; i < configurationSettings.TaskCount; i++)
{
tasksToAdd.Add(new CloudTask($"{configurationSettings.ApplicationId}-{i}", configurationSettings.TaskCommand));
}
// https://docs.microsoft.com/en-us/azure/batch/large-number-tasks#example-batch-net
BatchClientParallelOptions parallelOptions = new BatchClientParallelOptions()
{
MaxDegreeOfParallelism = configurationSettings.MaxDegreeOfParallelism
};
await batchClient.JobOperations.AddTaskAsync(jobId, tasksToAdd, parallelOptions);
}
/// <summary>
/// Submit a large number of tasks to the Batch Service.
/// </summary>
/// <param name="client">The batch client.</param>
private static void SubmitLargeNumberOfTasks(IBatchClient client)
{
const int taskCountToCreate = 5000;
// In order to simulate a "large" task object which has many properties set (such as resource files, environment variables, etc)
// we create a big environment variable so we have a big task object.
char[] env = new char[2048];
for (int i = 0; i < env.Length; i++)
{
env[i] = 'a';
}
string envStr = new string(env);
using (IWorkItemManager wm = client.OpenWorkItemManager())
{
//Create a work item
string workItemName = Environment.GetEnvironmentVariable("USERNAME") + DateTime.Now.ToString("yyyyMMdd-HHmmss");
Console.WriteLine("Creating work item {0}", workItemName);
ICloudWorkItem cloudWorkItem = wm.CreateWorkItem(workItemName);
cloudWorkItem.JobExecutionEnvironment = new JobExecutionEnvironment()
{
PoolName = PoolName
}; //Specify the pool to run on
cloudWorkItem.Commit();
//Wait for an active job
TimeSpan maxJobCreationTimeout = TimeSpan.FromSeconds(90);
DateTime jobCreationStartTime = DateTime.Now;
DateTime jobCreationTimeoutTime = jobCreationStartTime.Add(maxJobCreationTimeout);
cloudWorkItem = wm.GetWorkItem(workItemName);
Console.WriteLine("Waiting for a job to become active...");
while (cloudWorkItem.ExecutionInformation == null || cloudWorkItem.ExecutionInformation.RecentJob == null)
{
cloudWorkItem.Refresh();
if (DateTime.Now > jobCreationTimeoutTime)
{
throw new Exception("Timed out waiting for job.");
}
Thread.Sleep(TimeSpan.FromSeconds(5));
}
string jobName = cloudWorkItem.ExecutionInformation.RecentJob.Name;
Console.WriteLine("Found job {0}. Adding task objects...", jobName);
//Generate a large number of tasks to submit
List <ICloudTask> tasksToSubmit = new List <ICloudTask>();
for (int i = 0; i < taskCountToCreate; i++)
{
ICloudTask task = new CloudTask("echo" + i.ToString("D5"), "echo");
List <IEnvironmentSetting> environmentSettings = new List <IEnvironmentSetting>();
environmentSettings.Add(new EnvironmentSetting("envone", envStr));
task.EnvironmentSettings = environmentSettings;
tasksToSubmit.Add(task);
}
BatchClientParallelOptions parallelOptions = new BatchClientParallelOptions()
{
//This will result in at most 10 simultaneous Bulk Add requests to the Batch Service.
MaxDegreeOfParallelism = 10
};
Console.WriteLine("Submitting {0} tasks to work item: {1}, job: {2}, on pool: {3}",
taskCountToCreate,
cloudWorkItem.Name,
jobName,
cloudWorkItem.JobExecutionEnvironment.PoolName);
Stopwatch stopwatch = new Stopwatch();
stopwatch.Start();
//Use the AddTask overload which supports a list of tasks for best AddTask performence - internally this method performs an
//intelligent submission of tasks in batches in order to limit the number of REST API calls made to the Batch Service.
wm.AddTask(cloudWorkItem.Name, jobName, tasksToSubmit, parallelOptions);
stopwatch.Stop();
Console.WriteLine("Submitted {0} tasks in {1}", taskCountToCreate, stopwatch.Elapsed);
//Delete the work item to ensure the tasks are cleaned up
wm.DeleteWorkItem(workItemName);
}
}
/// <summary>
/// Submit a large number of tasks to the Batch Service.
/// </summary>
/// <param name="client">The batch client.</param>
private static void SubmitLargeNumberOfTasks(IBatchClient client)
{
const int taskCountToCreate = 5000;
// In order to simulate a "large" task object which has many properties set (such as resource files, environment variables, etc)
// we create a big environment variable so we have a big task object.
char[] env = new char[2048];
for (int i = 0; i < env.Length; i++)
{
env[i] = 'a';
}
string envStr = new string(env);
using (IWorkItemManager wm = client.OpenWorkItemManager())
{
//Create a work item
string workItemName = Environment.GetEnvironmentVariable("USERNAME") + DateTime.Now.ToString("yyyyMMdd-HHmmss");
Console.WriteLine("Creating work item {0}", workItemName);
ICloudWorkItem cloudWorkItem = wm.CreateWorkItem(workItemName);
cloudWorkItem.JobExecutionEnvironment = new JobExecutionEnvironment() {PoolName = PoolName}; //Specify the pool to run on
cloudWorkItem.Commit();
//Wait for an active job
TimeSpan maxJobCreationTimeout = TimeSpan.FromSeconds(90);
DateTime jobCreationStartTime = DateTime.Now;
DateTime jobCreationTimeoutTime = jobCreationStartTime.Add(maxJobCreationTimeout);
cloudWorkItem = wm.GetWorkItem(workItemName);
Console.WriteLine("Waiting for a job to become active...");
while (cloudWorkItem.ExecutionInformation == null || cloudWorkItem.ExecutionInformation.RecentJob == null)
{
cloudWorkItem.Refresh();
if (DateTime.Now > jobCreationTimeoutTime)
{
throw new Exception("Timed out waiting for job.");
}
Thread.Sleep(TimeSpan.FromSeconds(5));
}
string jobName = cloudWorkItem.ExecutionInformation.RecentJob.Name;
Console.WriteLine("Found job {0}. Adding task objects...", jobName);
//Generate a large number of tasks to submit
List<ICloudTask> tasksToSubmit = new List<ICloudTask>();
for (int i = 0; i < taskCountToCreate; i++)
{
ICloudTask task = new CloudTask("echo" + i.ToString("D5"), "echo");
List<IEnvironmentSetting> environmentSettings = new List<IEnvironmentSetting>();
environmentSettings.Add(new EnvironmentSetting("envone", envStr));
task.EnvironmentSettings = environmentSettings;
tasksToSubmit.Add(task);
}
BatchClientParallelOptions parallelOptions = new BatchClientParallelOptions()
{
//This will result in at most 10 simultaneous Bulk Add requests to the Batch Service.
MaxDegreeOfParallelism = 10
};
Console.WriteLine("Submitting {0} tasks to work item: {1}, job: {2}, on pool: {3}",
taskCountToCreate,
cloudWorkItem.Name,
jobName,
cloudWorkItem.JobExecutionEnvironment.PoolName);
Stopwatch stopwatch = new Stopwatch();
stopwatch.Start();
//Use the AddTask overload which supports a list of tasks for best AddTask performence - internally this method performs an
//intelligent submission of tasks in batches in order to limit the number of REST API calls made to the Batch Service.
wm.AddTask(cloudWorkItem.Name, jobName, tasksToSubmit, parallelOptions);
stopwatch.Stop();
Console.WriteLine("Submitted {0} tasks in {1}", taskCountToCreate, stopwatch.Elapsed);
//Delete the work item to ensure the tasks are cleaned up
wm.DeleteWorkItem(workItemName);
}
}
