public async Task Job_CanAddJobWithJobManagerAndAllowLowPriorityTrue()
{
Func <Task> test = async() =>
{
using (BatchClient batchCli = await TestUtilities.OpenBatchClientFromEnvironmentAsync().ConfigureAwait(false))
{
string jobId = "TestJobWithLowPriJobManager-" + TestUtilities.GetMyName();
try
{
PoolInformation poolInfo = new PoolInformation()
{
PoolId = "Fake"
};
CloudJob unboundJob = batchCli.JobOperations.CreateJob(jobId, poolInfo);
unboundJob.JobManagerTask = new JobManagerTask("foo", "cmd /c echo hi")
{
AllowLowPriorityNode = true
};
await unboundJob.CommitAsync().ConfigureAwait(false);
await unboundJob.RefreshAsync().ConfigureAwait(false);
Assert.True(unboundJob.JobManagerTask.AllowLowPriorityNode);
}
finally
{
await TestUtilities.DeleteJobIfExistsAsync(batchCli, jobId);
}
}
};
await SynchronizationContextHelper.RunTestAsync(test, TestTimeout);
}
public async Task ApplicationPackageReferencesOnCloudTaskAreRoundtripped()
{
string jobId = Constants.DefaultConveniencePrefix + TestUtilities.GetMyName() + "-APROnCloudTaskAreRoundtripped";
string taskId = "task-id";
const string applicationId = "blender";
const string applicationVerson = "beta";
Func <Task> test = async() =>
{
using (BatchClient client = await TestUtilities.OpenBatchClientFromEnvironmentAsync().ConfigureAwait(false))
{
var poolInfo = new PoolInformation
{
AutoPoolSpecification = new AutoPoolSpecification
{
PoolSpecification = new PoolSpecification
{
CloudServiceConfiguration = new CloudServiceConfiguration(PoolFixture.OSFamily),
VirtualMachineSize = PoolFixture.VMSize,
},
PoolLifetimeOption = PoolLifetimeOption.Job
}
};
try
{
CloudJob job = client.JobOperations.CreateJob(jobId, poolInfo);
await job.CommitAsync().ConfigureAwait(false);
var boundJob = await client.JobOperations.GetJobAsync(jobId).ConfigureAwait(false);
CloudTask cloudTask = new CloudTask(taskId, "cmd /c ping 127.0.0.1")
{
ApplicationPackageReferences = new[]
{
new ApplicationPackageReference
{
ApplicationId = applicationId,
Version = applicationVerson
}
}
};
await boundJob.AddTaskAsync(cloudTask).ConfigureAwait(false);
CloudTask boundCloudTask = await boundJob.GetTaskAsync(taskId).ConfigureAwait(false);
Assert.Equal(applicationId, boundCloudTask.ApplicationPackageReferences.Single().ApplicationId);
Assert.Equal(applicationVerson, boundCloudTask.ApplicationPackageReferences.Single().Version);
}
finally
{
TestUtilities.DeleteJobIfExistsAsync(client, jobId).Wait();
}
}
};
await SynchronizationContextHelper.RunTestAsync(test, TestTimeout);
}
public async Task AddTasksFailIfPoisonTaskTooLarge()
{
const string testName = "AddTasksFailIfPoisonTaskTooLarge";
List <ResourceFile> resourceFiles = new List <ResourceFile>();
ResourceFile resourceFile;
// If this test fails try increasing the size of the Task in case maximum size increase
for (int i = 0; i < 10000; i++)
{
resourceFile = ResourceFile.FromUrl("https://mystorageaccount.blob.core.windows.net/files/resourceFile" + i, "resourceFile" + i);
resourceFiles.Add(resourceFile);
}
await SynchronizationContextHelper.RunTestAsync(async() =>
{
using (BatchClient batchCli = await TestUtilities.OpenBatchClientFromEnvironmentAsync())
{
var exception = await TestUtilities.AssertThrowsAsync <ParallelOperationsException>(
async() => await this.AddTasksSimpleTestAsync(batchCli, testName, 1, resourceFiles: resourceFiles).ConfigureAwait(false)).ConfigureAwait(false);
var innerException = exception.InnerException;
Assert.IsType <BatchException>(innerException);
Assert.Equal(((BatchException)innerException).RequestInformation.BatchError.Code, BatchErrorCodeStrings.RequestBodyTooLarge);
}
},
TestTimeout);
}
public async Task BatchRequestWithShortClientSideTimeoutAndRetries()
{
const int maxRetries = 5;
int actualRequestCount = 0;
//The ThrowsAnyAsync must be outside the RunTestAsync or it will deadlock
await Assert.ThrowsAnyAsync <OperationCanceledException>(async() =>
await SynchronizationContextHelper.RunTestAsync(async() =>
{
using (BatchClient client = await TestUtilities.OpenBatchClientAsync(TestUtilities.GetCredentialsFromEnvironment(), addDefaultRetryPolicy: false))
{
client.CustomBehaviors.Add(RetryPolicyProvider.LinearRetryProvider(TimeSpan.FromMilliseconds(250), maxRetries));
List <BatchClientBehavior> customBehaviors = new List <BatchClientBehavior>();
customBehaviors.Add(new RequestInterceptor((req) =>
{
//Set the timeout to something small so it is guaranteed to expire before the service has responded
req.Timeout = TimeSpan.FromMilliseconds(25);
var castRequest = (JobGetBatchRequest)req;
Func <CancellationToken, Task <AzureOperationResponse <Microsoft.Azure.Batch.Protocol.Models.CloudJob, Microsoft.Azure.Batch.Protocol.Models.JobGetHeaders> > > oldFunc = castRequest.ServiceRequestFunc;
castRequest.ServiceRequestFunc = async(token) =>
{
actualRequestCount++; //Count the number of calls to the func
return(await oldFunc(token).ConfigureAwait(false));
};
}));
await client.JobOperations.GetJobAsync("Foo", additionalBehaviors: customBehaviors).ConfigureAwait(false);
}
},
TestTimeout));
Assert.Equal(maxRetries, actualRequestCount - 1); //Ensure that the number of retries is as expected
}
public async Task ComputeNodeUploadLogs()
{
Func <Task> test = async() =>
{
using (BatchClient batchCli = TestUtilities.OpenBatchClientFromEnvironmentAsync().Result)
{
const string containerName = "computenodelogscontainer";
// Generate a storage container URL
StagingStorageAccount storageAccount = TestUtilities.GetStorageCredentialsFromEnvironment();
BlobServiceClient blobClient = BlobUtilities.GetBlobServiceClient(storageAccount);
BlobContainerClient containerClient = BlobUtilities.GetBlobContainerClient(containerName, blobClient, storageAccount);
try
{
containerClient.CreateIfNotExists();
string sasUri = BlobUtilities.GetWriteableSasUri(containerClient, storageAccount);
var blobs = containerClient.GetAllBlobs();
// Ensure that there are no items in the container to begin with
Assert.Empty(blobs);
var startTime = DateTime.UtcNow.Subtract(TimeSpan.FromMinutes(5));
var node = batchCli.PoolOperations.ListComputeNodes(this.poolFixture.PoolId).First();
var result = batchCli.PoolOperations.UploadComputeNodeBatchServiceLogs(
this.poolFixture.PoolId,
node.Id,
sasUri,
startTime);
Assert.NotEqual(0, result.NumberOfFilesUploaded);
Assert.NotEmpty(result.VirtualDirectoryName);
// Allow up to 2m for files to get uploaded
DateTime timeoutAt = DateTime.UtcNow.AddMinutes(2);
while (DateTime.UtcNow < timeoutAt)
{
blobs = containerClient.GetAllBlobs();
if (blobs.Any())
{
break;
}
}
Assert.NotEmpty(blobs);
}
finally
{
await containerClient.DeleteIfExistsAsync();
}
}
};
await SynchronizationContextHelper.RunTestAsync(test, TestTimeout);
}
public async Task Bug1360227_AddTasksBatchFailure(bool useJobOperations)
{
const string testName = "Bug1360227_AddTasksBatchFailure";
int count = 0;
const int countToFailAt = 102;
const int taskCount = 407;
HashSet <string> taskIdsExpectedToFail = new HashSet <string>();
Func <AddTaskResult, CancellationToken, AddTaskResultStatus> resultHandlerFunc = (result, token) =>
{
this.testOutputHelper.WriteLine("Task: {0} got status code: {1}", result.TaskId, result.Status);
++count;
if (taskIdsExpectedToFail.Contains(result.TaskId))
{
return(AddTaskResultStatus.Retry);
}
else
{
if (count >= countToFailAt)
{
taskIdsExpectedToFail.Add(result.TaskId);
this.testOutputHelper.WriteLine("Forcing a failure");
//Throw an exception to cause a failure from the customers result handler -- this is a supported scenario which will
//terminate the add task operation
throw new HttpRequestException("Test");
}
else
{
return(AddTaskResultStatus.Success);
}
}
};
await SynchronizationContextHelper.RunTestAsync(async() =>
{
using (BatchClient batchCli = await TestUtilities.OpenBatchClientFromEnvironmentAsync())
{
BatchClientParallelOptions parallelOptions = new BatchClientParallelOptions()
{
MaxDegreeOfParallelism = 2
};
var exception = await TestUtilities.AssertThrowsAsync <ParallelOperationsException>(
async() => await this.AddTasksSimpleTestAsync(
batchCli,
testName,
taskCount,
parallelOptions,
resultHandlerFunc,
useJobOperations: useJobOperations).ConfigureAwait(false)).ConfigureAwait(false);
Assert.IsType <HttpRequestException>(exception.InnerException);
}
},
TestTimeout);
}
public async Task JobSchedulePatch()
{
Func <Task> test = async() =>
{
using (BatchClient batchCli = await TestUtilities.OpenBatchClientFromEnvironmentAsync().ConfigureAwait(false))
{
string jobScheduleId = "TestPatchJobSchedule-" + TestUtilities.GetMyName();
const string newJobManagerCommandLine = "cmd /c dir";
const string metadataKey = "Foo";
const string metadataValue = "Bar";
TimeSpan newRecurrenceInterval = TimeSpan.FromDays(2);
try
{
CloudJobSchedule jobSchedule = batchCli.JobScheduleOperations.CreateJobSchedule(
jobScheduleId,
new Schedule()
{
RecurrenceInterval = TimeSpan.FromDays(1)
},
new JobSpecification(new PoolInformation()
{
PoolId = "DummyPool"
})
{
JobManagerTask = new JobManagerTask(id: "Foo", commandLine: "Foo")
});
await jobSchedule.CommitAsync().ConfigureAwait(false);
await jobSchedule.RefreshAsync().ConfigureAwait(false);
jobSchedule.JobSpecification.JobManagerTask.CommandLine = newJobManagerCommandLine;
jobSchedule.Metadata = new List <MetadataItem>()
{
new MetadataItem(metadataKey, metadataValue)
};
jobSchedule.Schedule.RecurrenceInterval = newRecurrenceInterval;
await jobSchedule.CommitChangesAsync().ConfigureAwait(false);
await jobSchedule.RefreshAsync().ConfigureAwait(false);
Assert.Equal(newRecurrenceInterval, jobSchedule.Schedule.RecurrenceInterval);
Assert.Equal(newJobManagerCommandLine, jobSchedule.JobSpecification.JobManagerTask.CommandLine);
Assert.Equal(metadataKey, jobSchedule.Metadata.Single().Name);
Assert.Equal(metadataValue, jobSchedule.Metadata.Single().Value);
}
finally
{
await TestUtilities.DeleteJobScheduleIfExistsAsync(batchCli, jobScheduleId).ConfigureAwait(false);
}
}
};
await SynchronizationContextHelper.RunTestAsync(test, TestTimeout);
}
public async Task End2EndApplicationPackageScenario()
{
string accountName = TestCommon.Configuration.BatchAccountName;
Func <Task> test = async() =>
{
var poolId = "app-ref-test" + Guid.NewGuid();
using (BatchClient client = await TestUtilities.OpenBatchClientFromEnvironmentAsync())
{
using (var mgmtClient = IntegrationTestCommon.OpenBatchManagementClient())
{
// Give the application a display name
await mgmtClient.Application.UpdateAsync(TestCommon.Configuration.BatchAccountResourceGroup, accountName, ApplicationId, new UpdateApplicationParameters
{
AllowUpdates = true,
DefaultVersion = ApplicationIntegrationCommon.Version,
DisplayName = DisplayName
}).ConfigureAwait(false);
List <ApplicationSummary> applicationSummaries = await client.ApplicationOperations.ListApplicationSummaries().ToListAsync().ConfigureAwait(false);
ApplicationSummary applicationSummary = applicationSummaries.First();
Assert.Equal(ApplicationIntegrationCommon.Version, applicationSummary.Versions.First());
Assert.Equal(ApplicationId, applicationSummary.Id);
Assert.Equal(DisplayName, applicationSummary.DisplayName);
ApplicationSummary getApplicationSummary = await client.ApplicationOperations.GetApplicationSummaryAsync(applicationSummary.Id).ConfigureAwait(false);
Assert.Equal(getApplicationSummary.Id, applicationSummary.Id);
Assert.Equal(getApplicationSummary.Versions.Count(), applicationSummary.Versions.Count());
Assert.Equal(getApplicationSummary.DisplayName, applicationSummary.DisplayName);
var appPackage = await mgmtClient.ApplicationPackage.GetAsync(
TestCommon.Configuration.BatchAccountResourceGroup,
accountName,
ApplicationId,
ApplicationIntegrationCommon.Version).ConfigureAwait(false);
Assert.Equal(PackageState.Active, appPackage.State);
Assert.Equal(ApplicationIntegrationCommon.Version, appPackage.Version);
Assert.Equal(ApplicationId, appPackage.Id);
var application = await mgmtClient.Application.GetAsync(TestCommon.Configuration.BatchAccountResourceGroup, accountName, ApplicationId).ConfigureAwait(false);
Assert.Equal(ApplicationIntegrationCommon.Version, application.DefaultVersion);
Assert.Equal(ApplicationId, application.Id);
await AssertPoolWasCreatedWithApplicationReferences(client, poolId, ApplicationId).ConfigureAwait(false);
}
}
};
await SynchronizationContextHelper.RunTestAsync(test, LongRunningTestTimeout);
}
public async Task Bug1360227_AddTasksBatchRequestFailure(bool useJobOperations)
{
const string testName = "Bug1360227_AddTasksBatchRequestFailure";
Random rand = new Random();
object randLock = new object();
BatchClientBehavior customBehavior = new Protocol.RequestInterceptor(request =>
{
var typedRequest = request as Protocol.BatchRequests.TaskAddCollectionBatchRequest;
if (typedRequest != null)
{
var originalServiceRequestFunction = typedRequest.ServiceRequestFunc;
typedRequest.ServiceRequestFunc = token =>
{
lock (randLock)
{
double d = rand.NextDouble();
if (d > 0.3)
{
throw new HttpRequestException("Simulating a network problem");
}
else
{
return(originalServiceRequestFunction(token));
}
}
};
}
});
await SynchronizationContextHelper.RunTestAsync(async() =>
{
using (BatchClient batchCli = await TestUtilities.OpenBatchClientAsync(TestUtilities.GetCredentialsFromEnvironment(), addDefaultRetryPolicy: false))
{
batchCli.JobOperations.CustomBehaviors.Add(customBehavior);
BatchClientParallelOptions parallelOptions = new BatchClientParallelOptions()
{
MaxDegreeOfParallelism = 2
};
var exception = await TestUtilities.AssertThrowsAsync <ParallelOperationsException>(async() =>
await this.AddTasksSimpleTestAsync(batchCli, testName, 397, parallelOptions, useJobOperations: useJobOperations).ConfigureAwait(false)
).ConfigureAwait(false);
Assert.IsType <HttpRequestException>(exception.InnerException);
}
},
TestTimeout);
}
public async Task TestPoolCertificateReferencesWithUpdate()
{
Func <Task> test = async() =>
{
using (BatchClient batchCli = await TestUtilities.OpenBatchClientAsync(TestUtilities.GetCredentialsFromEnvironment()))
{
//Generate the certificates
const string certificatePrefix = "poolwithcertificatereferences";
string cerFilePath = IntegrationTestCommon.GetTemporaryCertificateFilePath(string.Format("{0}.cer", certificatePrefix));
string pfxFilePath = IntegrationTestCommon.GetTemporaryCertificateFilePath(string.Format("{0}.pfx", certificatePrefix));
IEnumerable <Certificate> certificates = GenerateCertificates(batchCli, cerFilePath, pfxFilePath);
try
{
foreach (Certificate certificate in certificates)
{
this.testOutputHelper.WriteLine("Adding certificate with thumbprint: {0}", certificate.Thumbprint);
await certificate.CommitAsync().ConfigureAwait(false);
}
List <CertificateReference> certificateReferences = certificates.Select(cer => new CertificateReference(cer)
{
StoreLocation = CertStoreLocation.LocalMachine,
StoreName = "My",
Visibility = CertificateVisibility.RemoteUser
}).ToList();
await TestPoolCreateAndUpdateWithCertificateReferencesAsync(batchCli, certificateReferences).ConfigureAwait(false);
await TestAutoPoolCreateAndUpdateWithCertificateReferencesAsync(batchCli, certificateReferences).ConfigureAwait(false);
}
finally
{
File.Delete(pfxFilePath);
File.Delete(cerFilePath);
foreach (Certificate certificate in certificates)
{
TestUtilities.DeleteCertificateIfExistsAsync(batchCli, certificate.ThumbprintAlgorithm, certificate.Thumbprint).Wait();
}
foreach (Certificate certificate in certificates)
{
TestUtilities.DeleteCertMonitor(batchCli.CertificateOperations, this.testOutputHelper, certificate.ThumbprintAlgorithm, certificate.Thumbprint);
}
}
}
};
await SynchronizationContextHelper.RunTestAsync(test, TestTimeout);
}
public async Task CanCreateJobAndAutoPoolWithAppPackageReferences()
{
var jobId = Guid.NewGuid().ToString();
const string applicationId = "blender";
Func <Task> test = async() =>
{
using (BatchClient client = await TestUtilities.OpenBatchClientFromEnvironmentAsync().ConfigureAwait(false))
{
var poolInfo = new PoolInformation
{
AutoPoolSpecification = new AutoPoolSpecification
{
PoolSpecification = new PoolSpecification
{
ApplicationPackageReferences = new[]
{
new ApplicationPackageReference
{
ApplicationId = applicationId,
Version = PoolFixture.VMSize,
}
},
CloudServiceConfiguration = new CloudServiceConfiguration(PoolFixture.OSFamily),
VirtualMachineSize = PoolFixture.VMSize,
},
PoolLifetimeOption = PoolLifetimeOption.Job
}
};
CloudJob response = null;
try
{
CloudJob job = client.JobOperations.CreateJob(jobId, poolInfo);
await job.CommitAsync().ConfigureAwait(false);
response = await client.JobOperations.GetJobAsync(jobId).ConfigureAwait(false);
Assert.Equal(response.PoolInformation.AutoPoolSpecification.PoolSpecification.ApplicationPackageReferences.First().ApplicationId, applicationId);
}
finally
{
if (response != null)
{
TestUtilities.DeleteJobIfExistsAsync(client, jobId).Wait();
}
}
}
};
await SynchronizationContextHelper.RunTestAsync(test, TestTimeout);
}
public async Task Bug1360227_AddTasksBatchMultipleBatches(bool useJobOperations)
{
const string testName = "Bug1360227_AddTasksBatchMultipleBatches";
await SynchronizationContextHelper.RunTestAsync(async() =>
{
using (BatchClient batchCli = await TestUtilities.OpenBatchClientFromEnvironmentAsync())
{
await this.AddTasksSimpleTestAsync(batchCli, testName, 550, useJobOperations: useJobOperations).ConfigureAwait(false);
}
},
TestTimeout);
}
public async Task Bug1360227_AddTasksBatchConfirmResultHandlerTaskReadOnly()
{
const string testName = "Bug1360227_ConfirmResultHandlerTaskReadOnly";
Func <AddTaskResult, CancellationToken, AddTaskResultStatus> resultHandlerFunc = (result, token) =>
{
//Count everything as a success
AddTaskResultStatus resultAction = AddTaskResultStatus.Success;
//Try to set a property of the cloud task
InvalidOperationException e = TestUtilities.AssertThrows <InvalidOperationException>(() =>
result.Task.Constraints = new TaskConstraints(TimeSpan.FromSeconds(5), null, null));
Assert.Contains("Write access is not allowed.", e.Message);
//Try to call a method of a CloudTask
//TODO: This should be blocked but isn't right now...
//try
//{
// result.Task.Terminate();
// Debug.Fail("Should not have gotten here");
//}
//catch (Exception e)
//{
// Console.WriteLine(e);
// //Swallow this exception as it is expected
//}
return(resultAction);
};
await SynchronizationContextHelper.RunTestAsync(async() =>
{
using (BatchClient batchCli = await TestUtilities.OpenBatchClientFromEnvironmentAsync())
{
BatchClientParallelOptions parallelOptions = new BatchClientParallelOptions()
{
MaxDegreeOfParallelism = 2
};
await this.AddTasksSimpleTestAsync(
batchCli,
testName,
55,
parallelOptions,
resultHandlerFunc).ConfigureAwait(false);
}
},
TestTimeout);
}
public async Task CanCreateJobAndAutoPoolWithAppPackageReferences()
{
string jobId = Guid.NewGuid().ToString();
var poolInformation = new PoolInformation
{
AutoPoolSpecification = new AutoPoolSpecification
{
PoolSpecification = new PoolSpecification
{
ApplicationPackageReferences = new List <ApplicationPackageReference>
{
new ApplicationPackageReference
{
ApplicationId = ApplicationId,
Version = Version
}
},
CloudServiceConfiguration = new CloudServiceConfiguration(PoolFixture.OSFamily),
VirtualMachineSize = PoolFixture.VMSize,
},
PoolLifetimeOption = PoolLifetimeOption.Job
}
};
async Task test()
{
using BatchClient client = await TestUtilities.OpenBatchClientFromEnvironmentAsync().ConfigureAwait(false);
try
{
var job = client.JobOperations.CreateJob(jobId, poolInformation);
await job.CommitAsync().ConfigureAwait(false);
CloudJob jobResponse = await client.JobOperations.GetJobAsync(jobId).ConfigureAwait(false);
ApplicationPackageReference apr = jobResponse.PoolInformation.AutoPoolSpecification.PoolSpecification.ApplicationPackageReferences.First();
Assert.Equal(ApplicationId, apr.ApplicationId);
Assert.Equal(Version, apr.Version);
}
finally
{
TestUtilities.DeleteJobIfExistsAsync(client, jobId).Wait();
}
}
await SynchronizationContextHelper.RunTestAsync(test, LongTestTimeout);
}
public async Task Job_GetTaskCounts_ReturnsCorrectCountNonZeroTaskSlots()
{
Func <Task> test = async() =>
{
using (BatchClient batchCli = await TestUtilities.OpenBatchClientFromEnvironmentAsync().ConfigureAwait(false))
{
string jobId = "NonZeroTaskSlots-" + TestUtilities.GetMyName();
try
{
PoolInformation poolInfo = new PoolInformation()
{
PoolId = "Fake"
};
CloudJob unboundJob = batchCli.JobOperations.CreateJob(jobId, poolInfo);
await unboundJob.CommitAsync().ConfigureAwait(false);
await unboundJob.RefreshAsync().ConfigureAwait(false);
CloudTask t1 = new CloudTask("t1", "cmd /c dir");
t1.RequiredSlots = 2;
CloudTask t2 = new CloudTask("t2", "cmd /c ping 127.0.0.1 -n 4");
t2.RequiredSlots = 3;
await unboundJob.AddTaskAsync(new[] { t1, t2 }).ConfigureAwait(false);
await Task.Delay(TimeSpan.FromSeconds(5)).ConfigureAwait(false); // Give the service some time to get the counts
var counts = await unboundJob.GetTaskCountsAsync().ConfigureAwait(false);
var retTask1 = await unboundJob.GetTaskAsync(t1.Id);
Assert.Equal(t1.RequiredSlots, retTask1.RequiredSlots);
var retTask2 = await unboundJob.GetTaskAsync(t1.Id);
Assert.Equal(t1.RequiredSlots, retTask2.RequiredSlots);
Assert.Equal(2, counts.TaskCounts.Active);
// Task slots counts is currently broken
// Assert.Equal(5, counts.TaskSlotCounts.Active);
}
finally
{
await TestUtilities.DeleteJobIfExistsAsync(batchCli, jobId);
}
}
};
await SynchronizationContextHelper.RunTestAsync(test, TestTimeout);
}
public async Task IfAPoolSpecifiesANonExistentApplicationPackage_ThenCommittingThePoolThrowsAnException()
{
await SynchronizationContextHelper.RunTestAsync(async() =>
{
var poolId = "app-ref-test-3-" + Guid.NewGuid();
using (BatchClient client = TestUtilities.OpenBatchClientFromEnvironmentAsync().Result)
{
CloudPool pool = client.PoolOperations.CreatePool(poolId, PoolFixture.VMSize, new CloudServiceConfiguration(PoolFixture.OSFamily));
pool.ApplicationPackageReferences = new[] { new ApplicationPackageReference {
ApplicationId = "dud", Version = Version
} };
await TestUtilities.AssertThrowsAsync <BatchException>(() => pool.CommitAsync()).ConfigureAwait(false);
}
}, LongTestTimeout);
}
public async Task Bug1360227_AddTasksBatchCancelation(bool useJobOperations)
{
const string testName = "Bug1360227_AddTasksBatchCancelation";
const int taskCount = 322;
await SynchronizationContextHelper.RunTestAsync(async() =>
{
using (BatchClient batchCli = await TestUtilities.OpenBatchClientFromEnvironmentAsync())
{
using (CancellationTokenSource source = new CancellationTokenSource())
{
BatchClientParallelOptions parallelOptions = new BatchClientParallelOptions()
{
MaxDegreeOfParallelism = 2,
CancellationToken = source.Token
};
System.Threading.Tasks.Task t = this.AddTasksSimpleTestAsync(
batchCli,
testName,
taskCount,
parallelOptions,
useJobOperations: useJobOperations);
Thread.Sleep(TimeSpan.FromSeconds(.3)); //Wait till we get into the workflow
this.testOutputHelper.WriteLine("Canceling the work flow");
source.Cancel();
try
{
await t.ConfigureAwait(false);
}
catch (Exception e)
{
//This is expected to throw one of two possible exception types...
if (!(e is TaskCanceledException) && !(e is OperationCanceledException))
{
throw new ThrowsException(typeof(TaskCanceledException), e);
}
}
}
}
},
TestTimeout);
}
public async Task Bug1360227_AddTasksBatchHugeTaskCount()
{
const string testName = "Bug1360227_AddTasksBatchHugeTaskCount";
await SynchronizationContextHelper.RunTestAsync(async() =>
{
using (BatchClient batchCli = await TestUtilities.OpenBatchClientFromEnvironmentAsync())
{
BatchClientParallelOptions parallelOptions = new BatchClientParallelOptions()
{
MaxDegreeOfParallelism = 25
};
await this.AddTasksSimpleTestAsync(batchCli, testName, 5025, parallelOptions).ConfigureAwait(false);
}
},
LongTestTimeout);
}
public async Task AddTasksRequestEntityTooLarge_ReduceChunkSize()
{
const string testName = "AddTasksRequestEntityTooLarge_ReduceChunkSize";
List <ResourceFile> resourceFiles = new List <ResourceFile>();
ResourceFile resourceFile;
int countChunksOf100 = 0;
int numTasks = 176;
int degreesOfParallelism = 2;
BatchClientBehavior customBehavior = new Protocol.RequestInterceptor(request =>
{
var typedRequest = request as Protocol.BatchRequests.TaskAddCollectionBatchRequest;
if (typedRequest != null)
{
if (typedRequest.Parameters.Count > 50)
{
Interlocked.Increment(ref countChunksOf100);
}
}
});
// If this test fails try increasing the size of the Task in case maximum size increase
for (int i = 0; i < 100; i++)
{
resourceFile = ResourceFile.FromUrl("https://mystorageaccount.blob.core.windows.net/files/resourceFile" + i, "resourceFile" + i);
resourceFiles.Add(resourceFile);
}
await SynchronizationContextHelper.RunTestAsync(async() =>
{
using (BatchClient batchCli = TestUtilities.OpenBatchClient(TestUtilities.GetCredentialsFromEnvironment(), addDefaultRetryPolicy: false))
{
batchCli.JobOperations.CustomBehaviors.Add(customBehavior);
BatchClientParallelOptions parallelOptions = new BatchClientParallelOptions()
{
MaxDegreeOfParallelism = degreesOfParallelism
};
await AddTasksSimpleTestAsync(batchCli, testName, numTasks, parallelOptions, resourceFiles: resourceFiles).ConfigureAwait(false);
}
},
TestTimeout);
Assert.True(countChunksOf100 <= Math.Min(Math.Ceiling(numTasks / 100.0), degreesOfParallelism));
}
public async Task BatchRequestWithShortClientSideTimeout()
{
//The ThrowsAnyAsync must be outside the RunTestAsync or it will deadlock
await Assert.ThrowsAnyAsync <OperationCanceledException>(async() =>
await SynchronizationContextHelper.RunTestAsync(async() =>
{
using (BatchClient client = await TestUtilities.OpenBatchClientAsync(TestUtilities.GetCredentialsFromEnvironment(), addDefaultRetryPolicy: false))
{
List <BatchClientBehavior> customBehaviors = new List <BatchClientBehavior>();
customBehaviors.Add(new RequestInterceptor((req) =>
{
//Set the timeout to something small so it is guaranteed to expire before the service has responded
req.Timeout = TimeSpan.FromMilliseconds(25);
}));
await client.JobOperations.GetJobAsync("Foo", additionalBehaviors: customBehaviors).ConfigureAwait(false);
}
},
TestTimeout));
}
public async Task IfAPoolIsCommittedWithApplicationPackageReferences_ThenThoseReferencesArePersistedInTheService()
{
var poolId = "app-ref-test-1-" + Guid.NewGuid();
async Task test()
{
using BatchClient client = await TestUtilities.OpenBatchClientFromEnvironmentAsync();
CloudPool newPool = null;
try
{
List <ApplicationSummary> applicationSummaries = await client.ApplicationOperations.ListApplicationSummaries().ToListAsync().ConfigureAwait(false);
foreach (var applicationSummary in applicationSummaries.Where(app => app.Id == ApplicationId))
{
Assert.True(true, string.Format("{0} was found.", applicationSummary.Id));
}
CloudPool pool = client.PoolOperations.CreatePool(poolId, PoolFixture.VMSize, new CloudServiceConfiguration(PoolFixture.OSFamily));
pool.ApplicationPackageReferences = new[] { new ApplicationPackageReference {
ApplicationId = ApplicationId, Version = Version
} };
await pool.CommitAsync().ConfigureAwait(false);
newPool = await client.PoolOperations.GetPoolAsync(poolId).ConfigureAwait(false);
ApplicationPackageReference apr = newPool.ApplicationPackageReferences.First();
Assert.Equal(ApplicationId, apr.ApplicationId);
Assert.Equal(Version, apr.Version);
}
finally
{
TestUtilities.DeletePoolIfExistsAsync(client, poolId).Wait();
}
}
await SynchronizationContextHelper.RunTestAsync(test, LongTestTimeout);
}
public async Task Bug1360227_AddTasksBatchTimeout(bool useJobOperations)
{
const string testName = "Bug1360227_AddTasksBatchTimeout";
await SynchronizationContextHelper.RunTestAsync(async() =>
{
using (BatchClient batchCli = await TestUtilities.OpenBatchClientFromEnvironmentAsync())
{
var exception = await TestUtilities.AssertThrowsAsync <ParallelOperationsException>(
async() => await this.AddTasksSimpleTestAsync(
batchCli,
testName,
311,
timeout: TimeSpan.FromSeconds(1),
useJobOperations: useJobOperations).ConfigureAwait(false)).ConfigureAwait(false);
Assert.IsType <TimeoutException>(exception.InnerException);
}
},
TestTimeout);
}
public async Task BatchRequestWithShortUserCancellationToken()
{
await SynchronizationContextHelper.RunTestAsync(async() =>
{
//Set the timeout to something small so it is guaranteed to expire before the service has responded
using (CancellationTokenSource tokenSource = new CancellationTokenSource(TimeSpan.FromMilliseconds(25)))
{
using (BatchClient client = await TestUtilities.OpenBatchClientAsync(TestUtilities.GetCredentialsFromEnvironment(), addDefaultRetryPolicy: false))
{
List <BatchClientBehavior> customBehaviors = new List <BatchClientBehavior>();
customBehaviors.Add(new RequestInterceptor((req) =>
{
req.CancellationToken = tokenSource.Token;
}));
await TestUtilities.AssertThrowsAsync <OperationCanceledException>(async() =>
await client.JobOperations.GetJobAsync("Foo", additionalBehaviors: customBehaviors).ConfigureAwait(false)).ConfigureAwait(false);
}
}
},
TestTimeout);
}
public async Task IfThereAreApplicationsInTheAccountThenListApplicationSummariesReturnsThem()
{
Func <Task> test = async() =>
{
using (BatchClient client = await TestUtilities.OpenBatchClientFromEnvironmentAsync().ConfigureAwait(false))
{
List <ApplicationSummary> applicationSummaries = await client.ApplicationOperations.ListApplicationSummaries().ToListAsync().ConfigureAwait(false);
var application = applicationSummaries.First(app => app.Id == ApplicationId);
Assert.Equal(ApplicationId, application.Id);
Assert.Equal(ApplicationIntegrationCommon.Version, application.Versions.FirstOrDefault());
ApplicationSummary getApplicationSummary = await client.ApplicationOperations.GetApplicationSummaryAsync(application.Id).ConfigureAwait(false);
Assert.Equal(ApplicationId, getApplicationSummary.Id);
Assert.Equal(ApplicationIntegrationCommon.Version, getApplicationSummary.Versions.First());
}
};
await SynchronizationContextHelper.RunTestAsync(test, TestTimeout);
}
public async Task UpdateAnExistingPoolWithNewApplicationPackageReferences_AndChecksTheApplicationPackageReferencesIsOnThePool()
{
var poolId = "app-ref-test-2-" + Guid.NewGuid();
Func <Task> test = async() =>
{
using (BatchClient client = await TestUtilities.OpenBatchClientFromEnvironmentAsync())
{
try
{
CloudPool pool = client.PoolOperations.CreatePool(poolId, PoolFixture.VMSize, new CloudServiceConfiguration(PoolFixture.OSFamily));
await pool.CommitAsync().ConfigureAwait(false);
pool = await client.PoolOperations.GetPoolAsync(poolId).ConfigureAwait(false);
Assert.Null(pool.ApplicationPackageReferences);
pool.ApplicationPackageReferences = new[] { new ApplicationPackageReference {
ApplicationId = ApplicationId, Version = Version
} };
await pool.CommitAsync().ConfigureAwait(false);
CloudPool updatedPool = await client.PoolOperations.GetPoolAsync(poolId).ConfigureAwait(false);
ApplicationPackageReference apr = updatedPool.ApplicationPackageReferences.First();
Assert.Equal(ApplicationId, apr.ApplicationId);
Assert.Equal(Version, apr.Version);
}
finally
{
TestUtilities.DeletePoolIfExistsAsync(client, poolId).Wait();
}
}
};
await SynchronizationContextHelper.RunTestAsync(test, LongTestTimeout);
}
public async Task TestCertificateVerbs()
{
Func <Task> test = async() =>
{
using (BatchClient batchCli = await TestUtilities.OpenBatchClientAsync(TestUtilities.GetCredentialsFromEnvironment()))
{
//Generate the certificates
const string certificatePrefix = "testcertificatecrud";
string cerFilePath = IntegrationTestCommon.GetTemporaryCertificateFilePath(string.Format("{0}.cer", certificatePrefix));
string pfxFilePath = IntegrationTestCommon.GetTemporaryCertificateFilePath(string.Format("{0}.pfx", certificatePrefix));
IEnumerable <Certificate> certificates = GenerateCertificates(batchCli, cerFilePath, pfxFilePath);
try
{
foreach (Certificate certificate in certificates)
{
this.testOutputHelper.WriteLine("Adding certificate with thumbprint: {0}", certificate.Thumbprint);
await certificate.CommitAsync().ConfigureAwait(false);
Certificate boundCert = await batchCli.CertificateOperations.GetCertificateAsync(
certificate.ThumbprintAlgorithm,
certificate.Thumbprint).ConfigureAwait(false);
Assert.Equal(certificate.Thumbprint, boundCert.Thumbprint);
Assert.Equal(certificate.ThumbprintAlgorithm, boundCert.ThumbprintAlgorithm);
Assert.NotNull(boundCert.Url);
Certificate certLowerDetail = await batchCli.CertificateOperations.GetCertificateAsync(
certificate.ThumbprintAlgorithm,
certificate.Thumbprint,
new ODATADetailLevel()
{
SelectClause = "thumbprint, thumbprintAlgorithm"
}).ConfigureAwait(false);
// confirm lower detail level
Assert.Null(certLowerDetail.Url);
//test refresh to higher detail level
await certLowerDetail.RefreshAsync();
// confirm higher detail level
Assert.NotNull(certLowerDetail.Url);
// test refresh can lower detail level
await certLowerDetail.RefreshAsync(new ODATADetailLevel()
{
SelectClause = "thumbprint, thumbprintAlgorithm"
});
// confirm lower detail level via refresh
Assert.Null(certLowerDetail.Url);
}
List <CertificateReference> certificateReferences = certificates.Select(cer => new CertificateReference(cer)
{
StoreLocation = CertStoreLocation.LocalMachine,
StoreName = "My",
Visibility = CertificateVisibility.RemoteUser
}).ToList();
await TestCancelDeleteCertificateAsync(batchCli, certificateReferences, certificates.First()).ConfigureAwait(false);
}
finally
{
File.Delete(pfxFilePath);
File.Delete(cerFilePath);
foreach (Certificate certificate in certificates)
{
TestUtilities.DeleteCertificateIfExistsAsync(batchCli, certificate.ThumbprintAlgorithm, certificate.Thumbprint).Wait();
}
foreach (Certificate certificate in certificates)
{
TestUtilities.DeleteCertMonitor(batchCli.CertificateOperations, this.testOutputHelper, certificate.ThumbprintAlgorithm, certificate.Thumbprint);
}
}
}
};
await SynchronizationContextHelper.RunTestAsync(test, TestTimeout);
}
public async Task OnlineOfflineTest()
{
await SynchronizationContextHelper.RunTestAsync(async() =>
{
using (BatchClient batchCli = await TestUtilities.OpenBatchClientFromEnvironmentAsync())
{
TimeSpan refreshPollingTimeout = TimeSpan.FromMinutes(3);
CloudPool pool = this.poolFixture.Pool;
List <ComputeNode> nodes = pool.ListComputeNodes().ToList();
Assert.True(nodes.Count > 0);
// pick a victim compute node. cleanup code needs this set
ComputeNode victim = nodes[0];
try
{
Assert.True(victim.SchedulingState.HasValue && (SchedulingState.Enabled == victim.SchedulingState));
Assert.True(victim.State.HasValue && (ComputeNodeState.Idle == victim.State));
// PoolOperations methods
{
// disable task scheduling
batchCli.PoolOperations.DisableComputeNodeScheduling(pool.Id, victim.Id, DisableComputeNodeSchedulingOption.Terminate);
// Li says state change is not atomic so we will sleep
// asserted above this node is idle so no need to wait for task fussery
await TestUtilities.RefreshBasedPollingWithTimeoutAsync(
refreshing: victim,
condition: () => Task.FromResult(victim.SchedulingState.HasValue && (SchedulingState.Disabled == victim.SchedulingState)),
timeout: refreshPollingTimeout).ConfigureAwait(false);
Assert.Equal <SchedulingState?>(SchedulingState.Disabled, victim.SchedulingState);
// enable task scheduling
batchCli.PoolOperations.EnableComputeNodeScheduling(pool.Id, victim.Id);
await TestUtilities.RefreshBasedPollingWithTimeoutAsync(
refreshing: victim,
condition: () => Task.FromResult(victim.SchedulingState.HasValue && (SchedulingState.Enabled == victim.SchedulingState)),
timeout: refreshPollingTimeout);
Assert.Equal <SchedulingState?>(SchedulingState.Enabled, victim.SchedulingState);
}
// ComputeNode methods
{
// disable task scheduling
victim.DisableScheduling(DisableComputeNodeSchedulingOption.TaskCompletion);
await TestUtilities.RefreshBasedPollingWithTimeoutAsync(
refreshing: victim,
condition: () => Task.FromResult(victim.SchedulingState.HasValue && (SchedulingState.Disabled == victim.SchedulingState)),
timeout: refreshPollingTimeout).ConfigureAwait(false);
Assert.Equal <SchedulingState?>(SchedulingState.Disabled, victim.SchedulingState);
// enable task scheduling
victim.EnableScheduling();
await TestUtilities.RefreshBasedPollingWithTimeoutAsync(
refreshing: victim,
condition: () => Task.FromResult(victim.SchedulingState.HasValue && (SchedulingState.Enabled == victim.SchedulingState)),
timeout: refreshPollingTimeout).ConfigureAwait(false);
Assert.Equal <SchedulingState?>(SchedulingState.Enabled, victim.SchedulingState);
// now test azureerror code for: NodeAlreadyInTargetSchedulingState
bool gotCorrectException = false;
try
{
victim.EnableScheduling(); // it is already enabled so this should trigger exception
}
catch (Exception ex)
{
TestUtilities.AssertIsBatchExceptionAndHasCorrectAzureErrorCode(ex, Microsoft.Azure.Batch.Common.BatchErrorCodeStrings.NodeAlreadyInTargetSchedulingState, this.testOutputHelper);
gotCorrectException = true;
}
if (!gotCorrectException)
{
throw new Exception("OnlineOfflineTest: failed to see an exception for NodeAlreadyInTargetSchedulingState test");
}
}
}
finally // restore state of victim compute node
{
try
{
// do not pollute the shared pool with disabled scheduling
if (null != victim)
{
victim.EnableScheduling();
}
}
catch (Exception ex)
{
this.testOutputHelper.WriteLine(string.Format("OnlineOfflineTest: exception during exit trying to restore scheduling state: {0}", ex.ToString()));
}
}
}
},
TestTimeout);
}
public async Task CanCreateAndUpdateJobScheduleWithApplicationReferences()
{
var jobId = Guid.NewGuid().ToString();
const string newVersion = "2.0";
var poolInformation = new PoolInformation
{
AutoPoolSpecification = new AutoPoolSpecification
{
PoolSpecification = new PoolSpecification
{
ApplicationPackageReferences = new List <ApplicationPackageReference>
{
new ApplicationPackageReference
{
ApplicationId = ApplicationId,
Version = Version
}
},
CloudServiceConfiguration = new CloudServiceConfiguration(PoolFixture.OSFamily),
VirtualMachineSize = PoolFixture.VMSize,
},
PoolLifetimeOption = PoolLifetimeOption.JobSchedule,
KeepAlive = false,
}
};
Schedule schedule = new Schedule {
DoNotRunAfter = DateTime.UtcNow.AddMinutes(5), RecurrenceInterval = TimeSpan.FromMinutes(2)
};
JobSpecification jobSpecification = new JobSpecification(poolInformation);
using BatchClient client = await TestUtilities.OpenBatchClientFromEnvironmentAsync().ConfigureAwait(false);
CloudJobSchedule cloudJobSchedule = client.JobScheduleOperations.CreateJobSchedule(jobId, schedule, jobSpecification);
async Task test()
{
CloudJobSchedule updatedBoundJobSchedule = null;
try
{
await cloudJobSchedule.CommitAsync().ConfigureAwait(false);
CloudJobSchedule boundJobSchedule = TestUtilities.WaitForJobOnJobSchedule(client.JobScheduleOperations, jobId);
ApplicationPackageReference apr = boundJobSchedule.JobSpecification.PoolInformation.AutoPoolSpecification.PoolSpecification.ApplicationPackageReferences.First();
Assert.Equal(ApplicationId, apr.ApplicationId);
Assert.Equal(Version, apr.Version);
boundJobSchedule.JobSpecification.PoolInformation.AutoPoolSpecification.PoolSpecification.ApplicationPackageReferences = new[]
{
new ApplicationPackageReference()
{
ApplicationId = ApplicationId,
Version = newVersion
}
};
await boundJobSchedule.CommitAsync().ConfigureAwait(false);
await boundJobSchedule.RefreshAsync().ConfigureAwait(false);
updatedBoundJobSchedule = await client.JobScheduleOperations.GetJobScheduleAsync(jobId).ConfigureAwait(false);
ApplicationPackageReference updatedApr =
updatedBoundJobSchedule.JobSpecification.PoolInformation.AutoPoolSpecification.PoolSpecification.ApplicationPackageReferences
.First();
Assert.Equal(ApplicationId, updatedApr.ApplicationId);
Assert.Equal(newVersion, updatedApr.Version);
}
finally
{
TestUtilities.DeleteJobScheduleIfExistsAsync(client, jobId).Wait();
}
}
await SynchronizationContextHelper.RunTestAsync(test, LongTestTimeout);
}
public async Task Bug1360227_AddTasksBatchWithFilesToStage(bool useJobOperations)
{
const string testName = "Bug1360227_AddTasksBatchWithFilesToStage";
const int taskCount = 499;
List <string> localFilesToStage = new List <string>();
localFilesToStage.Add("TestResources\\Data.txt");
ConcurrentBag <ConcurrentDictionary <Type, IFileStagingArtifact> > artifacts = new ConcurrentBag <ConcurrentDictionary <Type, IFileStagingArtifact> >();
List <int> legArtifactsCountList = new List <int>();
using (CancellationTokenSource cts = new CancellationTokenSource())
{
//Spawn a thread to monitor the files to stage as we go - we should observe that
Task t = Task.Factory.StartNew(() =>
{
while (!cts.Token.IsCancellationRequested)
{
legArtifactsCountList.Add(artifacts.Count);
Thread.Sleep(TimeSpan.FromSeconds(1));
}
});
await SynchronizationContextHelper.RunTestAsync(async() =>
{
StagingStorageAccount storageCredentials = TestUtilities.GetStorageCredentialsFromEnvironment();
using (BatchClient batchCli = await TestUtilities.OpenBatchClientFromEnvironmentAsync())
{
await this.AddTasksSimpleTestAsync(
batchCli,
testName,
taskCount,
parallelOptions: new BatchClientParallelOptions()
{
MaxDegreeOfParallelism = 2
},
storageCredentials: storageCredentials,
localFilesToStage: localFilesToStage,
fileStagingArtifacts: artifacts,
useJobOperations: useJobOperations).ConfigureAwait(false);
cts.Cancel();
await t.ConfigureAwait(false); //Wait for the spawned thread to exit
this.testOutputHelper.WriteLine("File staging leg count: [");
foreach (int fileStagingArtifactsCount in legArtifactsCountList)
{
this.testOutputHelper.WriteLine(fileStagingArtifactsCount + ", ");
}
this.testOutputHelper.WriteLine("]");
const int expectedFinalFileStagingArtifactsCount = taskCount / 100 + 1;
const int expectedInitialFileStagingArtifactsCount = 0;
Assert.Equal(expectedInitialFileStagingArtifactsCount, legArtifactsCountList.First());
Assert.Equal(expectedFinalFileStagingArtifactsCount, legArtifactsCountList.Last());
}
},
TestTimeout);
}
}
public async Task Bug1360227_AddTasksBatchRetry(bool useJobOperations)
{
const string testName = "Bug1360227_AddTasksBatchRetry";
Random rand = new Random();
object randLock = new object();
int numberOfTasksWhichHitClientError = 0;
int numberOfTasksWhichWereForcedToRetry = 0;
Func <AddTaskResult, CancellationToken, AddTaskResultStatus> resultHandlerFunc = (result, token) =>
{
this.testOutputHelper.WriteLine("Task: {0} got status code: {1}", result.TaskId, result.Status);
AddTaskResultStatus resultAction;
if (result.Status == AddTaskStatus.ClientError)
{
++numberOfTasksWhichHitClientError;
return(AddTaskResultStatus.Success); //Have to count client error as success
}
lock (randLock)
{
double d = rand.NextDouble();
if (d > 0.8)
{
this.testOutputHelper.WriteLine("Forcing retry for task: {0}", result.TaskId);
resultAction = AddTaskResultStatus.Retry;
++numberOfTasksWhichWereForcedToRetry;
}
else
{
resultAction = AddTaskResultStatus.Success;
}
}
return(resultAction);
};
await SynchronizationContextHelper.RunTestAsync(async() =>
{
StagingStorageAccount storageCredentials = TestUtilities.GetStorageCredentialsFromEnvironment();
using (BatchClient batchCli = await TestUtilities.OpenBatchClientFromEnvironmentAsync())
{
BatchClientParallelOptions parallelOptions = new BatchClientParallelOptions()
{
MaxDegreeOfParallelism = 2
};
await this.AddTasksSimpleTestAsync(
batchCli,
testName,
1281,
parallelOptions,
resultHandlerFunc,
storageCredentials,
new List <string> {
"TestResources\\Data.txt"
},
useJobOperations: useJobOperations).ConfigureAwait(false);
}
},
LongTestTimeout);
//Ensure that we forced some tasks to retry
this.testOutputHelper.WriteLine("Forced a total of {0} tasks to retry", numberOfTasksWhichWereForcedToRetry);
Assert.True(numberOfTasksWhichWereForcedToRetry > 0);
Assert.Equal(numberOfTasksWhichWereForcedToRetry, numberOfTasksWhichHitClientError);
}
