private static void AssertGoodJobReleaseTaskOM(JobReleaseTask jobRelease)
{
Assert.Equal(JobReleaseTaskCommandLine, jobRelease.CommandLine);
Assert.NotNull(jobRelease.EnvironmentSettings);
Assert.Equal(1, jobRelease.EnvironmentSettings.Count);
Assert.Equal(JobRelEnvSettingOM.Name, jobRelease.EnvironmentSettings[0].Name);
Assert.Equal(JobRelEnvSettingOM.Value, jobRelease.EnvironmentSettings[0].Value);
Assert.Equal(JobRelMaxWallClockTime, jobRelease.MaxWallClockTime);
Assert.Equal(JobRelId, jobRelease.Id);
Assert.Equal(JobRelRetentionTime, jobRelease.RetentionTime);
Assert.Equal(JobRelUserSpec.ElevationLevel, jobRelease.UserIdentity.AutoUser.ElevationLevel);
Assert.Equal(JobRelUserSpec.Scope, jobRelease.UserIdentity.AutoUser.Scope);
}
public static async Task <CloudJob> CreateJobIfNotExistAsync(
BatchClient batchClient,
string poolId,
string jobId,
bool usesTaskDependencies = false,
JobPreparationTask prepTask = null,
JobReleaseTask releaseTask = null)
{
CloudJob job = await GetJobIfExistAsync(batchClient, jobId).ConfigureAwait(continueOnCapturedContext: false);
if (job == null)
{
Console.WriteLine("Job {0} not found, creating...", jobId);
CloudJob unboundJob = batchClient.JobOperations.CreateJob(jobId, new PoolInformation()
{
PoolId = poolId
});
unboundJob.UsesTaskDependencies = usesTaskDependencies;
if (prepTask != null)
{
unboundJob.JobPreparationTask = prepTask;
}
if (releaseTask != null)
{
unboundJob.JobReleaseTask = releaseTask;
}
await unboundJob.CommitAsync().ConfigureAwait(continueOnCapturedContext: false);
// Get the bound version of the job with all of its properties populated
job = await batchClient.JobOperations.GetJobAsync(jobId).ConfigureAwait(continueOnCapturedContext: false);
}
return(job);
}
public void TestOMJobSpecAndRelease()
{
Action test = () =>
{
StagingStorageAccount stagingCreds = TestUtilities.GetStorageCredentialsFromEnvironment();
using (BatchClient client = TestUtilities.OpenBatchClient(TestUtilities.GetCredentialsFromEnvironment()))
{
string jsId = "JobPrepAndRelease-" + /* "OM-static-c" */ "dynamic-" + CraftTimeString() + "-" + TestUtilities.GetMyName();
try
{
// increase request timeout interceptor
Protocol.RequestInterceptor increaseTimeoutInterceptor =
new Protocol.RequestInterceptor((x) =>
{
this.testOutputHelper.WriteLine("TestOMJobSpecAndRelease: setting request timeout. Request type: " + x.GetType().ToString() + ", ClientRequestID: " + x.Options.ClientRequestId);
var timeoutOptions = x.Options as Protocol.Models.ITimeoutOptions;
timeoutOptions.Timeout = 5 * 60;
});
// lets use a timer too
CallTimerViaInterceptors timerInterceptor = new CallTimerViaInterceptors();
// seeing client side timeouts... so increase the durations on every call
client.CustomBehaviors.Add(increaseTimeoutInterceptor);
// add a call timer spammer/logger
client.CustomBehaviors.Add(timerInterceptor.ReqInterceptor);
// get some resource files to play with
IList <ResourceFile> resFiles = UploadFilesMakeResFiles(stagingCreds);
// create job schedule with prep/release
{
CloudJobSchedule unboundJobSchedule = client.JobScheduleOperations.CreateJobSchedule(jsId, null, null);
unboundJobSchedule.JobSpecification = new JobSpecification(new PoolInformation());
unboundJobSchedule.JobSpecification.PoolInformation.PoolId = this.poolFixture.PoolId;
unboundJobSchedule.Schedule = new Schedule()
{
RecurrenceInterval = TimeSpan.FromMinutes(3)
};
// add the jobPrep task to the job schedule
{
JobPreparationTask prep = new JobPreparationTask(JobPrepCommandLine);
unboundJobSchedule.JobSpecification.JobPreparationTask = prep;
List <EnvironmentSetting> prepEnvSettings = new List <EnvironmentSetting>();
prepEnvSettings.Add(JobPrepEnvSettingOM);
prep.EnvironmentSettings = prepEnvSettings;
prep.Id = JobPrepId;
prep.RerunOnComputeNodeRebootAfterSuccess = JobPrepRerunOnComputeNodeRebootAfterSuccess;
prep.ResourceFiles = resFiles; // bug: incorrect type this should be IList<>
/*
* prep.ResourceFiles = new List<ResourceFile>(); // this is actually read into our concurrent iList thing
*
* // why not, merge them in. exersize the concurent IList thing
* foreach (ResourceFile curRF in resFiles)
* {
* prep.ResourceFiles.Add(curRF);
* }
*/
prep.UserIdentity = new UserIdentity(JobPrepUserSpec);
prep.Constraints = JobPrepTaskConstraintsOM;
prep.WaitForSuccess = JobPrepWaitForSuccessCreate;
}
// add a jobRelease task to the job schedule
{
JobReleaseTask relTask = new JobReleaseTask(JobReleaseTaskCommandLine);
unboundJobSchedule.JobSpecification.JobReleaseTask = relTask;
List <EnvironmentSetting> relEnvSettings = new List <EnvironmentSetting>();
relEnvSettings.Add(JobRelEnvSettingOM);
relTask.EnvironmentSettings = relEnvSettings;
relTask.MaxWallClockTime = JobRelMaxWallClockTime;
relTask.Id = JobRelId;
relTask.ResourceFiles = null;
relTask.ResourceFiles = new List <ResourceFile>();
// why not, merge them in. work the concurrent IList thing
foreach (ResourceFile curRF in resFiles)
{
relTask.ResourceFiles.Add(curRF);
}
relTask.RetentionTime = JobRelRetentionTime;
relTask.UserIdentity = new UserIdentity(JobRelUserSpec);
}
// set JobCommonEnvSettings
{
List <EnvironmentSetting> jobCommonES = new List <EnvironmentSetting>();
jobCommonES.Add(JobCommonEnvSettingOM);
unboundJobSchedule.JobSpecification.CommonEnvironmentSettings = jobCommonES;
}
// add the job schedule to the service
unboundJobSchedule.Commit();
}
// now we have a jobschedule with jobprep/release...now test the values on the jobschedule
{
CloudJobSchedule boundJobSchedule = client.JobScheduleOperations.GetJobSchedule(jsId);
Assert.NotNull(boundJobSchedule);
Assert.NotNull(boundJobSchedule.JobSpecification);
Assert.NotNull(boundJobSchedule.JobSpecification.JobPreparationTask);
Assert.NotNull(boundJobSchedule.JobSpecification.JobReleaseTask);
Assert.NotNull(boundJobSchedule.JobSpecification.CommonEnvironmentSettings);
AssertGoodCommonEnvSettingsOM(boundJobSchedule.JobSpecification.CommonEnvironmentSettings);
AssertGoodJobPrepTaskOM(boundJobSchedule.JobSpecification.JobPreparationTask);
AssertGoodJobReleaseTaskOM(boundJobSchedule.JobSpecification.JobReleaseTask);
AssertGoodResourceFiles(resFiles, boundJobSchedule.JobSpecification.JobPreparationTask.ResourceFiles);
AssertGoodResourceFiles(resFiles, boundJobSchedule.JobSpecification.JobReleaseTask.ResourceFiles);
//todo: test mutability
}
CloudJobSchedule boundJobScheduleWithJob; // set on job test
// test the values on the job
{
boundJobScheduleWithJob = TestUtilities.WaitForJobOnJobSchedule(client.JobScheduleOperations, jsId);
CloudJob bndJob = client.JobOperations.GetJob(boundJobScheduleWithJob.ExecutionInformation.RecentJob.Id);
Assert.NotNull(bndJob);
Assert.NotNull(bndJob.CommonEnvironmentSettings);
Assert.NotNull(bndJob.JobPreparationTask);
Assert.NotNull(bndJob.JobReleaseTask);
AssertGoodCommonEnvSettingsOM(bndJob.CommonEnvironmentSettings as IList <EnvironmentSetting>
/* we know it is our internal IList */);
AssertGoodJobPrepTaskOM(bndJob.JobPreparationTask);
AssertGoodJobReleaseTaskOM(bndJob.JobReleaseTask);
AssertGoodResourceFiles(resFiles, bndJob.JobPreparationTask.ResourceFiles);
AssertGoodResourceFiles(resFiles, bndJob.JobReleaseTask.ResourceFiles);
//TODO: test immutability
}
// used for on get-status test
CloudJobSchedule updatedJobSchedule;
// test update on the WI jobprep/jobrelease
{
// change props
boundJobScheduleWithJob.JobSpecification.JobPreparationTask.WaitForSuccess = JobPrepWaitForSuccessUpdate;
// commit changes
boundJobScheduleWithJob.Commit();
// get new values
updatedJobSchedule = client.JobScheduleOperations.GetJobSchedule(jsId);
// confirm values changed
Assert.Equal(JobPrepWaitForSuccessUpdate, updatedJobSchedule.JobSpecification.JobPreparationTask.WaitForSuccess);
}
TestGetPrepReleaseStatusCalls(client, updatedJobSchedule, this.poolFixture.PoolId, resFiles);
}
finally
{
// cleanup
TestUtilities.DeleteJobScheduleIfExistsAsync(client, jsId).Wait();
}
}
};
SynchronizationContextHelper.RunTest(test, LongTestTimeout);
}
public void TestOMJobReleaseSchedulingError()
{
string jobId = "TestOMJobReleaseSchedulingError-" + CraftTimeString() + "-" + TestUtilities.GetMyName();
Action test = () =>
{
using (BatchClient client = TestUtilities.OpenBatchClient(TestUtilities.GetCredentialsFromEnvironment()))
{
try
{
// create job schedule with prep that succeeds and release the triggers scheduling error
{
PoolInformation poolInfo = new PoolInformation()
{
PoolId = this.poolFixture.PoolId
};
CloudJob unboundJob = client.JobOperations.CreateJob(jobId, poolInfo);
// add the jobPrep task to the job
{
JobPreparationTask prep = new JobPreparationTask("cmd /c echo the quick job prep jumped over the...");
unboundJob.JobPreparationTask = prep;
prep.WaitForSuccess = false; // we don't really care but why not set this
}
// add a jobRelease task to the job
{
JobReleaseTask relTask = new JobReleaseTask("cmd /c echo Job Release Task");
unboundJob.JobReleaseTask = relTask;
ResourceFile[] badResFiles = { ResourceFile.FromUrl("https://not.a.domain.invalid/file", "bob.txt") };
relTask.ResourceFiles = badResFiles;
relTask.Id = "jobRelease";
}
// add the job to the service
unboundJob.Commit();
}
// add a trivial task to force the JP
client.JobOperations.AddTask(jobId, new CloudTask("ForceJobPrep", "cmd /c echo TestOMJobReleaseSchedulingError"));
// wait for the task to complete
TaskStateMonitor tsm = client.Utilities.CreateTaskStateMonitor();
tsm.WaitAll(
client.JobOperations.ListTasks(jobId),
TaskState.Completed,
TimeSpan.FromMinutes(10),
additionalBehaviors:
new[]
{
// spam/logging interceptor
new Protocol.RequestInterceptor((x) =>
{
this.testOutputHelper.WriteLine("Issuing request type: " + x.GetType().ToString());
// print out the compute node states... we are actually waiting on the compute nodes
List <ComputeNode> allComputeNodes = client.PoolOperations.ListComputeNodes(this.poolFixture.PoolId).ToList();
this.testOutputHelper.WriteLine(" #compute nodes: " + allComputeNodes.Count);
allComputeNodes.ForEach((icn) =>
{
this.testOutputHelper.WriteLine(" computeNode.id: " + icn.Id + ", state: " + icn.State);
});
this.testOutputHelper.WriteLine("");
})
}
);
// ok terminate job to trigger job release
client.JobOperations.TerminateJob(jobId, "BUG: Server will throw 500 if I don't provide reason");
// the victim compute node. pool should have size 1.
List <ComputeNode> computeNodes = client.PoolOperations.ListComputeNodes(this.poolFixture.PoolId).ToList();
Assert.Single(computeNodes);
// now we have a job that should be trying to run the JP
// poll for the JP to have been run, and it must have a scheduling error
bool releaseNotCompleted = true;
// gotta poll to find out when the jp has been run
while (releaseNotCompleted)
{
List <JobPreparationAndReleaseTaskExecutionInformation> jrStatusList =
client.JobOperations.ListJobPreparationAndReleaseTaskStatus(jobId).ToList();
JobPreparationAndReleaseTaskExecutionInformation prepAndReleaseStatus = jrStatusList.FirstOrDefault();
if (prepAndReleaseStatus != null && null != prepAndReleaseStatus.JobReleaseTaskExecutionInformation)
{
if (JobReleaseTaskState.Completed == prepAndReleaseStatus.JobReleaseTaskExecutionInformation.State)
{
releaseNotCompleted = false; // we see a JP has been run
// now assert the failure info
Assert.NotNull(prepAndReleaseStatus);
Assert.NotNull(prepAndReleaseStatus.JobReleaseTaskExecutionInformation.FailureInformation);
Assert.Equal(TaskExecutionResult.Failure, prepAndReleaseStatus.JobReleaseTaskExecutionInformation.Result);
// spew the failure info
this.OutputFailureInfo(prepAndReleaseStatus.JobReleaseTaskExecutionInformation.FailureInformation);
}
}
Thread.Sleep(2000);
this.testOutputHelper.WriteLine("Job Release tasks still running (waiting for blob dl to timeout).");
}
}
finally
{
client.JobOperations.DeleteJob(jobId);
}
}
};
SynchronizationContextHelper.RunTest(test, LongTestTimeout);
}
private static async Task MainAsync()
{
string poolId = "JobPrepReleaseSamplePool";
string jobId = "JobPrepReleaseSampleJob";
var settings = Config.LoadAccountSettings();
// Location of the file that the job tasks will work with, a text file in the
// node's "shared" directory.
string taskOutputFile = "$AZ_BATCH_NODE_SHARED_DIR/job_prep_and_release.txt";
// The job prep task will write the node ID to the text file in the shared directory
string jobPrepCmdLine = $@"/bin/bash -c ""echo $AZ_BATCH_NODE_ID tasks: > {taskOutputFile}""";
// Each task then echoes its ID to the same text file
string taskCmdLine = $@"/bin/bash -c ""echo $AZ_BATCH_TASK_ID >> {taskOutputFile}""";
// The job release task will then delete the text file from the shared directory
string jobReleaseCmdLine = $@"/bin/bash -c ""rm {taskOutputFile}""";
BatchSharedKeyCredentials cred = new BatchSharedKeyCredentials(settings.BatchServiceUrl, settings.BatchAccountName, settings.BatchAccountKey);
using (BatchClient batchClient = BatchClient.Open(cred))
{
var pool = await BatchUtils.CreatePoolIfNotExistAsync(batchClient, poolId);
var prepTask = new JobPreparationTask {
CommandLine = jobPrepCmdLine
};
var releaseTask = new JobReleaseTask {
CommandLine = jobReleaseCmdLine
};
var job = await BatchUtils.CreateJobIfNotExistAsync(batchClient, pool.Id, jobId, prepTask : prepTask, releaseTask : releaseTask);
// Create the tasks that the job will execute
List <CloudTask> tasks = new List <CloudTask>();
for (int i = 1; i <= 8; i++)
{
string taskId = "task" + i.ToString().PadLeft(3, '0');
string taskCommandLine = taskCmdLine;
CloudTask task = new CloudTask(taskId, taskCommandLine);
tasks.Add(task);
}
// Add the tasks in one API call as opposed to a separate AddTask call for each. Bulk task
// submission helps to ensure efficient underlying API calls to the Batch service.
Console.WriteLine("Submitting tasks and awaiting completion...");
await batchClient.JobOperations.AddTaskAsync(job.Id, tasks);
// Wait for the tasks to complete before proceeding. The long timeout here is to allow time
// for the nodes within the pool to be created and started if the pool had not yet been created.
await batchClient.Utilities.CreateTaskStateMonitor().WhenAll(
job.ListTasks(),
TaskState.Completed,
TimeSpan.FromMinutes(30));
Console.WriteLine("All tasks completed.");
Console.WriteLine();
// Print the contents of the shared text file modified by the job preparation and other tasks.
ODATADetailLevel nodeDetail = new ODATADetailLevel(selectClause: "id, state");
IPagedEnumerable <ComputeNode> nodes = batchClient.PoolOperations.ListComputeNodes(poolId, nodeDetail);
await nodes.ForEachAsync(async (node) =>
{
// Check to ensure that the node is Idle before attempting to pull the text file.
// If the pool was just created, there is a chance that another node completed all
// of the tasks prior to the other node(s) completing their startup procedure.
if (node.State == ComputeNodeState.Idle)
{
var files = await node.ListNodeFiles().ToListAsync();
NodeFile sharedTextFile = await node.GetNodeFileAsync("shared/job_prep_and_release.txt");
Console.WriteLine("Contents of {0} on {1}:", sharedTextFile.Path, node.Id);
Console.WriteLine("-------------------------------------------");
Console.WriteLine(await sharedTextFile.ReadAsStringAsync());
}
});
// Terminate the job to mark it as Completed; this will initiate the Job Release Task on any node
// that executed job tasks. Note that the Job Release Task is also executed when a job is deleted,
// thus you need not call Terminate if you typically delete your jobs upon task completion.
await batchClient.JobOperations.TerminateJobAsync(job.Id);
// Wait for the job to reach state "Completed." Note that this wait is not typically necessary in
// production code, but is done here to enable the checking of the release tasks exit code below.
await BatchUtils.WaitForJobToReachStateAsync(batchClient, job.Id, JobState.Completed, TimeSpan.FromMinutes(2));
// Print the exit codes of the prep and release tasks by obtaining their execution info
List <JobPreparationAndReleaseTaskExecutionInformation> prepReleaseInfo = await batchClient.JobOperations.ListJobPreparationAndReleaseTaskStatus(job.Id).ToListAsync();
foreach (JobPreparationAndReleaseTaskExecutionInformation info in prepReleaseInfo)
{
Console.WriteLine();
Console.WriteLine("{0}: ", info.ComputeNodeId);
// If no tasks were scheduled to run on the node, the JobPreparationTaskExecutionInformation will be null
if (info.JobPreparationTaskExecutionInformation != null)
{
Console.WriteLine(" Prep task exit code: {0}", info.JobPreparationTaskExecutionInformation.ExitCode);
}
// If no tasks were scheduled to run on the node, the JobReleaseTaskExecutionInformation will be null
if (info.JobReleaseTaskExecutionInformation != null)
{
Console.WriteLine(" Release task exit code: {0}", info.JobReleaseTaskExecutionInformation.ExitCode);
}
}
// Clean up the resources we've created in the Batch account
Console.WriteLine();
Console.WriteLine("Delete job? [yes] no");
string response = Console.ReadLine().ToLower();
if (response != "n" && response != "no")
{
// Note that deleting the job will execute the job release task if the job was not previously terminated
await batchClient.JobOperations.DeleteJobAsync(job.Id);
}
Console.WriteLine("Delete pool? [yes] no");
response = Console.ReadLine();
if (response != "n" && response != "no")
{
await batchClient.PoolOperations.DeletePoolAsync(poolId);
}
}
}
