private static void AssertJobScheduleCorrectness(
JobScheduleOperations jobScheduleOperations,
CloudJobSchedule boundJobSchedule,
string expectedPoolId,
int expectedJobPriority,
string expectedJobManagerId,
string expectedJobManagerCommandLine,
TimeSpan?expectedRecurrenceInterval,
IEnumerable <MetadataItem> expectedMetadata)
{
boundJobSchedule.Refresh();
Assert.Equal(expectedPoolId, boundJobSchedule.JobSpecification.PoolInformation.PoolId);
Assert.Equal(expectedJobPriority, boundJobSchedule.JobSpecification.Priority);
Assert.Equal(expectedJobManagerId, boundJobSchedule.JobSpecification.JobManagerTask.Id);
Assert.Equal(expectedJobManagerCommandLine, boundJobSchedule.JobSpecification.JobManagerTask.CommandLine);
if (expectedRecurrenceInterval.HasValue)
{
Assert.Equal(expectedRecurrenceInterval, boundJobSchedule.Schedule.RecurrenceInterval);
}
else
{
Assert.Null(boundJobSchedule.Schedule);
}
Assert.Equal(expectedMetadata.Count(), boundJobSchedule.Metadata.Count());
foreach (MetadataItem metadataItem in expectedMetadata)
{
Assert.Equal(1, boundJobSchedule.Metadata.Count(item => item.Name == metadataItem.Name && item.Value == metadataItem.Value));
}
}
/// <summary>
/// Deletes the specified job schedule.
/// </summary>
/// <param name="context">The account to use.</param>
/// <param name="jobScheduleId">The id of the job schedule to delete.</param>
/// <param name="additionBehaviors">Additional client behaviors to perform.</param>
public void DeleteJobSchedule(BatchAccountContext context, string jobScheduleId, IEnumerable <BatchClientBehavior> additionBehaviors = null)
{
if (string.IsNullOrWhiteSpace(jobScheduleId))
{
throw new ArgumentNullException("jobScheduleId");
}
JobScheduleOperations jobScheduleOperations = context.BatchOMClient.JobScheduleOperations;
jobScheduleOperations.DeleteJobSchedule(jobScheduleId, additionBehaviors);
}
/// <summary>
/// Lists the jobs matching the specified filter options.
/// </summary>
/// <param name="options">The options to use when querying for jobs.</param>
/// <returns>The jobs matching the specified filter options.</returns>
public IEnumerable <PSCloudJob> ListJobs(ListJobOptions options)
{
if (options == null)
{
throw new ArgumentNullException("options");
}
// Get the single job matching the specified id
if (!string.IsNullOrEmpty(options.JobId))
{
WriteVerbose(string.Format(Resources.GetJobById, options.JobId));
JobOperations jobOperations = options.Context.BatchOMClient.JobOperations;
ODATADetailLevel getDetailLevel = new ODATADetailLevel(selectClause: options.Select, expandClause: options.Expand);
CloudJob job = jobOperations.GetJob(options.JobId, detailLevel: getDetailLevel, additionalBehaviors: options.AdditionalBehaviors);
PSCloudJob psJob = new PSCloudJob(job);
return(new PSCloudJob[] { psJob });
}
// List jobs using the specified filter
else
{
string jobScheduleId = options.JobSchedule == null ? options.JobScheduleId : options.JobSchedule.Id;
bool filterByJobSchedule = !string.IsNullOrEmpty(jobScheduleId);
ODATADetailLevel listDetailLevel = new ODATADetailLevel(selectClause: options.Select, expandClause: options.Expand);
string verboseLogString = null;
if (!string.IsNullOrEmpty(options.Filter))
{
verboseLogString = filterByJobSchedule ? Resources.GetJobByOData : string.Format(Resources.GetJobByODataAndJobSChedule, jobScheduleId);
listDetailLevel.FilterClause = options.Filter;
}
else
{
verboseLogString = filterByJobSchedule ? Resources.GetJobNoFilter : string.Format(Resources.GetJobByJobScheduleNoFilter, jobScheduleId);
}
WriteVerbose(verboseLogString);
IPagedEnumerable <CloudJob> jobs = null;
if (filterByJobSchedule)
{
JobScheduleOperations jobScheduleOperations = options.Context.BatchOMClient.JobScheduleOperations;
jobs = jobScheduleOperations.ListJobs(jobScheduleId, listDetailLevel, options.AdditionalBehaviors);
}
else
{
JobOperations jobOperations = options.Context.BatchOMClient.JobOperations;
jobs = jobOperations.ListJobs(listDetailLevel, options.AdditionalBehaviors);
}
Func <CloudJob, PSCloudJob> mappingFunction = j => { return(new PSCloudJob(j)); };
return(PSPagedEnumerable <PSCloudJob, CloudJob> .CreateWithMaxCount(
jobs, mappingFunction, options.MaxCount, () => WriteVerbose(string.Format(Resources.MaxCount, options.MaxCount))));
}
}
/// <summary>
/// Terminates the specified job schedule.
/// </summary>
/// <param name="context">The account to use.</param>
/// <param name="jobScheduleId">The id of the job schedule to terminate.</param>
/// <param name="additionBehaviors">Additional client behaviors to perform.</param>
public void TerminateJobSchedule(BatchAccountContext context, string jobScheduleId, IEnumerable <BatchClientBehavior> additionBehaviors = null)
{
if (string.IsNullOrWhiteSpace(jobScheduleId))
{
throw new ArgumentNullException("jobScheduleId");
}
WriteVerbose(string.Format(Resources.TerminateJobSchedule, jobScheduleId));
JobScheduleOperations jobScheduleOperations = context.BatchOMClient.JobScheduleOperations;
jobScheduleOperations.TerminateJobSchedule(jobScheduleId, additionBehaviors);
}
/// <summary>
/// Initializes client properties.
/// </summary>
private void Initialize()
{
Application = new ApplicationOperations(this);
Pool = new PoolOperations(this);
Account = new AccountOperations(this);
Job = new JobOperations(this);
Certificate = new CertificateOperations(this);
File = new FileOperations(this);
JobSchedule = new JobScheduleOperations(this);
Task = new TaskOperations(this);
ComputeNode = new ComputeNodeOperations(this);
ComputeNodeExtension = new ComputeNodeExtensionOperations(this);
BaseUri = "{batchUrl}";
ApiVersion = "2021-06-01.14.0";
AcceptLanguage = "en-US";
LongRunningOperationRetryTimeout = 30;
GenerateClientRequestId = true;
SerializationSettings = new JsonSerializerSettings
{
Formatting = Newtonsoft.Json.Formatting.Indented,
DateFormatHandling = Newtonsoft.Json.DateFormatHandling.IsoDateFormat,
DateTimeZoneHandling = Newtonsoft.Json.DateTimeZoneHandling.Utc,
NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore,
ReferenceLoopHandling = Newtonsoft.Json.ReferenceLoopHandling.Serialize,
ContractResolver = new ReadOnlyJsonContractResolver(),
Converters = new List <JsonConverter>
{
new Iso8601TimeSpanConverter()
}
};
DeserializationSettings = new JsonSerializerSettings
{
DateFormatHandling = Newtonsoft.Json.DateFormatHandling.IsoDateFormat,
DateTimeZoneHandling = Newtonsoft.Json.DateTimeZoneHandling.Utc,
NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore,
ReferenceLoopHandling = Newtonsoft.Json.ReferenceLoopHandling.Serialize,
ContractResolver = new ReadOnlyJsonContractResolver(),
Converters = new List <JsonConverter>
{
new Iso8601TimeSpanConverter()
}
};
CustomInitialize();
DeserializationSettings.Converters.Add(new CloudErrorJsonConverter());
}
/// <summary>
/// Waits for a job to be created -- This should be removed when we have a Utilities helper which does this
/// </summary>
public static CloudJobSchedule WaitForJobOnJobSchedule(JobScheduleOperations jobScheduleOperations, string jobScheduleId, string expectedJobId = null, TimeSpan?timeout = null)
{
//Wait for the job
TimeSpan jobCreationTimeout = timeout ?? TimeSpan.FromSeconds(30);
CloudJobSchedule refreshableJobSchedule = jobScheduleOperations.GetJobSchedule(jobScheduleId);
DateTime jobCreationWaitStartTime = DateTime.UtcNow;
while (refreshableJobSchedule.ExecutionInformation == null ||
refreshableJobSchedule.ExecutionInformation.RecentJob == null ||
(!string.IsNullOrEmpty(expectedJobId) && refreshableJobSchedule.ExecutionInformation.RecentJob.Id != expectedJobId))
{
Thread.Sleep(TimeSpan.FromSeconds(10));
refreshableJobSchedule.Refresh();
if (DateTime.UtcNow > jobCreationWaitStartTime.Add(jobCreationTimeout))
{
throw new Exception("Timed out waiting for job");
}
}
return(refreshableJobSchedule);
}
/// <summary>
/// Lists the job schedules matching the specified filter options.
/// </summary>
/// <param name="options">The options to use when querying for job schedules.</param>
/// <returns>The workitems matching the specified filter options.</returns>
public IEnumerable <PSCloudJobSchedule> ListJobSchedules(ListJobScheduleOptions options)
{
if (options == null)
{
throw new ArgumentNullException("options");
}
// Get the single job schedule matching the specified id
if (!string.IsNullOrWhiteSpace(options.JobScheduleId))
{
WriteVerbose(string.Format(Resources.GetJobScheduleById, options.JobScheduleId));
JobScheduleOperations jobScheduleOperations = options.Context.BatchOMClient.JobScheduleOperations;
ODATADetailLevel getDetailLevel = new ODATADetailLevel(selectClause: options.Select, expandClause: options.Expand);
CloudJobSchedule jobSchedule = jobScheduleOperations.GetJobSchedule(options.JobScheduleId, detailLevel: getDetailLevel, additionalBehaviors: options.AdditionalBehaviors);
PSCloudJobSchedule psJobSchedule = new PSCloudJobSchedule(jobSchedule);
return(new PSCloudJobSchedule[] { psJobSchedule });
}
// List job schedules using the specified filter
else
{
string verboseLogString = null;
ODATADetailLevel listDetailLevel = new ODATADetailLevel(selectClause: options.Select, expandClause: options.Expand);
if (!string.IsNullOrEmpty(options.Filter))
{
verboseLogString = Resources.GetJobScheduleByOData;
listDetailLevel.FilterClause = options.Filter;
}
else
{
verboseLogString = Resources.GetJobScheduleNoFilter;
}
WriteVerbose(verboseLogString);
JobScheduleOperations jobScheduleOperations = options.Context.BatchOMClient.JobScheduleOperations;
IPagedEnumerable <CloudJobSchedule> workItems = jobScheduleOperations.ListJobSchedules(listDetailLevel, options.AdditionalBehaviors);
Func <CloudJobSchedule, PSCloudJobSchedule> mappingFunction = j => { return(new PSCloudJobSchedule(j)); };
return(PSPagedEnumerable <PSCloudJobSchedule, CloudJobSchedule> .CreateWithMaxCount(
workItems, mappingFunction, options.MaxCount, () => WriteVerbose(string.Format(Resources.MaxCount, options.MaxCount))));
}
}
/// <summary>
/// Creates a new job schedule.
/// </summary>
/// <param name="parameters">The parameters to use when creating the job schedule.</param>
public void CreateJobSchedule(NewJobScheduleParameters parameters)
{
if (parameters == null)
{
throw new ArgumentNullException("parameters");
}
JobScheduleOperations jobScheduleOperations = parameters.Context.BatchOMClient.JobScheduleOperations;
CloudJobSchedule jobSchedule = jobScheduleOperations.CreateJobSchedule();
jobSchedule.Id = parameters.JobScheduleId;
jobSchedule.DisplayName = parameters.DisplayName;
if (parameters.Schedule != null)
{
jobSchedule.Schedule = parameters.Schedule.omObject;
}
if (parameters.JobSpecification != null)
{
Utils.Utils.JobSpecificationSyncCollections(parameters.JobSpecification);
jobSchedule.JobSpecification = parameters.JobSpecification.omObject;
}
if (parameters.Metadata != null)
{
jobSchedule.Metadata = new List <MetadataItem>();
foreach (DictionaryEntry d in parameters.Metadata)
{
MetadataItem metadata = new MetadataItem(d.Key.ToString(), d.Value.ToString());
jobSchedule.Metadata.Add(metadata);
}
}
WriteVerbose(string.Format(Resources.CreatingJobSchedule, parameters.JobScheduleId));
jobSchedule.Commit(parameters.AdditionalBehaviors);
}
/// <summary>
/// calls the two new get-status REST APIs and asserts their values
///
/// 1: add a single quick task (quick because we don't need it to run very long)
/// 2: this forces a victim compute node to run the JobPrep
/// 3: poll for this compute node, ignore others (sharedPool.size probably > 1)
/// 4: check status of JobPrep
/// 4a: assert as many values as makes sense... this is not a retry test
/// 5: JobPrep succeeds, task runs
/// 6: poll for JobRelease.. it is long running
/// 7: assert as many values as makes sense.
/// </summary>
/// <param name="batchCli"></param>
private void TestGetPrepReleaseStatusCalls(BatchClient batchCli, CloudJobSchedule boundJobSchedule, string sharedPool, IEnumerable <ResourceFile> correctResFiles)
{
// need this often enough lets just pull it out
string jobId = boundJobSchedule.ExecutionInformation.RecentJob.Id;
PoolOperations poolOps = batchCli.PoolOperations;
JobScheduleOperations jobScheduleOperations = batchCli.JobScheduleOperations;
{
DateTime beforeJobPrepRuns = DateTime.UtcNow; // used to test start time
// need a task to force JobPrep
CloudTask sillyTask = new CloudTask("forceJobPrep", "cmd /c hostname");
// add the task
batchCli.JobOperations.AddTask(jobId, sillyTask);
bool keepLooking = true;
while (keepLooking)
{
this.testOutputHelper.WriteLine("Waiting for task to be scheduled.");
foreach (CloudTask curTask in batchCli.JobOperations.GetJob(jobId).ListTasks())
{
if (curTask.State != TaskState.Active)
{
keepLooking = false;
break;
}
}
Thread.Sleep(1000);
}
List <JobPreparationAndReleaseTaskExecutionInformation> jobPrepStatusList = new List <JobPreparationAndReleaseTaskExecutionInformation>();
while (jobPrepStatusList.Count == 0)
{
jobPrepStatusList = batchCli.JobOperations.ListJobPreparationAndReleaseTaskStatus(jobId).ToList();
}
JobPreparationAndReleaseTaskExecutionInformation jptei = jobPrepStatusList.First();
ComputeNode victimComputeNodeRunningPrepAndRelease = poolOps.GetComputeNode(sharedPool, jptei.ComputeNodeId);
// job prep tests
{
Assert.NotNull(jptei);
Assert.Equal(0, jptei.JobPreparationTaskExecutionInformation.RetryCount);
Assert.True(beforeJobPrepRuns < jptei.JobPreparationTaskExecutionInformation.StartTime + TimeSpan.FromSeconds(10)); // test that the start time is rational -- 10s of wiggle room
Assert.Null(jptei.JobPreparationTaskExecutionInformation.FailureInformation);
this.testOutputHelper.WriteLine("");
this.testOutputHelper.WriteLine("listing files for compute node: " + victimComputeNodeRunningPrepAndRelease.Id);
// fiter the list so reduce noise
List <NodeFile> filteredListJobPrep = new List <NodeFile>();
foreach (NodeFile curTF in victimComputeNodeRunningPrepAndRelease.ListNodeFiles(recursive: true))
{
// filter on the jsId since we only run one job per job in this test.
if (curTF.Path.IndexOf(boundJobSchedule.Id, StringComparison.InvariantCultureIgnoreCase) >= 0)
{
this.testOutputHelper.WriteLine(" name:" + curTF.Path + ", size: " + ((curTF.IsDirectory.HasValue && curTF.IsDirectory.Value) ? "<dir>" : curTF.Properties.ContentLength.ToString()));
filteredListJobPrep.Add(curTF);
}
}
// confirm resource files made it
foreach (ResourceFile curCorrectRF in correctResFiles)
{
bool found = false;
foreach (NodeFile curTF in filteredListJobPrep)
{
// look for the resfile filepath in the taskfile name
found |= curTF.Path.IndexOf(curCorrectRF.FilePath, StringComparison.InvariantCultureIgnoreCase) >= 0;
}
Assert.True(found, "Looking for resourcefile: " + curCorrectRF.FilePath);
}
// poll for completion
while (JobPreparationTaskState.Completed != jptei.JobPreparationTaskExecutionInformation.State)
{
this.testOutputHelper.WriteLine("waiting for jopPrep to complete");
Thread.Sleep(2000);
// refresh the state info
ODATADetailLevel detailLevel = new ODATADetailLevel()
{
FilterClause = string.Format("nodeId eq '{0}'", victimComputeNodeRunningPrepAndRelease.Id)
};
jobPrepStatusList = batchCli.JobOperations.ListJobPreparationAndReleaseTaskStatus(jobId, detailLevel: detailLevel).ToList();
jptei = jobPrepStatusList.First();
}
// need success
Assert.Equal(0, jptei.JobPreparationTaskExecutionInformation.ExitCode);
// check stdout to confirm prep ran
//Why do I have to use the hardcoded string job-1 here...?
string stdOutFileSpec = Path.Combine("workitems", boundJobSchedule.Id, "job-1", boundJobSchedule.JobSpecification.JobPreparationTask.Id, Constants.StandardOutFileName);
string stdOut = victimComputeNodeRunningPrepAndRelease.GetNodeFile(stdOutFileSpec).ReadAsString();
string stdErrFileSpec = Path.Combine("workitems", boundJobSchedule.Id, "job-1", boundJobSchedule.JobSpecification.JobPreparationTask.Id, Constants.StandardErrorFileName);
string stdErr = string.Empty;
try
{
stdErr = victimComputeNodeRunningPrepAndRelease.GetNodeFile(stdErrFileSpec).ReadAsString();
}
catch (Exception)
{
//Swallow any exceptions here since stderr may not exist
}
this.testOutputHelper.WriteLine(stdOut);
this.testOutputHelper.WriteLine(stdErr);
Assert.True(!string.IsNullOrWhiteSpace(stdOut));
Assert.Contains("jobpreparation", stdOut.ToLower());
}
// jobPrep tests completed. let JobPrep complete and task run and wait for JobRelease
TaskStateMonitor tsm = batchCli.Utilities.CreateTaskStateMonitor();
// spam/logging interceptor
Protocol.RequestInterceptor consoleSpammer =
new Protocol.RequestInterceptor((x) =>
{
this.testOutputHelper.WriteLine("TestGetPrepReleaseStatusCalls: waiting for JobPrep and task to complete");
ODATADetailLevel detailLevel = new ODATADetailLevel()
{
FilterClause = string.Format("nodeId eq '{0}'", victimComputeNodeRunningPrepAndRelease.Id)
};
jobPrepStatusList = batchCli.JobOperations.ListJobPreparationAndReleaseTaskStatus(jobId, detailLevel: detailLevel).ToList();
JobPreparationAndReleaseTaskExecutionInformation jpteiInterceptor =
jobPrepStatusList.First();
this.testOutputHelper.WriteLine(" JobPrep.State: " + jpteiInterceptor.JobPreparationTaskExecutionInformation.State);
this.testOutputHelper.WriteLine("");
});
// waiting for the task to complete means so JobRelease is run.
tsm.WaitAll(
batchCli.JobOperations.GetJob(jobId).ListTasks(additionalBehaviors: new[] { consoleSpammer }),
TaskState.Completed,
TimeSpan.FromSeconds(120),
additionalBehaviors: new[] { consoleSpammer });
// trigger JobRelease
batchCli.JobOperations.TerminateJob(jobId, terminateReason: "die! I want JobRelease to run!");
// now that the task has competed, we are racing with the JobRelease... but it is sleeping so we can can catch it
while (true)
{
ODATADetailLevel detailLevel = new ODATADetailLevel()
{
FilterClause = string.Format("nodeId eq '{0}'", victimComputeNodeRunningPrepAndRelease.Id)
};
jobPrepStatusList = batchCli.JobOperations.ListJobPreparationAndReleaseTaskStatus(jobId, detailLevel: detailLevel).ToList();
JobPreparationAndReleaseTaskExecutionInformation jrtei = jobPrepStatusList.FirstOrDefault();
if ((jrtei == null) || (null == jrtei.JobReleaseTaskExecutionInformation))
{
Thread.Sleep(2000);
}
else
{
Assert.NotNull(jrtei);
if (jrtei.JobReleaseTaskExecutionInformation.State != JobReleaseTaskState.Completed)
{
this.testOutputHelper.WriteLine("JobReleaseTask state is: " + jrtei.JobReleaseTaskExecutionInformation.State);
Thread.Sleep(5000);
}
else
{
this.testOutputHelper.WriteLine("JobRelease commpleted!");
// we are done
break;
}
}
}
}
}
public void Bug1910530_ConcurrentChangeTrackedListThreadsafeTest()
{
const string testName = "Bug1910530_ConcurrentChangeTrackedListThreadsafeTest";
using (BatchClient batchCli = ClientUnitTestCommon.CreateDummyClient())
{
JobScheduleOperations jobScheduleOperations = batchCli.JobScheduleOperations;
string jobScheduleId = Microsoft.Azure.Batch.Constants.DefaultConveniencePrefix + "-" + testName;
//
//Unbound job schedule properties
//
this.testOutputHelper.WriteLine("Creating job schedule {0}", jobScheduleId);
CloudJobSchedule unboundJobSchedule = jobScheduleOperations.CreateJobSchedule(jobScheduleId, null, null);
//Create a new threadsafe collection
unboundJobSchedule.Metadata = new List <MetadataItem>();
//Now it should be magically threadsafe
Action addAction = () =>
{
this.testOutputHelper.WriteLine("Adding an item");
unboundJobSchedule.Metadata.Add(new MetadataItem("test", "test"));
};
Action removeAction = () =>
{
this.testOutputHelper.WriteLine("Removing an item");
try
{
unboundJobSchedule.Metadata.RemoveAt(0);
}
catch (ArgumentOutOfRangeException)
{
}
};
Random rand = new Random();
object randLock = new object();
Parallel.For(0, 100, new ParallelOptions()
{
MaxDegreeOfParallelism = 10
}, (i) =>
{
int randomInt;
lock (randLock)
{
randomInt = rand.Next(0, 2);
}
if (randomInt == 0)
{
addAction();
}
else
{
removeAction();
}
});
}
}
