private CloudJob CreateBoundJob(BatchClient client, string jobId, Action <CloudJob> jobSetup = null)
{
CloudJob cloudJob = client.JobOperations.CreateJob(jobId, new PoolInformation());
cloudJob.PoolInformation.PoolId = poolFixture.PoolId;
jobSetup?.Invoke(cloudJob);
cloudJob.Commit();
return(client.JobOperations.GetJob(jobId));
}
/// <summary>
/// Creates a job in Batch service with the specified id and associated with the specified pool.
/// </summary>
/// <param name="batchClient"></param>
/// <param name="jobId"></param>
/// <param name="poolId"></param>
private static async Task CreateJobAsync(BatchClient batchClient, string jobId, string poolId)
{
// Create the job to which the multi-instance task will be added.
Console.WriteLine($"Creating job [{jobId}]...");
CloudJob unboundJob = batchClient.JobOperations.CreateJob(jobId, new PoolInformation()
{
PoolId = poolId
});
await unboundJob.CommitAsync();
}
public void SetUpdateJobConditionalHeader()
{
Action test = () =>
{
using (BatchClient batchCli = TestUtilities.OpenBatchClientFromEnvironmentAsync().Result)
{
string jobId = "JobConditionalHeaders-" + TestUtilities.GetMyName();
try
{
PoolInformation poolInfo = new PoolInformation()
{
PoolId = "Fake"
};
CloudJob unboundJob = batchCli.JobOperations.CreateJob(jobId, poolInfo);
unboundJob.Commit();
CloudJob boundJob = batchCli.JobOperations.GetJob(jobId);
string capturedEtag1 = boundJob.ETag;
this.testOutputHelper.WriteLine("Etag is: {0}", capturedEtag1);
Assert.NotNull(capturedEtag1);
boundJob.Constraints = new JobConstraints(TimeSpan.FromMinutes(60), 0);
BatchClientBehavior updateInterceptor = new Protocol.RequestInterceptor(
(req) =>
{
var typedParams = req.Options as Protocol.Models.JobUpdateOptions;
if (typedParams != null)
{
typedParams.IfMatch = capturedEtag1;
}
});
//Update bound job with if-match header, it should succeed
boundJob.Commit(additionalBehaviors: new[] { updateInterceptor });
boundJob = batchCli.JobOperations.GetJob(jobId);
boundJob.Constraints = new JobConstraints(TimeSpan.FromMinutes(30), 1);
//Update bound job with if-match header, it should fail
Exception e = TestUtilities.AssertThrows <BatchException>(() => boundJob.Commit(additionalBehaviors: new[] { updateInterceptor }));
TestUtilities.AssertIsBatchExceptionAndHasCorrectAzureErrorCode(e, BatchErrorCodeStrings.ConditionNotMet, this.testOutputHelper);
}
finally
{
TestUtilities.DeleteJobIfExistsAsync(batchCli, jobId).Wait();
}
}
};
SynchronizationContextHelper.RunTest(test, LongTestTimeout);
}
/// <summary>
/// Gets the name of the Azure blob storage container for the outputs of a <see cref="CloudJob"/>.
/// </summary>
/// <param name="job">The job for which to get the container name.</param>
/// <returns>The name of the container in which to save the outputs of this job.</returns>
public static string OutputStorageContainerName(this CloudJob job)
{
if (job == null)
{
throw new ArgumentNullException(nameof(job));
}
var jobOutputContainerName = ContainerNameUtils.GetSafeContainerName(job.Id);
return(jobOutputContainerName);
}
/// <summary>
/// Creates a job in the specified pool.
/// </summary>
/// <param name="batchClient">A BatchClient object.</param>
/// <param name="jobId">ID of the job to create.</param>
/// <param name="poolId">ID of the CloudPool object in which to create the job.</param>
private static async Task CreateJobAsync(BatchClient batchClient, string jobId, string poolId)
{
Console.WriteLine("Creating job [{0}]...", jobId);
CloudJob job = batchClient.JobOperations.CreateJob();
job.Id = jobId;
job.PoolInformation = new PoolInformation {
PoolId = poolId
};
// Commit the Job
try
{
await job.CommitAsync();
}
catch (BatchException be)
{
// Catch specific error code JobExists as that is expected if the job already exists
if (be.RequestInformation.BatchError.Code == BatchErrorCodeStrings.JobExists)
{
Console.WriteLine("Job [{0}] already exists.", jobId);
Console.WriteLine("Deleting job [{0}]...", jobId);
await batchClient.JobOperations.DeleteJobAsync(JobId);
// try creating a job again
bool tryAgain = true;
while (tryAgain)
{
try
{
tryAgain = false;
var n = 10;
Thread.Sleep(n * 1000);
Console.WriteLine($"Creating job [{jobId}] after {n} sec.");
await job.CommitAsync();
}
catch (BatchException innerException)
{
if (innerException.RequestInformation.BatchError.Code == BatchErrorCodeStrings.JobBeingDeleted)
{
Console.WriteLine("Job is being deleted.. Try again..");
tryAgain = true;
}
}
}
}
else
{
// Any other exception is unexpected
Shared.Logger.Error($"CreateJobAsync(): BatchException: Code - {be.RequestInformation.BatchError.Code}");
throw;
}
}
}
public async Task CreateJobAsync(CreateJobOptions createJobOptions)
{
CloudJob unboundJob = this.Client.JobOperations.CreateJob();
unboundJob.Id = createJobOptions.JobId;
unboundJob.Priority = createJobOptions.Priority;
unboundJob.Constraints = new JobConstraints(createJobOptions.MaxWallClockTime, createJobOptions.MaxRetryCount);
PoolInformation poolInformation = new PoolInformation();
if (createJobOptions.AutoPoolOptions.UseAutoPool.HasValue && createJobOptions.AutoPoolOptions.UseAutoPool.Value)
{
AutoPoolSpecification autoPoolSpecification = new AutoPoolSpecification()
{
AutoPoolIdPrefix = createJobOptions.AutoPoolOptions.AutoPoolPrefix,
KeepAlive = createJobOptions.AutoPoolOptions.KeepAlive,
PoolLifetimeOption = (PoolLifetimeOption)Enum.Parse(typeof(PoolLifetimeOption), createJobOptions.AutoPoolOptions.LifeTimeOption),
PoolSpecification = new PoolSpecification()
{
CloudServiceConfiguration = new CloudServiceConfiguration(createJobOptions.AutoPoolOptions.OSFamily),
VirtualMachineSize = createJobOptions.AutoPoolOptions.VirutalMachineSize,
TargetDedicatedComputeNodes = createJobOptions.AutoPoolOptions.TargetDedicated,
TargetLowPriorityComputeNodes = createJobOptions.AutoPoolOptions.TargetLowPriority
}
};
poolInformation.AutoPoolSpecification = autoPoolSpecification;
}
else
{
poolInformation.PoolId = createJobOptions.PoolId;
}
unboundJob.PoolInformation = poolInformation;
if (createJobOptions.CreateJobManager.HasValue && createJobOptions.CreateJobManager.Value == true)
{
JobManagerTask jobManager = new JobManagerTask()
{
CommandLine = createJobOptions.JobManagerOptions.CommandLine,
KillJobOnCompletion = createJobOptions.JobManagerOptions.KillOnCompletion,
Id = createJobOptions.JobManagerOptions.JobManagerId
};
jobManager.Constraints = new TaskConstraints(
createJobOptions.JobManagerOptions.MaxTaskWallClockTime,
createJobOptions.JobManagerOptions.RetentionTime,
createJobOptions.JobManagerOptions.MaxTaskRetryCount);
unboundJob.JobManagerTask = jobManager;
}
await unboundJob.CommitAsync();
}
public async Task UnboundJobDirectRefreshFailsWithMissingPathVariables()
{
using (BatchClient batchClient = ClientUnitTestCommon.CreateDummyClient())
{
CloudJob job = batchClient.JobOperations.CreateJob(null, new PoolInformation());
ValidationException e = await Assert.ThrowsAsync <ValidationException>(async() => await job.RefreshAsync());
Assert.Contains("'jobId' cannot be null", e.Message);
}
}
/// <summary>
/// Creates a job in the specified pool.
/// </summary>
/// <param name="batchClient">A BatchClient object.</param>
/// <param name="jobId">ID of the job to create.</param>
/// <param name="poolId">ID of the CloudPool object in which to create the job.</param>
private static async Task CreateJobAsync(BatchClient batchClient, string jobId, string poolId)
{
Console.WriteLine("Creating job [{0}]...", jobId);
CloudJob job = batchClient.JobOperations.CreateJob();
job.Id = jobId;
job.PoolInformation = new PoolInformation { PoolId = poolId };
await job.CommitAsync();
}
public void CannotGetOutputStorageUrlWithNegativeExpiryTime()
{
using (var batchClient = BatchClient.Open(new FakeBatchServiceClient()))
{
CloudJob job = batchClient.JobOperations.CreateJob();
job.Id = "fakejob";
CloudStorageAccount storageAccount = new CloudStorageAccount(new StorageCredentials("fake", "ZmFrZQ=="), true);
var ex = Assert.Throws <ArgumentException>(() => job.GetOutputStorageContainerUrl(storageAccount, TimeSpan.FromMinutes(-5)));
Assert.Equal("expiryTime", ex.ParamName);
}
}
public void CannotCreateOutputStorageForNullStorageAccount()
{
using (var batchClient = BatchClient.Open(new FakeBatchServiceClient()))
{
CloudJob job = batchClient.JobOperations.CreateJob();
job.Id = "fakejob";
CloudStorageAccount storageAccount = null;
var ex = Assert.Throws <ArgumentNullException>(() => job.OutputStorage(storageAccount));
Assert.Equal("storageAccount", ex.ParamName);
}
}
public void JobOutputStorageContainerNameAgreesWithSafeContainerName(BatchId jobId)
{
using (var batchClient = BatchClient.Open(new FakeBatchServiceClient())) // FsCheck doesn't like async tests
{
CloudJob job = batchClient.JobOperations.CreateJob();
job.Id = jobId.ToString();
var actualContainerName = job.OutputStorageContainerName();
var expectedContainerName = ContainerNameUtils.GetSafeContainerName(job.Id);
Assert.Equal(expectedContainerName, actualContainerName); // We have other tests for validating the outputs of GetSafeContainerName - we do not need to reproduce those here
}
}
public async Task CannotGetOutputStorageUrlWithZeroExpiryTime()
{
using (var batchClient = await BatchClient.OpenAsync(new FakeBatchServiceClient()))
{
CloudJob job = batchClient.JobOperations.CreateJob();
job.Id = "fakejob";
CloudStorageAccount storageAccount = new CloudStorageAccount(new StorageCredentials("fake", new byte[] { 65, 66, 67, 68 }), true);
var ex = Assert.Throws <ArgumentException>(() => job.GetOutputStorageContainerUrl(storageAccount, TimeSpan.FromMinutes(0)));
Assert.Equal("expiryTime", ex.ParamName);
}
}
public void CannotGetOutputStorageUrlForNullStorageAccount()
{
using (var batchClient = BatchClient.Open(new FakeBatchServiceClient()))
{
CloudJob job = batchClient.JobOperations.CreateJob();
job.Id = "fakejob";
CloudStorageAccount storageAccount = null;
var ex = Assert.Throws <ArgumentNullException>(() => job.GetOutputStorageContainerUrl(storageAccount, TimeSpan.FromMinutes(5)));
Assert.Equal("storageAccount", ex.ParamName);
}
}
public async Task ApplicationPackageReferencesOnJobManagerTaskAreRoundtripped()
{
string jobId = Constants.DefaultConveniencePrefix + TestUtilities.GetMyName() + "-APROnJobManagerTaskAreRoundtripped";
const string applicationId = "blender";
const string applicationVersion = "beta";
Func <Task> test = async() =>
{
using (BatchClient client = await TestUtilities.OpenBatchClientFromEnvironmentAsync().ConfigureAwait(false))
{
var poolInfo = new PoolInformation
{
AutoPoolSpecification = new AutoPoolSpecification
{
PoolSpecification = new PoolSpecification
{
CloudServiceConfiguration = new CloudServiceConfiguration(PoolFixture.OSFamily),
VirtualMachineSize = PoolFixture.VMSize,
},
PoolLifetimeOption = PoolLifetimeOption.Job
}
};
try
{
CloudJob job = client.JobOperations.CreateJob(jobId, poolInfo);
job.JobManagerTask = new JobManagerTask
{
Id = jobId,
CommandLine = "cmd /c ping 127.0.0.1",
ApplicationPackageReferences = new[] { new ApplicationPackageReference
{
ApplicationId = applicationId,
Version = applicationVersion
} }
};
await job.CommitAsync().ConfigureAwait(false);
var boundJob = await client.JobOperations.GetJobAsync(jobId).ConfigureAwait(false);
Assert.Equal(applicationId, boundJob.JobManagerTask.ApplicationPackageReferences.Single().ApplicationId);
Assert.Equal(applicationVersion, boundJob.JobManagerTask.ApplicationPackageReferences.Single().Version);
}
finally
{
TestUtilities.DeleteJobIfExistsAsync(client, jobId).Wait();
}
}
};
await SynchronizationContextHelper.RunTestAsync(test, TestTimeout);
}
public void CloudJob_WhenReturnedFromServer_HasExpectedBoundProperties()
{
const string jobId = "id-123";
const string displayName = "DisplayNameFoo";
string applicationVersion = "beta";
string applicationId = "test";
MetadataItem metadataItem = new MetadataItem("foo", "bar");
const int priority = 0;
var onAllTasksComplete = OnAllTasksComplete.TerminateJob;
using (BatchClient client = ClientUnitTestCommon.CreateDummyClient())
{
DateTime creationTime = DateTime.Now;
Models.CloudJob protoJob = new Models.CloudJob(
jobId,
displayName,
jobManagerTask: new Models.JobManagerTask()
{
ApplicationPackageReferences = new [] { new Models.ApplicationPackageReference()
{
ApplicationId = applicationId, Version = applicationVersion
} }
},
metadata: new[] { new Models.MetadataItem {
Name = metadataItem.Name, Value = metadataItem.Value
} },
creationTime: creationTime,
priority: priority,
url: ClientUnitTestCommon.DummyBaseUrl,
onAllTasksComplete: Models.OnAllTasksComplete.NoAction);
CloudJob boundJob = client.JobOperations.GetJob(jobId, additionalBehaviors: InterceptorFactory.CreateGetJobRequestInterceptor(protoJob));
Assert.Equal(jobId, boundJob.Id); // reading is allowed from a job that is returned from the server.
Assert.Equal(creationTime, boundJob.CreationTime);
Assert.Equal(displayName, boundJob.DisplayName);
Assert.Equal(applicationId, boundJob.JobManagerTask.ApplicationPackageReferences.First().ApplicationId);
Assert.Equal(applicationVersion, boundJob.JobManagerTask.ApplicationPackageReferences.First().Version);
AssertPatchableJobPropertiesCanBeWritten(boundJob, priority, metadataItem, onAllTasksComplete);
// Can only read a url from a returned object.
Assert.Equal(ClientUnitTestCommon.DummyBaseUrl, boundJob.Url);
// Cannot change a bound displayName, Id and any property on a JobManagerTask.
Assert.Throws <InvalidOperationException>(() => boundJob.DisplayName = "cannot-change-display-name");
Assert.Throws <InvalidOperationException>(() => boundJob.Id = "cannot-change-id");
Assert.Throws <InvalidOperationException>(() => boundJob.JobManagerTask.ApplicationPackageReferences = new List <ApplicationPackageReference>());
Assert.Throws <InvalidOperationException>(() => boundJob.JobManagerTask = new JobManagerTask());
}
}
public void CanSpecifyTaskDependencyIds()
{
using (BatchClient batchCli = TestUtilities.OpenBatchClientFromEnvironmentAsync().Result)
{
string jobId = GenerateJobId();
try
{
CloudJob unboundJob = batchCli.JobOperations.CreateJob(jobId, new PoolInformation());
unboundJob.PoolInformation = new PoolInformation()
{
PoolId = this.poolFixture.PoolId
};
unboundJob.UsesTaskDependencies = true;
unboundJob.Commit();
var taskId = Guid.NewGuid().ToString();
IList <TaskIdRange> taskIdRanges = new List <TaskIdRange>
{
new TaskIdRange(1, 5),
new TaskIdRange(8, 8)
};
IList <string> taskIds = new List <string> {
"1"
};
var boundJob = batchCli.JobOperations.GetJob(jobId);
CloudTask taskToAdd = new CloudTask(taskId, "cmd.exe")
{
DependsOn = new TaskDependencies(taskIds, taskIdRanges),
};
boundJob.AddTask(taskToAdd);
CloudTask task = boundJob.GetTask(taskId);
var dependedOnRange = task.DependsOn.TaskIdRanges.First();
var dependedOnTaskId = task.DependsOn.TaskIds.First();
Assert.Equal(1, dependedOnRange.Start);
Assert.Equal(5, dependedOnRange.End);
Assert.Equal("1", dependedOnTaskId);
Assert.True(boundJob.UsesTaskDependencies);
}
finally
{
TestUtilities.DeleteJobIfExistsAsync(batchCli, jobId).Wait();
}
}
}
public void SubmitJobTest()
{
string jobId = Guid.NewGuid().ToString();
// Craft response
SetJobResponseMessage(jobId);
// Create Job
IWorkspace workspace = GetWorkspace();
JobDetails jobDetails = CreateJobDetails(jobId);
CloudJob job = new CloudJob(workspace, jobDetails);
CloudJob receivedJob;
// -ve cases
try
{
jobDetails.ContainerUri = null;
receivedJob = workspace.SubmitJob(job);
Assert.Fail();
}
catch (WorkspaceClientException)
{
jobDetails.ContainerUri = "https://uri";
}
try
{
jobDetails.ProviderId = null;
receivedJob = workspace.SubmitJob(job);
Assert.Fail();
}
catch (WorkspaceClientException)
{
jobDetails.ProviderId = TestConstants.ProviderId;
}
// Success
receivedJob = workspace.SubmitJob(job);
// Validate request
ValidateJobRequestMessage(jobId, HttpMethod.Put);
// Validate response
Assert.IsNotNull(receivedJob);
Assert.IsNotNull(receivedJob.Workspace);
Assert.AreEqual(
expected: jobId,
actual: receivedJob.Details.Id);
}
public async Task CanCreateJobAndAutoPoolWithAppPackageReferences()
{
var jobId = Guid.NewGuid().ToString();
const string applicationId = "blender";
Func <Task> test = async() =>
{
using (BatchClient client = await TestUtilities.OpenBatchClientFromEnvironmentAsync().ConfigureAwait(false))
{
var poolInfo = new PoolInformation
{
AutoPoolSpecification = new AutoPoolSpecification
{
PoolSpecification = new PoolSpecification
{
ApplicationPackageReferences = new[]
{
new ApplicationPackageReference
{
ApplicationId = applicationId,
Version = PoolFixture.VMSize,
}
},
CloudServiceConfiguration = new CloudServiceConfiguration(PoolFixture.OSFamily),
VirtualMachineSize = PoolFixture.VMSize,
},
PoolLifetimeOption = PoolLifetimeOption.Job
}
};
CloudJob response = null;
try
{
CloudJob job = client.JobOperations.CreateJob(jobId, poolInfo);
await job.CommitAsync().ConfigureAwait(false);
response = await client.JobOperations.GetJobAsync(jobId).ConfigureAwait(false);
Assert.Equal(response.PoolInformation.AutoPoolSpecification.PoolSpecification.ApplicationPackageReferences.First().ApplicationId, applicationId);
}
finally
{
if (response != null)
{
TestUtilities.DeleteJobIfExistsAsync(client, jobId).Wait();
}
}
}
};
await SynchronizationContextHelper.RunTestAsync(test, TestTimeout);
}
/// <summary>
/// Lists the jobs matching the specified filter options.
/// </summary>
/// <param name="options">The options to use when querying for jobs.</param>
/// <returns>The jobs matching the specified filter options.</returns>
public IEnumerable <PSCloudJob> ListJobs(ListJobOptions options)
{
if (options == null)
{
throw new ArgumentNullException("options");
}
// Get the single job matching the specified id
if (!string.IsNullOrEmpty(options.JobId))
{
WriteVerbose(string.Format(Resources.GetJobById, options.JobId));
JobOperations jobOperations = options.Context.BatchOMClient.JobOperations;
ODATADetailLevel getDetailLevel = new ODATADetailLevel(selectClause: options.Select, expandClause: options.Expand);
CloudJob job = jobOperations.GetJob(options.JobId, detailLevel: getDetailLevel, additionalBehaviors: options.AdditionalBehaviors);
PSCloudJob psJob = new PSCloudJob(job);
return(new PSCloudJob[] { psJob });
}
// List jobs using the specified filter
else
{
string jobScheduleId = options.JobSchedule == null ? options.JobScheduleId : options.JobSchedule.Id;
bool filterByJobSchedule = !string.IsNullOrEmpty(jobScheduleId);
ODATADetailLevel listDetailLevel = new ODATADetailLevel(selectClause: options.Select, expandClause: options.Expand);
string verboseLogString = null;
if (!string.IsNullOrEmpty(options.Filter))
{
verboseLogString = filterByJobSchedule ? Resources.GetJobByOData : string.Format(Resources.GetJobByODataAndJobSChedule, jobScheduleId);
listDetailLevel.FilterClause = options.Filter;
}
else
{
verboseLogString = filterByJobSchedule ? Resources.GetJobNoFilter : string.Format(Resources.GetJobByJobScheduleNoFilter, jobScheduleId);
}
WriteVerbose(verboseLogString);
IPagedEnumerable <CloudJob> jobs = null;
if (filterByJobSchedule)
{
JobScheduleOperations jobScheduleOperations = options.Context.BatchOMClient.JobScheduleOperations;
jobs = jobScheduleOperations.ListJobs(jobScheduleId, listDetailLevel, options.AdditionalBehaviors);
}
else
{
JobOperations jobOperations = options.Context.BatchOMClient.JobOperations;
jobs = jobOperations.ListJobs(listDetailLevel, options.AdditionalBehaviors);
}
Func <CloudJob, PSCloudJob> mappingFunction = j => { return(new PSCloudJob(j)); };
return(PSPagedEnumerable <PSCloudJob, CloudJob> .CreateWithMaxCount(
jobs, mappingFunction, options.MaxCount, () => WriteVerbose(string.Format(Resources.MaxCount, options.MaxCount))));
}
}
/// <summary>
/// Gets the <see cref="JobOutputStorage"/> for a specified <see cref="CloudJob"/>.
/// </summary>
/// <param name="job">The job for which to get output storage.</param>
/// <param name="storageAccount">The storage account linked to the Azure Batch account.</param>
/// <returns>A JobOutputStorage for the specified job.</returns>
public static JobOutputStorage OutputStorage(this CloudJob job, CloudStorageAccount storageAccount)
{
if (job == null)
{
throw new ArgumentNullException(nameof(job));
}
if (storageAccount == null)
{
throw new ArgumentNullException(nameof(storageAccount));
}
return(new JobOutputStorage(storageAccount, job.Id));
}
private async Task SubmitJobAsync(BatchClient batchClient, string jobId, string poolId)
{
//create an empty unbound Job
CloudJob unboundJob = batchClient.JobOperations.CreateJob();
unboundJob.Id = jobId;
unboundJob.PoolInformation = new PoolInformation {
PoolId = poolId
};
//Commit Job to create it in the service
await unboundJob.CommitAsync();
}
internal static Dictionary <string, object?> ToDictionary(this CloudJob cloudJob) =>
new Dictionary <string, object?>()
{
["id"] = cloudJob.Id,
["name"] = cloudJob.Details.Name,
["status"] = cloudJob.Status,
["uri"] = cloudJob.Uri.ToString(),
["provider"] = cloudJob.Details.ProviderId,
["target"] = cloudJob.Details.Target,
["creation_time"] = cloudJob.Details.CreationTime.ToDateTime()?.ToUniversalTime(),
["begin_execution_time"] = cloudJob.Details.BeginExecutionTime.ToDateTime()?.ToUniversalTime(),
["end_execution_time"] = cloudJob.Details.EndExecutionTime.ToDateTime()?.ToUniversalTime(),
};
private static async Task JobCreation(BatchClient p_batchClient, string p_jobId, string p_poolId)
{
Console.WriteLine("Creating the job");
CloudJob demo_job = p_batchClient.JobOperations.CreateJob();
demo_job.Id = p_jobId;
demo_job.PoolInformation = new PoolInformation {
PoolId = p_poolId
};
await demo_job.CommitAsync();
}
internal static Dictionary <string, object?> ToDictionary(this CloudJob cloudJob) =>
new Dictionary <string, object?>()
{
// TODO: add cloudJob.Uri after https://github.com/microsoft/qsharp-runtime/issues/236 is fixed.
["id"] = cloudJob.Id,
["name"] = cloudJob.Details.Name,
["status"] = cloudJob.Status,
["provider"] = cloudJob.Details.ProviderId,
["target"] = cloudJob.Details.Target,
["creation_time"] = cloudJob.Details.CreationTime.ToDateTime()?.ToUniversalTime(),
["begin_execution_time"] = cloudJob.Details.BeginExecutionTime.ToDateTime()?.ToUniversalTime(),
["end_execution_time"] = cloudJob.Details.EndExecutionTime.ToDateTime()?.ToUniversalTime(),
};
public void CloudJob_WhenSendingToTheServer_HasExpectedUnboundProperties()
{
const string jobId = "id-123";
const string displayName = "DisplayNameFoo";
MetadataItem metadataItem = new MetadataItem("foo", "bar");
const int priority = 0;
const string applicationId = "testApp";
const string applicationVersion = "beta";
BatchSharedKeyCredentials credentials = ClientUnitTestCommon.CreateDummySharedKeyCredential();
using (BatchClient client = BatchClient.Open(credentials))
{
CloudJob cloudJob = client.JobOperations.CreateJob(jobId, new PoolInformation {
AutoPoolSpecification = new AutoPoolSpecification {
KeepAlive = false
}
});
cloudJob.Id = jobId;
cloudJob.DisplayName = displayName;
cloudJob.Metadata = new List <MetadataItem> {
metadataItem
};
cloudJob.Priority = priority;
cloudJob.JobManagerTask = new JobManagerTask {
ApplicationPackageReferences = new List <ApplicationPackageReference>
{
new ApplicationPackageReference {
ApplicationId = applicationId, Version = applicationVersion
}
}
};
cloudJob.OnAllTasksComplete = OnAllTasksComplete.NoAction;
cloudJob.OnTaskFailure = OnTaskFailure.NoAction;
Assert.Throws <InvalidOperationException>(() => cloudJob.Url); // cannot read a Url since it's unbound at this point.
Assert.Equal(cloudJob.Id, jobId); // can set an unbound object
Assert.Equal(cloudJob.Metadata.First().Name, metadataItem.Name);
Assert.Equal(cloudJob.Metadata.First().Value, metadataItem.Value);
Assert.Equal(cloudJob.OnAllTasksComplete, OnAllTasksComplete.NoAction);
Assert.Equal(cloudJob.OnTaskFailure, OnTaskFailure.NoAction);
cloudJob.Commit(additionalBehaviors: InterceptorFactory.CreateAddJobRequestInterceptor());
// writing isn't allowed for a job that is in an invalid state.
Assert.Throws <InvalidOperationException>(() => cloudJob.Id = "cannot-change-id");
Assert.Throws <InvalidOperationException>(() => cloudJob.DisplayName = "cannot-change-display-name");
}
}
public async Task CanCreateJobAndAutoPoolWithAppPackageReferences()
{
string jobId = Guid.NewGuid().ToString();
var poolInformation = new PoolInformation
{
AutoPoolSpecification = new AutoPoolSpecification
{
PoolSpecification = new PoolSpecification
{
ApplicationPackageReferences = new List <ApplicationPackageReference>
{
new ApplicationPackageReference
{
ApplicationId = ApplicationId,
Version = Version
}
},
CloudServiceConfiguration = new CloudServiceConfiguration(PoolFixture.OSFamily),
VirtualMachineSize = PoolFixture.VMSize,
},
PoolLifetimeOption = PoolLifetimeOption.Job
}
};
async Task test()
{
using BatchClient client = await TestUtilities.OpenBatchClientFromEnvironmentAsync().ConfigureAwait(false);
try
{
var job = client.JobOperations.CreateJob(jobId, poolInformation);
await job.CommitAsync().ConfigureAwait(false);
CloudJob jobResponse = await client.JobOperations.GetJobAsync(jobId).ConfigureAwait(false);
ApplicationPackageReference apr = jobResponse.PoolInformation.AutoPoolSpecification.PoolSpecification.ApplicationPackageReferences.First();
Assert.Equal(ApplicationId, apr.ApplicationId);
Assert.Equal(Version, apr.Version);
}
finally
{
TestUtilities.DeleteJobIfExistsAsync(client, jobId).Wait();
}
}
await SynchronizationContextHelper.RunTestAsync(test, LongTestTimeout);
}
/// <summary>
/// Creates a job in the specified pool.
/// </summary>
/// <param name="batchClient">A <see cref="BatchClient"/>.</param>
/// <param name="jobId">The id of the job to be created.</param>
/// <param name="poolId">The id of the <see cref="CloudPool"/> in which to create the job.</param>
/// <returns>A <see cref="System.Threading.Tasks.Task"/> object that represents the asynchronous operation.</returns>
private static async Task CreateJobAsync(BatchClient batchClient, string jobId, string poolId, CloudStorageAccount linkedStorageAccount)
{
Console.WriteLine("Creating job [{0}]...", jobId);
CloudJob job = batchClient.JobOperations.CreateJob();
job.Id = jobId;
job.PoolInformation = new PoolInformation {
PoolId = poolId
};
job.UsesTaskDependencies = true;
await job.CommitAsync();
}
/// <summary>
/// Tests if a job has been completed.
/// </summary>
/// <returns>True if the job was completed or disabled, otherwise false.</returns>
private bool IsJobComplete()
{
ODATADetailLevel detailLevel = new ODATADetailLevel {
SelectClause = "id"
};
// This is how it was done in MARS. Not sure that both of these are necessary.
CloudJob tmpJob = batchClient.JobOperations.ListJobs(detailLevel).FirstOrDefault(j => string.Equals(jobId.ToString(), j.Id));
CloudJob job = tmpJob == null ? tmpJob : batchClient.JobOperations.GetJob(jobId.ToString());
return(job == null || job.State == JobState.Completed || job.State == JobState.Disabled);
// a simpler solution would be -
//return batchClient.JobOperations.GetJob(jobId.ToString()).Id;
}
private async Task CreateJobTask(BatchClient batchClient)
{
Console.WriteLine("Creating job [{0}]...", PoolKeys.JobId);
CloudJob job = batchClient.JobOperations.CreateJob();
job.Id = PoolKeys.JobId;
job.Priority = 1000;
job.PoolInformation = new PoolInformation {
PoolId = PoolKeys.PoolId
};
await job.CommitAsync();
}
public async Task Job_GetTaskCounts_ReturnsCorrectCountNonZeroTaskSlots()
{
Func <Task> test = async() =>
{
using (BatchClient batchCli = await TestUtilities.OpenBatchClientFromEnvironmentAsync().ConfigureAwait(false))
{
string jobId = "NonZeroTaskSlots-" + TestUtilities.GetMyName();
try
{
PoolInformation poolInfo = new PoolInformation()
{
PoolId = "Fake"
};
CloudJob unboundJob = batchCli.JobOperations.CreateJob(jobId, poolInfo);
await unboundJob.CommitAsync().ConfigureAwait(false);
await unboundJob.RefreshAsync().ConfigureAwait(false);
CloudTask t1 = new CloudTask("t1", "cmd /c dir");
t1.RequiredSlots = 2;
CloudTask t2 = new CloudTask("t2", "cmd /c ping 127.0.0.1 -n 4");
t2.RequiredSlots = 3;
await unboundJob.AddTaskAsync(new[] { t1, t2 }).ConfigureAwait(false);
await Task.Delay(TimeSpan.FromSeconds(5)).ConfigureAwait(false); // Give the service some time to get the counts
var counts = await unboundJob.GetTaskCountsAsync().ConfigureAwait(false);
var retTask1 = await unboundJob.GetTaskAsync(t1.Id);
Assert.Equal(t1.RequiredSlots, retTask1.RequiredSlots);
var retTask2 = await unboundJob.GetTaskAsync(t1.Id);
Assert.Equal(t1.RequiredSlots, retTask2.RequiredSlots);
Assert.Equal(2, counts.TaskCounts.Active);
// Task slots counts is currently broken
// Assert.Equal(5, counts.TaskSlotCounts.Active);
}
finally
{
await TestUtilities.DeleteJobIfExistsAsync(batchCli, jobId);
}
}
};
await SynchronizationContextHelper.RunTestAsync(test, TestTimeout);
}
// Calls the Batch service to get metrics for a single job. The first time the
// MetricMonitor sees a job, it creates a TaskStateCache to hold task state information,
// and queries the states of *all* tasks in the job. Subsequent times, it queries
// only for tasks whose states have changed since the previous query -- this significant
// reduces download volumes for large jobs. In either case, it then updates the
// cached task states and aggregates them into a TaskStateCounts object.
private async Task CollectTaskMetricsAsync(MetricEvent.Builder metricsBuilder, CloudJob job)
{
TaskStateCache taskStateCache;
bool firstTime = !this.jobStateCache.ContainsKey(job.Id);
if (firstTime)
{
taskStateCache = new TaskStateCache();
this.jobStateCache.Add(job.Id, taskStateCache);
}
else
{
taskStateCache = this.jobStateCache[job.Id];
}
// If the monitor API is called for the first time, it has to issue a query to enumerate all the tasks once to get its state.
// This is a relatively slow query.
// Subsequent calls to the monitor API will only look for changes to the task state since the last time the query was issued and
// a clock skew (which is within 30 seconds approximately for Azure). Thus if the monitoring API periodicity is 1 minute, then the query
// should look for changes in the last minute and 30 seconds.
// TODO: it would be better to record the time at which the last query was issued and use that,
// rather than subtracting the monitor interval from the current time
DateTime since = DateTime.UtcNow - (this.monitorInterval + MaximumClockSkew);
var tasksToList = firstTime ? DetailLevels.IdAndState.AllEntities : DetailLevels.IdAndState.OnlyChangedAfter(since);
var listTasksTimer = Stopwatch.StartNew();
var tasks = await job.ListTasks(tasksToList).ToListAsync(this.runCancel.Token);
listTasksTimer.Stop();
var listTasksLatency = listTasksTimer.Elapsed;
foreach (var task in tasks)
{
taskStateCache.UpdateTaskState(task.Id, task.State.Value);
}
var taskStateCounts = taskStateCache.GetTaskStateCounts();
metricsBuilder.JobStats.Add(job.Id, new JobMetrics(listTasksLatency, taskStateCounts));
}
public JobModel(CloudJob job)
{
this.Job = job;
this.Tasks = new List<TaskModel>();
this.LastUpdatedTime = DateTime.UtcNow;
this.TaskCollection = CollectionViewSource.GetDefaultView(this.Tasks);
this.UpdateTaskView();
}
/// <summary>
/// Monitors the specified job's tasks and returns each as they complete. When all
/// of the tasks in the job have completed, the method returns.
/// </summary>
/// <param name="job">The <see cref="CloudJob"/> containing the tasks to monitor.</param>
/// <returns>One or more completed <see cref="CloudTask"/>.</returns>
private static IEnumerable<CloudTask> CompletedTasks(CloudJob job)
{
HashSet<string> yieldedTasks = new HashSet<string>();
ODATADetailLevel detailLevel = new ODATADetailLevel();
detailLevel.SelectClause = "id,state,url";
while (true)
{
List<CloudTask> tasks = job.ListTasks(detailLevel).ToList();
IEnumerable<CloudTask> newlyCompleted = tasks.Where(t => t.State == Microsoft.Azure.Batch.Common.TaskState.Completed)
.Where(t => !yieldedTasks.Contains(t.Id));
foreach (CloudTask task in newlyCompleted)
{
yield return task;
yieldedTasks.Add(task.Id);
}
if (yieldedTasks.Count == tasks.Count)
{
yield break;
}
}
}
static void AddTasks(BatchClient client, CloudJob cloudJob, string jobId, IEnumerable<IComputeTask> computeTasks)
{
foreach (var computeTask in computeTasks)
{
var definition = computeTask.Definition;
var executable = new ResourceFile($"{definition.StorageUri}/{definition.ExecutableName}", definition.ExecutableName);
var resources = definition.Resources.Select(resource => new ResourceFile($"{definition.StorageUri}/{resource}", resource));
var inputs = computeTask.Inputs.Select(input => new ResourceFile($"{definition.StorageUri}/{input}", input));
var resourceFiles = new List<ResourceFile> { executable };
resourceFiles.AddRange(resources);
resourceFiles.AddRange(inputs);
var task = client.JobOperations.ListTasks(jobId).SingleOrDefault(t => t.Id == computeTask.Id);
if (task == null)
{
task = new CloudTask(computeTask.Id, computeTask.CommandLine)
{
ResourceFiles = resourceFiles
};
cloudJob.AddTask(task);
cloudJob.Commit();
cloudJob.Refresh();
}
}
client.Utilities.CreateTaskStateMonitor().WaitAll(cloudJob.ListTasks(), TaskState.Completed, new TimeSpan(0, 30, 0));
}
