public async Task CreateJobAsync(CreateJobOptions createJobOptions)
{
CloudJob unboundJob = this.Client.JobOperations.CreateJob();
unboundJob.Id = createJobOptions.JobId;
unboundJob.Priority = createJobOptions.Priority;
unboundJob.Constraints = new JobConstraints(createJobOptions.MaxWallClockTime, createJobOptions.MaxRetryCount);
PoolInformation poolInformation = new PoolInformation();
if (createJobOptions.AutoPoolOptions.UseAutoPool.HasValue && createJobOptions.AutoPoolOptions.UseAutoPool.Value)
{
AutoPoolSpecification autoPoolSpecification = new AutoPoolSpecification()
{
AutoPoolIdPrefix = createJobOptions.AutoPoolOptions.AutoPoolPrefix,
KeepAlive = createJobOptions.AutoPoolOptions.KeepAlive,
PoolLifetimeOption = (PoolLifetimeOption)Enum.Parse(typeof(PoolLifetimeOption), createJobOptions.AutoPoolOptions.LifeTimeOption),
PoolSpecification = new PoolSpecification()
{
CloudServiceConfiguration = new CloudServiceConfiguration(createJobOptions.AutoPoolOptions.OSFamily),
VirtualMachineSize = createJobOptions.AutoPoolOptions.VirutalMachineSize,
TargetDedicatedComputeNodes = createJobOptions.AutoPoolOptions.TargetDedicated,
TargetLowPriorityComputeNodes = createJobOptions.AutoPoolOptions.TargetLowPriority
}
};
poolInformation.AutoPoolSpecification = autoPoolSpecification;
}
else
{
poolInformation.PoolId = createJobOptions.PoolId;
}
unboundJob.PoolInformation = poolInformation;
if (createJobOptions.CreateJobManager.HasValue && createJobOptions.CreateJobManager.Value == true)
{
JobManagerTask jobManager = new JobManagerTask()
{
CommandLine = createJobOptions.JobManagerOptions.CommandLine,
KillJobOnCompletion = createJobOptions.JobManagerOptions.KillOnCompletion,
Id = createJobOptions.JobManagerOptions.JobManagerId
};
jobManager.Constraints = new TaskConstraints(
createJobOptions.JobManagerOptions.MaxTaskWallClockTime,
createJobOptions.JobManagerOptions.RetentionTime,
createJobOptions.JobManagerOptions.MaxTaskRetryCount);
unboundJob.JobManagerTask = jobManager;
}
await unboundJob.CommitAsync();
}
public void TestAutoPoolSpecificationUnboundConstraints()
{
const string idPrefix = "Bar";
const bool keepAlive = false;
const PoolLifetimeOption poolLifetimeOption = PoolLifetimeOption.Job;
PoolSpecification poolSpecification = new PoolSpecification();
AutoPoolSpecification autoPoolSpecification = new AutoPoolSpecification();
//Properties should start out as their defaults
Assert.Equal(default, autoPoolSpecification.AutoPoolIdPrefix);
/// <summary>
/// This function controls how the Azure pools/VMs are setup.
/// This is not really controllable by the user but probably should be.
/// </summary>
/// <param name="job">Job parameters.</param>
private PoolInformation GetPoolInfo(JobParameters job)
{
var autoPoolSpecification = new AutoPoolSpecification
{
PoolLifetimeOption = PoolLifetimeOption.Job,
PoolSpecification = new PoolSpecification
{
ResizeTimeout = TimeSpan.FromMinutes(15),
// todo: look into using ComputeNodeFillType.Pack
TaskSchedulingPolicy = new TaskSchedulingPolicy(ComputeNodeFillType.Spread),
// This specifies the OS that our VM will be running.
// OS Family 5 means .NET 4.6 will be installed.
// For more info see:
// https://docs.microsoft.com/en-us/azure/cloud-services/cloud-services-guestos-update-matrix#releases
CloudServiceConfiguration = new CloudServiceConfiguration("5"),
// For now, always use standard_d5_v2 VM type.
// This VM has 16 vCPUs, 56 GiB of memory and 800 GiB temp (SSD) storage.
// todo: should make this user-controllable
// For other VM sizes, see:
// https://docs.microsoft.com/azure/batch/batch-pool-vm-sizes
// https://docs.microsoft.com/azure/virtual-machines/windows/sizes-general
VirtualMachineSize = "standard_d5_v2",
// Each task needs only one vCPU. Therefore number of tasks per VM will be number of vCPUs per VM.
MaxTasksPerComputeNode = 16
}
};
// We only use one pool, so number of nodes per pool will be total number of vCPUs (as specified by the user)
// divided by number of vCPUs per VM. We've hardcoded VM size to standard_d5_v2, which has 16 vCPUs.
if (job.LowPriority)
{
autoPoolSpecification.PoolSpecification.TargetLowPriorityComputeNodes = job.CpuCount / 16;
}
else
{
autoPoolSpecification.PoolSpecification.TargetDedicatedComputeNodes = job.CpuCount / 16;
}
return(new PoolInformation {
AutoPoolSpecification = autoPoolSpecification
});
}
public void TestAutoPoolSpecificationUnboundConstraints()
{
const string idPrefix = "Bar";
const bool keepAlive = false;
const PoolLifetimeOption poolLifetimeOption = PoolLifetimeOption.Job;
PoolSpecification poolSpecification = new PoolSpecification();
AutoPoolSpecification autoPoolSpecification = new AutoPoolSpecification();
//Properties should start out as their defaults
Assert.Equal(default(string), autoPoolSpecification.AutoPoolIdPrefix);
Assert.Equal(default(bool?), autoPoolSpecification.KeepAlive);
Assert.Equal(default(PoolLifetimeOption), autoPoolSpecification.PoolLifetimeOption);
Assert.Equal(default(PoolSpecification), autoPoolSpecification.PoolSpecification);
Assert.False(((IModifiable)autoPoolSpecification).HasBeenModified);
//Should be able to set all properties
autoPoolSpecification.AutoPoolIdPrefix = idPrefix;
autoPoolSpecification.KeepAlive = keepAlive;
autoPoolSpecification.PoolLifetimeOption = poolLifetimeOption;
autoPoolSpecification.PoolSpecification = poolSpecification;
Assert.True(((IModifiable)autoPoolSpecification).HasBeenModified);
Protocol.Models.AutoPoolSpecification protoAutoPoolSpecification = autoPoolSpecification.GetTransportObject();
((IReadOnly)autoPoolSpecification).IsReadOnly = true; //Forces read-onlyness
//After MarkReadOnly, the original object should be unsettable
Assert.Throws <InvalidOperationException>(() => autoPoolSpecification.AutoPoolIdPrefix = "bar");
Assert.Throws <InvalidOperationException>(() => autoPoolSpecification.KeepAlive = false);
Assert.Throws <InvalidOperationException>(() => autoPoolSpecification.PoolLifetimeOption = PoolLifetimeOption.JobSchedule);
InvalidOperationException e = Assert.Throws <InvalidOperationException>(() => autoPoolSpecification.PoolSpecification = new PoolSpecification());
this.testOutputHelper.WriteLine(e.ToString());
//After GetProtocolObject, the child objects should be unreadable too
Assert.Throws <InvalidOperationException>(() => poolSpecification.TaskSlotsPerNode = 4);
//The original data should be on the protocol specification
Assert.Equal(idPrefix, protoAutoPoolSpecification.AutoPoolIdPrefix);
Assert.Equal(keepAlive, protoAutoPoolSpecification.KeepAlive);
Assert.Equal(poolLifetimeOption, UtilitiesInternal.MapEnum <Protocol.Models.PoolLifetimeOption, PoolLifetimeOption>(protoAutoPoolSpecification.PoolLifetimeOption));
}
public void TestAutoPoolSpecificationBoundConstraints()
{
Protocol.Models.AutoPoolSpecification protoAutoPoolSpecification = new Protocol.Models.AutoPoolSpecification
{
KeepAlive = true,
PoolLifetimeOption = Protocol.Models.PoolLifetimeOption.JobSchedule,
AutoPoolIdPrefix = "Matt",
Pool = new Protocol.Models.PoolSpecification
{
CloudServiceConfiguration = new Protocol.Models.CloudServiceConfiguration(
"4"),
ResizeTimeout = TimeSpan.FromDays(1),
StartTask = new Protocol.Models.StartTask
{
CommandLine = "Bar"
}
}
};
AutoPoolSpecification autoPoolSpecification = new AutoPoolSpecification(protoAutoPoolSpecification);
//Assert that the wrapped properties are equal as we expect
Assert.Equal(protoAutoPoolSpecification.AutoPoolIdPrefix, autoPoolSpecification.AutoPoolIdPrefix);
Assert.Equal(protoAutoPoolSpecification.KeepAlive, autoPoolSpecification.KeepAlive);
Assert.Equal(UtilitiesInternal.MapEnum <Protocol.Models.PoolLifetimeOption, PoolLifetimeOption>(protoAutoPoolSpecification.PoolLifetimeOption), autoPoolSpecification.PoolLifetimeOption);
Assert.NotNull(autoPoolSpecification.PoolSpecification);
Assert.NotNull(protoAutoPoolSpecification.Pool.CloudServiceConfiguration);
Assert.Equal(protoAutoPoolSpecification.Pool.CloudServiceConfiguration.OsFamily, autoPoolSpecification.PoolSpecification.CloudServiceConfiguration.OSFamily);
Assert.Equal(protoAutoPoolSpecification.Pool.ResizeTimeout, autoPoolSpecification.PoolSpecification.ResizeTimeout);
Assert.NotNull(autoPoolSpecification.PoolSpecification.StartTask);
Assert.Equal(protoAutoPoolSpecification.Pool.StartTask.CommandLine, autoPoolSpecification.PoolSpecification.StartTask.CommandLine);
//When we change a property on the underlying object PoolSpecification, AutoPoolSpecification should notice the change
Assert.False(((IModifiable)autoPoolSpecification).HasBeenModified);
autoPoolSpecification.PoolSpecification.ResizeTimeout = TimeSpan.FromSeconds(122);
Assert.True(((IModifiable)autoPoolSpecification).HasBeenModified);
}
public void Bug1965363_2384616_Wat7OSVersionFeatures()
{
Action test = () =>
{
using (BatchClient batchCli = TestUtilities.OpenBatchClientAsync(TestUtilities.GetCredentialsFromEnvironment()).Result)
{
PoolOperations poolOperations = batchCli.PoolOperations;
try
{
this.testOutputHelper.WriteLine("Listing OS Versions:");
/* bug 2384616 ListOsVersions hidden for wat 8
*
* // test ListOSVersion
* foreach (OSVersion curOSV in poolMgr.ListOSVersions())
* {
* this.testOutputHelper.WriteLine("Label: " + curOSV.Label);
* this.testOutputHelper.WriteLine(" Version: " + curOSV.Version);
* this.testOutputHelper.WriteLine(" Family: " + curOSV.Family);
* this.testOutputHelper.WriteLine(" FamilyLabel: " + curOSV.FamilyLabel);
* this.testOutputHelper.WriteLine(" isDefault: " + curOSV.IsDefault);
* this.testOutputHelper.WriteLine(" IsActive: " + curOSV.IsActive);
*
* string expDate;
*
* if (curOSV.ExpirationDate.HasValue)
* {
* expDate = curOSV.ExpirationDate.Value.ToString();
* }
* else
* {
* expDate = "<null/novalue>";
* }
*
* this.testOutputHelper.WriteLine(" ExpirationDate: " + expDate);
* }
*
*/
// create pool tests
// forget to set CloudServiceConfiguration on Create, get error
{
CloudPool noArgs = poolOperations.CreatePool("Bug1965363ButNoOSFamily-" + TestUtilities.GetMyName(), PoolFixture.VMSize, default(CloudServiceConfiguration), targetDedicated: 0);
BatchException ex = TestUtilities.AssertThrows <BatchException>(() => noArgs.Commit());
string exStr = ex.ToString();
// we are expecting an exception, assert if the exception is not the correct one.
Assert.Contains("cloudServiceConfiguration", exStr);
}
// create a pool WITH an osFamily
{
string poolIdHOSF = "Bug1965363HasOSF-" + TestUtilities.GetMyName();
try
{
CloudPool hasOSF = poolOperations.CreatePool(poolIdHOSF, PoolFixture.VMSize, new CloudServiceConfiguration(PoolFixture.OSFamily), targetDedicated: 0);
hasOSF.Commit();
}
finally
{
poolOperations.DeletePool(poolIdHOSF);
}
}
// TODO: ultimately we will either need to find (via list) a family with more than one version or
// manually update these strings as OS versions are depricated
//See here for other OS versions if this test fails: http://azure.microsoft.com/en-us/documentation/articles/cloud-services-guestos-update-matrix/
const string familyVersion0 = "*";
const string familyVersion1 = "WA-GUEST-OS-4.32_201605-01";
// "UpdatePoolOS" tests (ChangeOSVersion in OM)
// PoolManager
{
string poolIdChangeOSV = "Bug1965363ChangeOSVviaMGR-" + TestUtilities.GetMyName();
try
{
CloudPool unboundPool = poolOperations.CreatePool(
poolIdChangeOSV,
PoolFixture.VMSize,
new CloudServiceConfiguration(PoolFixture.OSFamily, familyVersion0), // start with version 0
targetDedicated: 0);
unboundPool.Commit();
// fetch the bound pool
CloudPool boundPool = poolOperations.GetPool(poolIdChangeOSV);
Assert.Equal(familyVersion0, boundPool.CloudServiceConfiguration.CurrentOSVersion);
// switch to new version
poolOperations.ChangeOSVersion(poolIdChangeOSV, familyVersion1);
// UpdatePoolOS is has latency???
PollForOSVersionChange(boundPool, familyVersion1);
// check to make sure the new value is set
boundPool.Refresh();
Assert.Equal(familyVersion1, boundPool.CloudServiceConfiguration.CurrentOSVersion);
}
finally
{
TestUtilities.DeletePoolIfExistsAsync(batchCli, poolIdChangeOSV).Wait();
}
}
// ICloudPool
{
string poolIdChangeOSV = "Bug1965363ChangeOSVviaPool-" + TestUtilities.GetMyName();
try
{
CloudPool unboundPool = poolOperations.CreatePool(
poolIdChangeOSV,
PoolFixture.VMSize,
new CloudServiceConfiguration(PoolFixture.OSFamily, familyVersion0), // start with version 0
targetDedicated: 0);
unboundPool.Commit();
// fetch the bound pool
CloudPool boundPool = poolOperations.GetPool(poolIdChangeOSV);
Assert.Equal(familyVersion0, boundPool.CloudServiceConfiguration.CurrentOSVersion);
// switch to new version
boundPool.ChangeOSVersion(familyVersion1);
// UpdatePoolOS is has latency???
PollForOSVersionChange(boundPool, familyVersion1);
// check to make sure the new value is set
boundPool.Refresh();
Assert.Equal(familyVersion1, boundPool.CloudServiceConfiguration.CurrentOSVersion);
}
finally
{
TestUtilities.DeletePoolIfExistsAsync(batchCli, poolIdChangeOSV).Wait();
}
}
// autopoolspec tests
{
string jobId = "Bug1965363WIName-" + TestUtilities.GetMyName();
// test not setting osversion
try
{
CloudJob unboundJob = batchCli.JobOperations.CreateJob(jobId, new PoolInformation());
AutoPoolSpecification aps = new AutoPoolSpecification();
PoolSpecification ps = new PoolSpecification();
// test unbound set constraint
ps.CloudServiceConfiguration = new CloudServiceConfiguration(PoolFixture.OSFamily);
// test unbound get constraint
this.testOutputHelper.WriteLine("pus.CloudServiceConfiguration.OSFamily == " + ps.CloudServiceConfiguration.OSFamily);
ps.VirtualMachineSize = PoolFixture.VMSize;
ps.TargetDedicated = 0; // trivial size for testing purposes
aps.PoolSpecification = ps;
aps.PoolLifetimeOption = PoolLifetimeOption.Job;
unboundJob.PoolInformation.AutoPoolSpecification = aps;
// commit to test validation
unboundJob.Commit();
// get bound job
CloudJob boundJob = batchCli.JobOperations.GetJob(jobId);
// test bound get constraints
this.testOutputHelper.WriteLine(" OSFamily == " +
boundJob.PoolInformation.AutoPoolSpecification.PoolSpecification.CloudServiceConfiguration.OSFamily);
string targetOSVersion =
boundJob.PoolInformation.AutoPoolSpecification.PoolSpecification.CloudServiceConfiguration.TargetOSVersion;
if (string.IsNullOrEmpty(targetOSVersion))
{
targetOSVersion = "<null or empty";
}
this.testOutputHelper.WriteLine(" TargetOSVersion == " + targetOSVersion);
}
finally
{
// cleanup
TestUtilities.DeleteJobIfExistsAsync(batchCli, jobId).Wait();
}
{
string jobScheduleId = "Bug1965363WINameSettingAndChanging-" + TestUtilities.GetMyName();
// test setting osversion
try
{
AutoPoolSpecification aps = new AutoPoolSpecification();
PoolSpecification ps = new PoolSpecification();
CloudJobSchedule unboundJobSchedule = batchCli.JobScheduleOperations.CreateJobSchedule(
jobScheduleId,
new Schedule()
{
RecurrenceInterval = TimeSpan.FromDays(7)
},
new JobSpecification(new PoolInformation()
{
AutoPoolSpecification = aps
}));
// test unbound set constraint
ps.CloudServiceConfiguration = new CloudServiceConfiguration(PoolFixture.OSFamily, familyVersion0);
// test unbound get constraint
this.testOutputHelper.WriteLine("pus.CloudServiceConfiguration.OSFamily == " + ps.CloudServiceConfiguration.OSFamily);
this.testOutputHelper.WriteLine("pus.CloudServiceConfiguration.TargetOSVersion == " + ps.CloudServiceConfiguration.TargetOSVersion);
ps.VirtualMachineSize = PoolFixture.VMSize;
ps.TargetDedicated = 0; // trivial size for testing purposes
aps.PoolSpecification = ps;
aps.PoolLifetimeOption = PoolLifetimeOption.Job;
unboundJobSchedule.Commit();
// get bound job schedule
CloudJobSchedule boundJobSchedule = batchCli.JobScheduleOperations.GetJobSchedule(jobScheduleId);
// test bound get constraints
this.testOutputHelper.WriteLine(" OSFamily == " + boundJobSchedule.JobSpecification.PoolInformation.AutoPoolSpecification.PoolSpecification.CloudServiceConfiguration.OSFamily);
this.testOutputHelper.WriteLine(" TargetOSVersion == " + boundJobSchedule.JobSpecification.PoolInformation.AutoPoolSpecification.PoolSpecification.CloudServiceConfiguration.TargetOSVersion);
// assert the value is as set above
Assert.Equal(familyVersion0, boundJobSchedule.JobSpecification.PoolInformation.AutoPoolSpecification.PoolSpecification.CloudServiceConfiguration.TargetOSVersion);
// change values
const string altFamily = "3";
const string altOSVersion = "WA-GUEST-OS-3.39_201605-01";
// change values on the bound PUS
PoolSpecification boundPS = boundJobSchedule.JobSpecification.PoolInformation.AutoPoolSpecification.PoolSpecification;
boundPS.CloudServiceConfiguration = new CloudServiceConfiguration(altFamily, altOSVersion);
// flush changes
boundJobSchedule.Commit();
// confirm changes took
boundJobSchedule.Refresh();
Assert.Equal(altFamily, boundJobSchedule.JobSpecification.PoolInformation.AutoPoolSpecification.PoolSpecification.CloudServiceConfiguration.OSFamily);
Assert.Equal(altOSVersion, boundJobSchedule.JobSpecification.PoolInformation.AutoPoolSpecification.PoolSpecification.CloudServiceConfiguration.TargetOSVersion);
}
finally
{
// cleanup
TestUtilities.DeleteJobScheduleIfExistsAsync(batchCli, jobScheduleId).Wait();
}
}
}
}
catch (Exception ex)
{
// special case os version beacuse it is a common failure and requires human intervention/editing
// test for expired os version
Assert.DoesNotContain("The specified OS Version does not exists", ex.ToString());
throw;
}
}
};
SynchronizationContextHelper.RunTest(test, TestTimeout);
}
/// <summary>
/// Populates Azure Storage with the required files, and
/// submits the work item to the Azure Batch service.
/// </summary>
public async Task RunAsync()
{
Console.WriteLine("Running with the following settings: ");
Console.WriteLine("----------------------------------------");
Console.WriteLine(this.configurationSettings.ToString());
//Upload resources if required.
if (this.configurationSettings.ShouldUploadResources)
{
Console.WriteLine("Splitting file: {0} into {1} subfiles",
Constants.TextFilePath,
this.configurationSettings.NumberOfMapperTasks);
//Split the text file into the correct number of files for consumption by the mapper tasks.
FileSplitter splitter = new FileSplitter();
List <string> mapperTaskFiles = await splitter.SplitAsync(
Constants.TextFilePath,
this.configurationSettings.NumberOfMapperTasks);
await this.UploadResourcesAsync(mapperTaskFiles);
}
//Generate a SAS for the container.
string containerSasUrl = Helpers.ConstructContainerSas(
this.configurationSettings.StorageAccountName,
this.configurationSettings.StorageAccountKey,
this.configurationSettings.StorageServiceUrl,
this.configurationSettings.BlobContainer);
//Set up the Batch Service credentials used to authenticate with the Batch Service.
BatchCredentials batchCredentials = new BatchCredentials(
this.configurationSettings.BatchAccountName,
this.configurationSettings.BatchAccountKey);
using (IBatchClient batchClient = BatchClient.Connect(this.configurationSettings.BatchServiceUrl, batchCredentials))
{
using (IWorkItemManager workItemManager = batchClient.OpenWorkItemManager())
{
//Create the unbound work item in local memory. An object which exists only in local memory (and not on the Batch Service) is "unbound".
string workItemName = Environment.GetEnvironmentVariable("USERNAME") + DateTime.UtcNow.ToString("yyyyMMdd-HHmmss");
ICloudWorkItem unboundWorkItem = workItemManager.CreateWorkItem(workItemName);
//
// Construct the work item properties in local memory before commiting them to the Batch Service.
//
//Allow enough VMs in the pool to run each mapper task, and 1 extra to run the job manager.
int numberOfPoolVMs = 1 + this.configurationSettings.NumberOfMapperTasks;
//Define the pool specification for the pool which the work item will run on.
IPoolSpecification poolSpecification = new PoolSpecification()
{
TargetDedicated = numberOfPoolVMs,
VMSize = "small",
//You can learn more about os families and versions at:
//http://msdn.microsoft.com/en-us/library/azure/ee924680.aspx
OSFamily = "4",
TargetOSVersion = "*"
};
//Use the auto pool feature of the Batch Service to create a pool when the work item is created.
//This creates a new pool for each work item which is added.
IAutoPoolSpecification autoPoolSpecification = new AutoPoolSpecification()
{
AutoPoolNamePrefix = "TextSearchPool",
KeepAlive = false,
PoolLifeTimeOption = PoolLifeTimeOption.WorkItem,
PoolSpecification = poolSpecification
};
//Define the execution environment for this work item -- it will run on the pool defined by the auto pool specification above.
unboundWorkItem.JobExecutionEnvironment = new JobExecutionEnvironment()
{
AutoPoolSpecification = autoPoolSpecification
};
//Define the job manager for this work item. This job manager will run when any job is created and will submit the tasks for
//the work item. The job manager is the executable which manages the lifetime of the job
//and all tasks which should run for the job. In this case, the job manager submits the mapper and reducer tasks.
string jobManagerCommandLine = string.Format("{0} -JobManagerTask", Constants.TextSearchExe);
List <IResourceFile> jobManagerResourceFiles = Helpers.GetResourceFiles(containerSasUrl, Constants.RequiredExecutableFiles);
const string jobManagerTaskName = "JobManager";
unboundWorkItem.JobSpecification = new JobSpecification()
{
JobManager = new JobManager()
{
ResourceFiles = jobManagerResourceFiles,
CommandLine = jobManagerCommandLine,
//Determines if the job should terminate when the job manager process exits
KillJobOnCompletion = false,
Name = jobManagerTaskName
}
};
try
{
//Commit the unbound work item to the Batch Service.
Console.WriteLine("Adding work item: {0} to the Batch Service.", unboundWorkItem.Name);
await unboundWorkItem.CommitAsync(); //Issues a request to the Batch Service to add the work item which was defined above.
//
// Wait for the job manager task to complete.
//
//An object which is backed by a corresponding Batch Service object is "bound."
ICloudWorkItem boundWorkItem = await workItemManager.GetWorkItemAsync(workItemName);
//Wait for the job to be created automatically by the Batch Service.
string boundJobName = await Helpers.WaitForActiveJobAsync(boundWorkItem);
ICloudTask boundJobManagerTask = await workItemManager.GetTaskAsync(
workItemName,
boundJobName,
jobManagerTaskName);
TimeSpan maxJobCompletionTimeout = TimeSpan.FromMinutes(30);
IToolbox toolbox = batchClient.OpenToolbox();
ITaskStateMonitor monitor = toolbox.CreateTaskStateMonitor();
bool timedOut = await monitor.WaitAllAsync(new List <ICloudTask> {
boundJobManagerTask
}, TaskState.Completed, maxJobCompletionTimeout);
Console.WriteLine("Done waiting for job manager task.");
await boundJobManagerTask.RefreshAsync();
//Check to ensure the job manager task exited successfully.
await Helpers.CheckForTaskSuccessAsync(boundJobManagerTask, dumpStandardOutOnTaskSuccess : true);
if (timedOut)
{
throw new TimeoutException(string.Format("Timed out waiting for job manager task to complete."));
}
}
catch (AggregateException e)
{
e.Handle(
(innerE) =>
{
//We print all the inner exceptions for debugging purposes.
Console.WriteLine(innerE.ToString());
return(false);
});
throw;
}
catch (Exception e)
{
Console.WriteLine("Hit unexpected exception: {0}", e.ToString());
throw;
}
finally
{
//Delete the work item.
//This will delete the auto pool associated with the work item as long as the pool
//keep alive property is set to false.
if (this.configurationSettings.ShouldDeleteWorkItem)
{
Console.WriteLine("Deleting work item {0}", workItemName);
workItemManager.DeleteWorkItem(workItemName);
}
//Note that there were files uploaded to a container specified in the
//configuration file. This container will not be deleted or cleaned up by this sample.
}
}
}
}
public void TestJobUpdateWithAndWithoutPoolInfo()
{
void test()
{
using BatchClient batchCli = TestUtilities.OpenBatchClient(TestUtilities.GetCredentialsFromEnvironment());
const string testName = "TestJobUpdateWithAndWithoutPoolInfo";
// Create a job
string jobId = testName + "_" + TestUtilities.GetMyName();
CloudJob unboundJob = batchCli.JobOperations.CreateJob();
unboundJob.Id = jobId;
// Use an auto pool with the job, since PoolInformation can't be updated otherwise.
PoolSpecification poolSpec = new PoolSpecification();
poolSpec.CloudServiceConfiguration = new CloudServiceConfiguration(PoolFixture.OSFamily, "*");
poolSpec.TargetDedicatedComputeNodes = 0;
poolSpec.VirtualMachineSize = PoolFixture.VMSize;
AutoPoolSpecification autoPoolSpec = new AutoPoolSpecification();
string autoPoolPrefix = "UpdPIAuto_" + TestUtilities.GetMyName();
autoPoolSpec.AutoPoolIdPrefix = autoPoolPrefix;
const bool originalKeepAlive = false;
autoPoolSpec.KeepAlive = originalKeepAlive;
autoPoolSpec.PoolLifetimeOption = PoolLifetimeOption.Job;
autoPoolSpec.PoolSpecification = poolSpec;
PoolInformation poolInfo = new PoolInformation();
poolInfo.AutoPoolSpecification = autoPoolSpec;
unboundJob.PoolInformation = poolInfo;
const int originalPriority = 0;
unboundJob.Priority = originalPriority;
List <MetadataItem> originalMetadata = new List <MetadataItem>
{
new MetadataItem("meta1", "value1"),
new MetadataItem("meta2", "value2")
};
unboundJob.Metadata = originalMetadata;
testOutputHelper.WriteLine("Creating job {0}", jobId);
unboundJob.Commit();
try
{
// Get bound job
CloudJob createdJob = batchCli.JobOperations.GetJob(jobId);
// Verify that we can update something besides PoolInformation without getting an error for not being in the Disabled state.
Assert.NotEqual(JobState.Disabled, createdJob.State);
int updatedPriority = originalPriority + 1;
List <MetadataItem> updatedMetadata = new List <MetadataItem> {
new MetadataItem("updatedMeta1", "value1")
};
createdJob.Priority = updatedPriority;
createdJob.Metadata = updatedMetadata;
testOutputHelper.WriteLine("Updating job {0} without altering PoolInformation", jobId);
createdJob.Commit();
// Verify update occurred
CloudJob updatedJob = batchCli.JobOperations.GetJob(jobId);
Assert.Equal(updatedPriority, updatedJob.Priority);
Assert.Equal(updatedJob.Metadata.Count, updatedJob.Priority);
Assert.Equal(updatedJob.Metadata[0].Name, updatedMetadata[0].Name);
Assert.Equal(updatedJob.Metadata[0].Value, updatedMetadata[0].Value);
// Verify that updating the PoolInformation works.
// PoolInformation can only be changed in the Disabled state.
testOutputHelper.WriteLine("Disabling job {0}", jobId);
updatedJob.Disable(DisableJobOption.Terminate);
while (updatedJob.State != JobState.Disabled)
{
Thread.Sleep(500);
updatedJob.Refresh();
}
Assert.Equal(JobState.Disabled, updatedJob.State);
bool updatedKeepAlive = !originalKeepAlive;
updatedJob.PoolInformation.AutoPoolSpecification.KeepAlive = updatedKeepAlive;
int updatedAgainPriority = updatedPriority + 1;
updatedJob.Priority = updatedAgainPriority;
testOutputHelper.WriteLine("Updating job {0} properties, including PoolInformation", jobId);
updatedJob.Commit();
CloudJob updatedPoolInfoJob = batchCli.JobOperations.GetJob(jobId);
Assert.Equal(updatedKeepAlive, updatedPoolInfoJob.PoolInformation.AutoPoolSpecification.KeepAlive);
Assert.Equal(updatedAgainPriority, updatedPoolInfoJob.Priority);
}
finally
{
testOutputHelper.WriteLine("Deleting job {0}", jobId);
TestUtilities.DeleteJobIfExistsAsync(batchCli, jobId).Wait();
// Explicitly delete auto pool
foreach (CloudPool pool in batchCli.PoolOperations.ListPools(new ODATADetailLevel(filterClause: string.Format("startswith(id,'{0}')", autoPoolPrefix))))
{
testOutputHelper.WriteLine("Deleting pool {0}", pool.Id);
TestUtilities.DeletePoolIfExistsAsync(batchCli, pool.Id).Wait();
}
}
}
SynchronizationContextHelper.RunTest(test, TestTimeout);
}
internal PoolInformation(string poolId, AutoPoolSpecification autoPoolSpecification)
{
PoolId = poolId;
AutoPoolSpecification = autoPoolSpecification;
}
/// <summary>
/// Populates Azure Storage with the required files, and
/// submits the job to the Azure Batch service.
/// </summary>
public async Task RunAsync()
{
Console.WriteLine("Running with the following settings: ");
Console.WriteLine("----------------------------------------");
Console.WriteLine(this.textSearchSettings.ToString());
Console.WriteLine(this.accountSettings.ToString());
CloudStorageAccount cloudStorageAccount = new CloudStorageAccount(
new StorageCredentials(
this.accountSettings.StorageAccountName,
this.accountSettings.StorageAccountKey),
this.accountSettings.StorageServiceUrl,
useHttps: true);
//Upload resources if required
Console.WriteLine($"Creating container {this.textSearchSettings.OutputBlobContainer} if it doesn't exist...");
var blobClient = cloudStorageAccount.CreateCloudBlobClient();
var outputContainer = blobClient.GetContainerReference(this.textSearchSettings.OutputBlobContainer);
await outputContainer.CreateIfNotExistsAsync();
if (this.textSearchSettings.ShouldUploadResources)
{
Console.WriteLine("Splitting file: {0} into {1} subfiles",
Constants.TextFilePath,
this.textSearchSettings.NumberOfMapperTasks);
//Split the text file into the correct number of files for consumption by the mapper tasks.
FileSplitter splitter = new FileSplitter();
List <string> mapperTaskFiles = await splitter.SplitAsync(
Constants.TextFilePath,
this.textSearchSettings.NumberOfMapperTasks);
List <string> files = Constants.RequiredExecutableFiles.Union(mapperTaskFiles).ToList();
await SampleHelpers.UploadResourcesAsync(
cloudStorageAccount,
this.textSearchSettings.InputBlobContainer,
files);
}
//Generate a SAS for the container.
string inputContainerSasUrl = SampleHelpers.ConstructContainerSas(
cloudStorageAccount,
this.textSearchSettings.InputBlobContainer,
permissions: WindowsAzure.Storage.Blob.SharedAccessBlobPermissions.Read);
string outputContainerSasUrl = SampleHelpers.ConstructContainerSas(
cloudStorageAccount,
this.textSearchSettings.OutputBlobContainer,
permissions: WindowsAzure.Storage.Blob.SharedAccessBlobPermissions.Read |
WindowsAzure.Storage.Blob.SharedAccessBlobPermissions.Write);
//Set up the Batch Service credentials used to authenticate with the Batch Service.
BatchSharedKeyCredentials credentials = new BatchSharedKeyCredentials(
this.accountSettings.BatchServiceUrl,
this.accountSettings.BatchAccountName,
this.accountSettings.BatchAccountKey);
using (BatchClient batchClient = BatchClient.Open(credentials))
{
//
// Construct the job properties in local memory before commiting them to the Batch Service.
//
//Allow enough compute nodes in the pool to run each mapper task
int numberOfPoolComputeNodes = this.textSearchSettings.NumberOfMapperTasks;
//Define the pool specification for the pool which the job will run on.
PoolSpecification poolSpecification = new PoolSpecification()
{
TargetDedicatedComputeNodes = numberOfPoolComputeNodes,
VirtualMachineSize = "standard_d1_v2",
//You can learn more about os families and versions at:
//http://azure.microsoft.com/documentation/articles/cloud-services-guestos-update-matrix
CloudServiceConfiguration = new CloudServiceConfiguration(osFamily: "5")
};
//Use the auto pool feature of the Batch Service to create a pool when the job is created.
//This creates a new pool for each job which is added.
AutoPoolSpecification autoPoolSpecification = new AutoPoolSpecification()
{
AutoPoolIdPrefix = "TextSearchPool",
KeepAlive = false,
PoolLifetimeOption = PoolLifetimeOption.Job,
PoolSpecification = poolSpecification
};
//Define the pool information for this job -- it will run on the pool defined by the auto pool specification above.
PoolInformation poolInformation = new PoolInformation()
{
AutoPoolSpecification = autoPoolSpecification
};
//Create the unbound job in local memory. An object which exists only in local memory (and not on the Batch Service) is "unbound".
string jobId = Environment.GetEnvironmentVariable("USERNAME") + DateTime.UtcNow.ToString("yyyyMMdd-HHmmss");
CloudJob unboundJob = batchClient.JobOperations.CreateJob(jobId, poolInformation);
unboundJob.UsesTaskDependencies = true;
try
{
//Commit the unbound job to the Batch Service.
Console.WriteLine($"Adding job: {unboundJob.Id} to the Batch Service.");
await unboundJob.CommitAsync(); //Issues a request to the Batch Service to add the job which was defined above.
// Add tasks to the job
var mapperTasks = CreateMapperTasks(inputContainerSasUrl, outputContainerSasUrl);
var reducerTask = CreateReducerTask(inputContainerSasUrl, outputContainerSasUrl, mapperTasks);
var tasksToAdd = Enumerable.Concat(mapperTasks, new[] { reducerTask });
//Submit the unbound task collection to the Batch Service.
//Use the AddTask method which takes a collection of CloudTasks for the best performance.
Console.WriteLine("Submitting {0} mapper tasks", this.textSearchSettings.NumberOfMapperTasks);
Console.WriteLine("Submitting 1 reducer task");
await batchClient.JobOperations.AddTaskAsync(jobId, tasksToAdd);
//An object which is backed by a corresponding Batch Service object is "bound."
CloudJob boundJob = await batchClient.JobOperations.GetJobAsync(jobId);
// Update the job now that we've added tasks so that when all of the tasks which we have added
// are complete, the job will automatically move to the completed state.
boundJob.OnAllTasksComplete = OnAllTasksComplete.TerminateJob;
boundJob.Commit();
boundJob.Refresh();
//
// Wait for the tasks to complete.
//
List <CloudTask> tasks = await batchClient.JobOperations.ListTasks(jobId).ToListAsync();
TimeSpan maxJobCompletionTimeout = TimeSpan.FromMinutes(30);
// Monitor the current tasks to see when they are done.
// Occasionally a task may get killed and requeued during an upgrade or hardware failure,
// Robustness against this was not added into the sample for
// simplicity, but should be added into any production code.
Console.WriteLine("Waiting for job's tasks to complete");
TaskStateMonitor taskStateMonitor = batchClient.Utilities.CreateTaskStateMonitor();
try
{
await taskStateMonitor.WhenAll(tasks, TaskState.Completed, maxJobCompletionTimeout);
}
finally
{
Console.WriteLine("Done waiting for all tasks to complete");
// Refresh the task list
tasks = await batchClient.JobOperations.ListTasks(jobId).ToListAsync();
//Check to ensure the job manager task exited successfully.
foreach (var task in tasks)
{
await Helpers.CheckForTaskSuccessAsync(task, dumpStandardOutOnTaskSuccess : false);
}
}
//
// Download and write out the reducer tasks output
//
string reducerText = await SampleHelpers.DownloadBlobTextAsync(cloudStorageAccount, this.textSearchSettings.OutputBlobContainer, Constants.ReducerTaskResultBlobName);
Console.WriteLine("Reducer reuslts:");
Console.WriteLine(reducerText);
}
finally
{
//Delete the job.
//This will delete the auto pool associated with the job as long as the pool
//keep alive property is set to false.
if (this.textSearchSettings.ShouldDeleteJob)
{
Console.WriteLine($"Deleting job {jobId}");
await batchClient.JobOperations.DeleteJobAsync(jobId);
}
if (this.textSearchSettings.ShouldDeleteContainers)
{
Console.WriteLine("Deleting containers");
var inputContainer = blobClient.GetContainerReference(this.textSearchSettings.InputBlobContainer);
await inputContainer.DeleteIfExistsAsync();
await outputContainer.DeleteIfExistsAsync();
}
}
}
}
/// <summary>
/// Creates a job schedule with the specified options.
/// </summary>
/// <param name="options">The options describing the job schedule to create.</param>
/// <returns></returns>
public async Task CreateJobScheduleAsync(CreateJobScheduleOptions options)
{
CloudJobSchedule unboundJobSchedule = this.Client.JobScheduleOperations.CreateJobSchedule();
unboundJobSchedule.Id = options.JobScheduleId;
PoolInformation poolInformation = new PoolInformation();
if (options.AutoPoolOptions.UseAutoPool.HasValue && options.AutoPoolOptions.UseAutoPool.Value)
{
AutoPoolSpecification autoPoolSpecification = new AutoPoolSpecification()
{
AutoPoolIdPrefix = options.AutoPoolOptions.AutoPoolPrefix,
KeepAlive = options.AutoPoolOptions.KeepAlive,
PoolLifetimeOption = (PoolLifetimeOption)Enum.Parse(typeof(PoolLifetimeOption), options.AutoPoolOptions.LifeTimeOption),
PoolSpecification = new PoolSpecification()
{
OSFamily = options.AutoPoolOptions.OSFamily,
VirtualMachineSize = options.AutoPoolOptions.VirutalMachineSize,
TargetDedicated = options.AutoPoolOptions.TargetDedicated
}
};
poolInformation.AutoPoolSpecification = autoPoolSpecification;
}
else
{
poolInformation.PoolId = options.PoolId;
}
unboundJobSchedule.JobSpecification = new JobSpecification()
{
Priority = options.Priority,
PoolInformation = poolInformation
};
// TODO: These are read only
unboundJobSchedule.JobSpecification.Constraints = new JobConstraints(options.MaxWallClockTime, options.MaxRetryCount);
Schedule schedule = new Schedule()
{
DoNotRunAfter = options.DoNotRunAfter,
DoNotRunUntil = options.DoNotRunUntil,
RecurrenceInterval = options.RecurrenceInterval,
StartWindow = options.StartWindow
};
unboundJobSchedule.Schedule = schedule;
if (options.CreateJobManager.HasValue && options.CreateJobManager.Value == true)
{
JobManagerTask jobManager = new JobManagerTask()
{
CommandLine = options.JobManagerOptions.CommandLine,
KillJobOnCompletion = options.JobManagerOptions.KillOnCompletion,
Id = options.JobManagerOptions.JobManagerId
};
jobManager.Constraints = new TaskConstraints(options.JobManagerOptions.MaxTaskWallClockTime, options.JobManagerOptions.RetentionTime, options.JobManagerOptions.MaxTaskRetryCount);
unboundJobSchedule.JobSpecification.JobManagerTask = jobManager;
}
await unboundJobSchedule.CommitAsync();
}
/// <summary>
/// Populates Azure Storage with the required files, and
/// submits the work item to the Azure Batch service.
/// </summary>
public async Task RunAsync()
{
Console.WriteLine("Running with the following settings: ");
Console.WriteLine("----------------------------------------");
Console.WriteLine(this.configurationSettings.ToString());
//Upload resources if required.
if (this.configurationSettings.ShouldUploadResources)
{
Console.WriteLine("Splitting file: {0} into {1} subfiles",
Constants.TextFilePath,
this.configurationSettings.NumberOfMapperTasks);
//Split the text file into the correct number of files for consumption by the mapper tasks.
FileSplitter splitter = new FileSplitter();
List<string> mapperTaskFiles = await splitter.SplitAsync(
Constants.TextFilePath,
this.configurationSettings.NumberOfMapperTasks);
await this.UploadResourcesAsync(mapperTaskFiles);
}
//Generate a SAS for the container.
string containerSasUrl = Helpers.ConstructContainerSas(
this.configurationSettings.StorageAccountName,
this.configurationSettings.StorageAccountKey,
this.configurationSettings.StorageServiceUrl,
this.configurationSettings.BlobContainer);
//Set up the Batch Service credentials used to authenticate with the Batch Service.
BatchCredentials batchCredentials = new BatchCredentials(
this.configurationSettings.BatchAccountName,
this.configurationSettings.BatchAccountKey);
using (IBatchClient batchClient = BatchClient.Connect(this.configurationSettings.BatchServiceUrl, batchCredentials))
{
using (IWorkItemManager workItemManager = batchClient.OpenWorkItemManager())
{
//Create the unbound work item in local memory. An object which exists only in local memory (and not on the Batch Service) is "unbound".
string workItemName = Environment.GetEnvironmentVariable("USERNAME") + DateTime.UtcNow.ToString("yyyyMMdd-HHmmss");
ICloudWorkItem unboundWorkItem = workItemManager.CreateWorkItem(workItemName);
//
// Construct the work item properties in local memory before commiting them to the Batch Service.
//
//Allow enough VMs in the pool to run each mapper task, and 1 extra to run the job manager.
int numberOfPoolVMs = 1 + this.configurationSettings.NumberOfMapperTasks;
//Define the pool specification for the pool which the work item will run on.
IPoolSpecification poolSpecification = new PoolSpecification()
{
TargetDedicated = numberOfPoolVMs,
VMSize = "small",
//You can learn more about os families and versions at:
//http://msdn.microsoft.com/en-us/library/azure/ee924680.aspx
OSFamily = "4",
TargetOSVersion = "*"
};
//Use the auto pool feature of the Batch Service to create a pool when the work item is created.
//This creates a new pool for each work item which is added.
IAutoPoolSpecification autoPoolSpecification = new AutoPoolSpecification()
{
AutoPoolNamePrefix = "TextSearchPool",
KeepAlive = false,
PoolLifeTimeOption = PoolLifeTimeOption.WorkItem,
PoolSpecification = poolSpecification
};
//Define the execution environment for this work item -- it will run on the pool defined by the auto pool specification above.
unboundWorkItem.JobExecutionEnvironment = new JobExecutionEnvironment()
{
AutoPoolSpecification = autoPoolSpecification
};
//Define the job manager for this work item. This job manager will run when any job is created and will submit the tasks for
//the work item. The job manager is the executable which manages the lifetime of the job
//and all tasks which should run for the job. In this case, the job manager submits the mapper and reducer tasks.
string jobManagerCommandLine = string.Format("{0} -JobManagerTask", Constants.TextSearchExe);
List<IResourceFile> jobManagerResourceFiles = Helpers.GetResourceFiles(containerSasUrl, Constants.RequiredExecutableFiles);
const string jobManagerTaskName = "JobManager";
unboundWorkItem.JobSpecification = new JobSpecification()
{
JobManager = new JobManager()
{
ResourceFiles = jobManagerResourceFiles,
CommandLine = jobManagerCommandLine,
//Determines if the job should terminate when the job manager process exits
KillJobOnCompletion = false,
Name = jobManagerTaskName
}
};
try
{
//Commit the unbound work item to the Batch Service.
Console.WriteLine("Adding work item: {0} to the Batch Service.", unboundWorkItem.Name);
await unboundWorkItem.CommitAsync(); //Issues a request to the Batch Service to add the work item which was defined above.
//
// Wait for the job manager task to complete.
//
//An object which is backed by a corresponding Batch Service object is "bound."
ICloudWorkItem boundWorkItem = await workItemManager.GetWorkItemAsync(workItemName);
//Wait for the job to be created automatically by the Batch Service.
string boundJobName = await Helpers.WaitForActiveJobAsync(boundWorkItem);
ICloudTask boundJobManagerTask = await workItemManager.GetTaskAsync(
workItemName,
boundJobName,
jobManagerTaskName);
TimeSpan maxJobCompletionTimeout = TimeSpan.FromMinutes(30);
IToolbox toolbox = batchClient.OpenToolbox();
ITaskStateMonitor monitor = toolbox.CreateTaskStateMonitor();
bool timedOut = await monitor.WaitAllAsync(new List<ICloudTask> { boundJobManagerTask }, TaskState.Completed, maxJobCompletionTimeout);
Console.WriteLine("Done waiting for job manager task.");
await boundJobManagerTask.RefreshAsync();
//Check to ensure the job manager task exited successfully.
await Helpers.CheckForTaskSuccessAsync(boundJobManagerTask, dumpStandardOutOnTaskSuccess: true);
if (timedOut)
{
throw new TimeoutException(string.Format("Timed out waiting for job manager task to complete."));
}
}
catch (AggregateException e)
{
e.Handle(
(innerE) =>
{
//We print all the inner exceptions for debugging purposes.
Console.WriteLine(innerE.ToString());
return false;
});
throw;
}
catch (Exception e)
{
Console.WriteLine("Hit unexpected exception: {0}", e.ToString());
throw;
}
finally
{
//Delete the work item.
//This will delete the auto pool associated with the work item as long as the pool
//keep alive property is set to false.
if (this.configurationSettings.ShouldDeleteWorkItem)
{
Console.WriteLine("Deleting work item {0}", workItemName);
workItemManager.DeleteWorkItem(workItemName);
}
//Note that there were files uploaded to a container specified in the
//configuration file. This container will not be deleted or cleaned up by this sample.
}
}
}
}
/// <summary>
/// Populates Azure Storage with the required files, and
/// submits the job to the Azure Batch service.
/// </summary>
public async Task RunAsync()
{
Console.WriteLine("Running with the following settings: ");
Console.WriteLine("----------------------------------------");
Console.WriteLine(this.configurationSettings.ToString());
//Upload resources if required.
if (this.configurationSettings.ShouldUploadResources)
{
Console.WriteLine("Splitting file: {0} into {1} subfiles",
Constants.TextFilePath,
this.configurationSettings.NumberOfMapperTasks);
//Split the text file into the correct number of files for consumption by the mapper tasks.
FileSplitter splitter = new FileSplitter();
List<string> mapperTaskFiles = await splitter.SplitAsync(
Constants.TextFilePath,
this.configurationSettings.NumberOfMapperTasks);
await this.UploadResourcesAsync(mapperTaskFiles);
}
//Generate a SAS for the container.
string containerSasUrl = Helpers.ConstructContainerSas(
this.configurationSettings.StorageAccountName,
this.configurationSettings.StorageAccountKey,
this.configurationSettings.StorageServiceUrl,
this.configurationSettings.BlobContainer);
//Set up the Batch Service credentials used to authenticate with the Batch Service.
BatchSharedKeyCredentials credentials = new BatchSharedKeyCredentials(
this.configurationSettings.BatchServiceUrl,
this.configurationSettings.BatchAccountName,
this.configurationSettings.BatchAccountKey);
using (BatchClient batchClient = await BatchClient.OpenAsync(credentials))
{
//
// Construct the job properties in local memory before commiting them to the Batch Service.
//
//Allow enough compute nodes in the pool to run each mapper task, and 1 extra to run the job manager.
int numberOfPoolComputeNodes = 1 + this.configurationSettings.NumberOfMapperTasks;
//Define the pool specification for the pool which the job will run on.
PoolSpecification poolSpecification = new PoolSpecification()
{
TargetDedicated = numberOfPoolComputeNodes,
VirtualMachineSize = "small",
//You can learn more about os families and versions at:
//http://azure.microsoft.com/documentation/articles/cloud-services-guestos-update-matrix
OSFamily = "4",
TargetOSVersion = "*"
};
//Use the auto pool feature of the Batch Service to create a pool when the job is created.
//This creates a new pool for each job which is added.
AutoPoolSpecification autoPoolSpecification = new AutoPoolSpecification()
{
AutoPoolIdPrefix= "TextSearchPool",
KeepAlive = false,
PoolLifetimeOption = PoolLifetimeOption.Job,
PoolSpecification = poolSpecification
};
//Define the pool information for this job -- it will run on the pool defined by the auto pool specification above.
PoolInformation poolInformation = new PoolInformation()
{
AutoPoolSpecification = autoPoolSpecification
};
//Define the job manager for this job. This job manager will run first and will submit the tasks for
//the job. The job manager is the executable which manages the lifetime of the job
//and all tasks which should run for the job. In this case, the job manager submits the mapper and reducer tasks.
List<ResourceFile> jobManagerResourceFiles = Helpers.GetResourceFiles(containerSasUrl, Constants.RequiredExecutableFiles);
const string jobManagerTaskId = "JobManager";
JobManagerTask jobManagerTask = new JobManagerTask()
{
ResourceFiles = jobManagerResourceFiles,
CommandLine = Constants.JobManagerExecutable,
//Determines if the job should terminate when the job manager process exits.
KillJobOnCompletion = true,
Id = jobManagerTaskId
};
//Create the unbound job in local memory. An object which exists only in local memory (and not on the Batch Service) is "unbound".
string jobId = Environment.GetEnvironmentVariable("USERNAME") + DateTime.UtcNow.ToString("yyyyMMdd-HHmmss");
CloudJob unboundJob = batchClient.JobOperations.CreateJob(jobId, poolInformation);
unboundJob.JobManagerTask = jobManagerTask; //Assign the job manager task to this job
try
{
//Commit the unbound job to the Batch Service.
Console.WriteLine("Adding job: {0} to the Batch Service.", unboundJob.Id);
await unboundJob.CommitAsync(); //Issues a request to the Batch Service to add the job which was defined above.
//
// Wait for the job manager task to complete.
//
//An object which is backed by a corresponding Batch Service object is "bound."
CloudJob boundJob = await batchClient.JobOperations.GetJobAsync(jobId);
CloudTask boundJobManagerTask = await boundJob.GetTaskAsync(jobManagerTaskId);
TimeSpan maxJobCompletionTimeout = TimeSpan.FromMinutes(30);
// Monitor the current tasks to see when they are done.
// Occasionally a task may get killed and requeued during an upgrade or hardware failure, including the job manager
// task. The job manager will be re-run in this case. Robustness against this was not added into the sample for
// simplicity, but should be added into any production code.
Console.WriteLine("Waiting for job's tasks to complete");
TaskStateMonitor taskStateMonitor = batchClient.Utilities.CreateTaskStateMonitor();
bool timedOut = await taskStateMonitor.WaitAllAsync(new List<CloudTask> { boundJobManagerTask }, TaskState.Completed, maxJobCompletionTimeout);
Console.WriteLine("Done waiting for job manager task.");
await boundJobManagerTask.RefreshAsync();
//Check to ensure the job manager task exited successfully.
await Helpers.CheckForTaskSuccessAsync(boundJobManagerTask, dumpStandardOutOnTaskSuccess: false);
if (timedOut)
{
throw new TimeoutException(string.Format("Timed out waiting for job manager task to complete."));
}
//
// Download and write out the reducer tasks output
//
CloudStorageAccount cloudStorageAccount = new CloudStorageAccount(
new StorageCredentials(
this.configurationSettings.StorageAccountName,
this.configurationSettings.StorageAccountKey),
this.configurationSettings.StorageServiceUrl,
useHttps: true);
string reducerText = await Helpers.DownloadBlobTextAsync(cloudStorageAccount, this.configurationSettings.BlobContainer, Constants.ReducerTaskResultBlobName);
Console.WriteLine("Reducer reuslts:");
Console.WriteLine(reducerText);
}
finally
{
//Delete the job.
//This will delete the auto pool associated with the job as long as the pool
//keep alive property is set to false.
if (this.configurationSettings.ShouldDeleteJob)
{
Console.WriteLine("Deleting job {0}", jobId);
batchClient.JobOperations.DeleteJob(jobId);
}
//Note that there were files uploaded to a container specified in the
//configuration file. This container will not be deleted or cleaned up by this sample.
}
}
}
/// <summary>
/// Creates a work item with the specified work item options.
/// </summary>
/// <param name="options">The options describing the work item to create.</param>
/// <returns></returns>
public async Task CreateWorkItemAsync(CreateWorkItemOptions options)
{
try
{
using (IWorkItemManager workItemManager = this.Client.OpenWorkItemManager())
{
ICloudWorkItem unboundWorkItem = workItemManager.CreateWorkItem(options.WorkItemName);
IJobExecutionEnvironment jobExecutionEnvironment = new JobExecutionEnvironment();
if (options.UseAutoPool.HasValue && options.UseAutoPool.Value)
{
IAutoPoolSpecification autoPoolSpecification = new AutoPoolSpecification()
{
AutoPoolNamePrefix = options.AutoPoolPrefix,
KeepAlive = options.KeepAlive,
PoolLifeTimeOption = options.LifeTimeOption.Equals("Job", StringComparison.OrdinalIgnoreCase) ? PoolLifeTimeOption.Job : PoolLifeTimeOption.WorkItem
};
jobExecutionEnvironment.AutoPoolSpecification = autoPoolSpecification;
}
else
{
jobExecutionEnvironment.PoolName = options.PoolName;
}
unboundWorkItem.JobExecutionEnvironment = jobExecutionEnvironment;
unboundWorkItem.JobSpecification = new JobSpecification()
{
Priority = options.Priority
};
// TODO: These are read only
unboundWorkItem.JobSpecification.JobConstraints = new JobConstraints(options.MaxWallClockTime, options.MaxRetryCount);
if (options.CreateSchedule.HasValue && options.CreateSchedule.Value == true)
{
IWorkItemSchedule schedule = new WorkItemSchedule()
{
DoNotRunAfter = options.DoNotRunAfter,
DoNotRunUntil = options.DoNotRunUntil,
RecurrenceInterval = options.RecurrenceInterval,
StartWindow = options.StartWindow
};
unboundWorkItem.Schedule = schedule;
}
if (options.CreateJobManager.HasValue && options.CreateJobManager.Value == true)
{
IJobManager jobManager = new JobManager()
{
CommandLine = options.CommandLine,
KillJobOnCompletion = options.KillOnCompletion,
Name = options.JobManagerName
};
jobManager.TaskConstraints = new TaskConstraints(options.MaxTaskWallClockTime, options.RetentionTime, options.MaxTaskRetryCount);
unboundWorkItem.JobSpecification.JobManager = jobManager;
}
await unboundWorkItem.CommitAsync();
}
}
catch
{
throw;
}
}
/// <summary>
/// Populates Azure Storage with the required files, and
/// submits the job to the Azure Batch service.
/// </summary>
public async Task RunAsync()
{
Console.WriteLine("Running with the following settings: ");
Console.WriteLine("----------------------------------------");
Console.WriteLine(this.textSearchSettings.ToString());
Console.WriteLine(this.accountSettings.ToString());
CloudStorageAccount cloudStorageAccount = new CloudStorageAccount(
new StorageCredentials(
this.accountSettings.StorageAccountName,
this.accountSettings.StorageAccountKey),
this.accountSettings.StorageServiceUrl,
useHttps: true);
//Upload resources if required.
if (this.textSearchSettings.ShouldUploadResources)
{
Console.WriteLine("Splitting file: {0} into {1} subfiles",
Constants.TextFilePath,
this.textSearchSettings.NumberOfMapperTasks);
//Split the text file into the correct number of files for consumption by the mapper tasks.
FileSplitter splitter = new FileSplitter();
List <string> mapperTaskFiles = await splitter.SplitAsync(
Constants.TextFilePath,
this.textSearchSettings.NumberOfMapperTasks);
List <string> files = Constants.RequiredExecutableFiles.Union(mapperTaskFiles).ToList();
await SampleHelpers.UploadResourcesAsync(
cloudStorageAccount,
this.textSearchSettings.BlobContainer,
files);
}
//Generate a SAS for the container.
string containerSasUrl = SampleHelpers.ConstructContainerSas(
cloudStorageAccount,
this.textSearchSettings.BlobContainer);
//Set up the Batch Service credentials used to authenticate with the Batch Service.
BatchSharedKeyCredentials credentials = new BatchSharedKeyCredentials(
this.accountSettings.BatchServiceUrl,
this.accountSettings.BatchAccountName,
this.accountSettings.BatchAccountKey);
using (BatchClient batchClient = await BatchClient.OpenAsync(credentials))
{
//
// Construct the job properties in local memory before commiting them to the Batch Service.
//
//Allow enough compute nodes in the pool to run each mapper task, and 1 extra to run the job manager.
int numberOfPoolComputeNodes = 1 + this.textSearchSettings.NumberOfMapperTasks;
//Define the pool specification for the pool which the job will run on.
PoolSpecification poolSpecification = new PoolSpecification()
{
TargetDedicated = numberOfPoolComputeNodes,
VirtualMachineSize = "small",
//You can learn more about os families and versions at:
//http://azure.microsoft.com/documentation/articles/cloud-services-guestos-update-matrix
OSFamily = "4",
TargetOSVersion = "*"
};
//Use the auto pool feature of the Batch Service to create a pool when the job is created.
//This creates a new pool for each job which is added.
AutoPoolSpecification autoPoolSpecification = new AutoPoolSpecification()
{
AutoPoolIdPrefix = "TextSearchPool",
KeepAlive = false,
PoolLifetimeOption = PoolLifetimeOption.Job,
PoolSpecification = poolSpecification
};
//Define the pool information for this job -- it will run on the pool defined by the auto pool specification above.
PoolInformation poolInformation = new PoolInformation()
{
AutoPoolSpecification = autoPoolSpecification
};
//Define the job manager for this job. This job manager will run first and will submit the tasks for
//the job. The job manager is the executable which manages the lifetime of the job
//and all tasks which should run for the job. In this case, the job manager submits the mapper and reducer tasks.
List <ResourceFile> jobManagerResourceFiles = SampleHelpers.GetResourceFiles(containerSasUrl, Constants.RequiredExecutableFiles);
const string jobManagerTaskId = "JobManager";
JobManagerTask jobManagerTask = new JobManagerTask()
{
ResourceFiles = jobManagerResourceFiles,
CommandLine = Constants.JobManagerExecutable,
//Determines if the job should terminate when the job manager process exits.
KillJobOnCompletion = true,
Id = jobManagerTaskId
};
//Create the unbound job in local memory. An object which exists only in local memory (and not on the Batch Service) is "unbound".
string jobId = Environment.GetEnvironmentVariable("USERNAME") + DateTime.UtcNow.ToString("yyyyMMdd-HHmmss");
CloudJob unboundJob = batchClient.JobOperations.CreateJob(jobId, poolInformation);
unboundJob.JobManagerTask = jobManagerTask; //Assign the job manager task to this job
try
{
//Commit the unbound job to the Batch Service.
Console.WriteLine("Adding job: {0} to the Batch Service.", unboundJob.Id);
await unboundJob.CommitAsync(); //Issues a request to the Batch Service to add the job which was defined above.
//
// Wait for the job manager task to complete.
//
//An object which is backed by a corresponding Batch Service object is "bound."
CloudJob boundJob = await batchClient.JobOperations.GetJobAsync(jobId);
CloudTask boundJobManagerTask = await boundJob.GetTaskAsync(jobManagerTaskId);
TimeSpan maxJobCompletionTimeout = TimeSpan.FromMinutes(30);
// Monitor the current tasks to see when they are done.
// Occasionally a task may get killed and requeued during an upgrade or hardware failure, including the job manager
// task. The job manager will be re-run in this case. Robustness against this was not added into the sample for
// simplicity, but should be added into any production code.
Console.WriteLine("Waiting for job's tasks to complete");
TaskStateMonitor taskStateMonitor = batchClient.Utilities.CreateTaskStateMonitor();
bool timedOut = await taskStateMonitor.WaitAllAsync(new List <CloudTask> {
boundJobManagerTask
}, TaskState.Completed, maxJobCompletionTimeout);
Console.WriteLine("Done waiting for job manager task.");
await boundJobManagerTask.RefreshAsync();
//Check to ensure the job manager task exited successfully.
await Helpers.CheckForTaskSuccessAsync(boundJobManagerTask, dumpStandardOutOnTaskSuccess : false);
if (timedOut)
{
throw new TimeoutException(string.Format("Timed out waiting for job manager task to complete."));
}
//
// Download and write out the reducer tasks output
//
string reducerText = await SampleHelpers.DownloadBlobTextAsync(cloudStorageAccount, this.textSearchSettings.BlobContainer, Constants.ReducerTaskResultBlobName);
Console.WriteLine("Reducer reuslts:");
Console.WriteLine(reducerText);
}
finally
{
//Delete the job.
//This will delete the auto pool associated with the job as long as the pool
//keep alive property is set to false.
if (this.textSearchSettings.ShouldDeleteJob)
{
Console.WriteLine("Deleting job {0}", jobId);
batchClient.JobOperations.DeleteJob(jobId);
}
//Note that there were files uploaded to a container specified in the
//configuration file. This container will not be deleted or cleaned up by this sample.
}
}
}
public void SampleCreateJobScheduleAutoPool()
{
Action test = () =>
{
using (BatchClient batchCli = TestUtilities.OpenBatchClientAsync(TestUtilities.GetCredentialsFromEnvironment()).Result)
{
string jsId = Microsoft.Azure.Batch.Constants.DefaultConveniencePrefix + TestUtilities.GetMyName() + "-CreateWiAutoPoolTest";
try
{
CloudJobSchedule newJobSchedule = batchCli.JobScheduleOperations.CreateJobSchedule(jsId, null, null);
{
newJobSchedule.Metadata = MakeMetaData("onCreateName", "onCreateValue");
PoolInformation poolInformation = new PoolInformation();
AutoPoolSpecification iaps = new AutoPoolSpecification();
Schedule schedule = new Schedule()
{
RecurrenceInterval = TimeSpan.FromMinutes(18)
};
poolInformation.AutoPoolSpecification = iaps;
iaps.AutoPoolIdPrefix = Microsoft.Azure.Batch.Constants.DefaultConveniencePrefix + TestUtilities.GetMyName();
iaps.PoolLifetimeOption = Microsoft.Azure.Batch.Common.PoolLifetimeOption.Job;
iaps.KeepAlive = false;
PoolSpecification ps = new PoolSpecification();
iaps.PoolSpecification = ps;
ps.TargetDedicated = 1;
ps.VirtualMachineSize = PoolFixture.VMSize;
ps.CloudServiceConfiguration = new CloudServiceConfiguration(PoolFixture.OSFamily);
ps.Metadata = MakeMetaData("pusMDIName", "pusMDIValue");
JobSpecification jobSpec = newJobSchedule.JobSpecification;
Assert.Null(jobSpec);
jobSpec = new JobSpecification(poolInformation);
JobManagerTask jobMgr = jobSpec.JobManagerTask;
Assert.Null(jobMgr);
jobMgr = new JobManagerTask(TestUtilities.GetMyName() + "-JobManagerTest", "hostname");
jobMgr.KillJobOnCompletion = false;
// set the JobManagerTask on the JobSpecification
jobSpec.JobManagerTask = jobMgr;
// set the JobSpecifcation on the Job Schedule
newJobSchedule.JobSpecification = jobSpec;
newJobSchedule.Schedule = schedule;
newJobSchedule.Commit();
}
CloudJobSchedule jobSchedule = batchCli.JobScheduleOperations.GetJobSchedule(jsId);
{
TestUtilities.DisplayJobScheduleLong(this.testOutputHelper, jobSchedule);
List <MetadataItem> mdi = new List <MetadataItem>(jobSchedule.Metadata);
// check the values specified for AddJobSchedule are correct.
foreach (MetadataItem curIMDI in mdi)
{
Assert.Equal("onCreateName", curIMDI.Name);
Assert.Equal("onCreateValue", curIMDI.Value);
}
// add metadata items
mdi.Add(new MetadataItem("modifiedName", "modifiedValue"));
jobSchedule.Metadata = mdi;
jobSchedule.Commit();
// confirm metadata updated correctly
CloudJobSchedule jsUpdated = batchCli.JobScheduleOperations.GetJobSchedule(jsId);
{
List <MetadataItem> updatedMDI = new List <MetadataItem>(jsUpdated.Metadata);
Assert.Equal(2, updatedMDI.Count);
Assert.Equal("onCreateName", updatedMDI[0].Name);
Assert.Equal("onCreateValue", updatedMDI[0].Value);
Assert.Equal("modifiedName", updatedMDI[1].Name);
Assert.Equal("modifiedValue", updatedMDI[1].Value);
}
jobSchedule.Refresh();
TestUtilities.DisplayJobScheduleLong(this.testOutputHelper, jobSchedule);
}
}
finally
{
// clean up
TestUtilities.DeleteJobScheduleIfExistsAsync(batchCli, jsId).Wait();
}
}
};
SynchronizationContextHelper.RunTest(test, TestTimeout);
}
public async Task CreateJobAsync(CreateJobOptions createJobOptions)
{
CloudJob unboundJob = this.Client.JobOperations.CreateJob();
unboundJob.Id = createJobOptions.JobId;
unboundJob.Priority = createJobOptions.Priority;
unboundJob.Constraints = new JobConstraints(createJobOptions.MaxWallClockTime, createJobOptions.MaxRetryCount);
PoolInformation poolInformation = new PoolInformation();
if (createJobOptions.AutoPoolOptions.UseAutoPool.HasValue && createJobOptions.AutoPoolOptions.UseAutoPool.Value)
{
AutoPoolSpecification autoPoolSpecification = new AutoPoolSpecification()
{
AutoPoolIdPrefix = createJobOptions.AutoPoolOptions.AutoPoolPrefix,
KeepAlive = createJobOptions.AutoPoolOptions.KeepAlive,
PoolLifetimeOption = (PoolLifetimeOption)Enum.Parse(typeof(PoolLifetimeOption), createJobOptions.AutoPoolOptions.LifeTimeOption),
PoolSpecification = new PoolSpecification()
{
OSFamily = createJobOptions.AutoPoolOptions.OSFamily,
VirtualMachineSize = createJobOptions.AutoPoolOptions.VirutalMachineSize,
TargetDedicated = createJobOptions.AutoPoolOptions.TargetDedicated
}
};
poolInformation.AutoPoolSpecification = autoPoolSpecification;
}
else
{
poolInformation.PoolId = createJobOptions.PoolId;
}
unboundJob.PoolInformation = poolInformation;
if (createJobOptions.CreateJobManager.HasValue && createJobOptions.CreateJobManager.Value == true)
{
JobManagerTask jobManager = new JobManagerTask()
{
CommandLine = createJobOptions.JobManagerOptions.CommandLine,
KillJobOnCompletion = createJobOptions.JobManagerOptions.KillOnCompletion,
Id = createJobOptions.JobManagerOptions.JobManagerId
};
jobManager.Constraints = new TaskConstraints(
createJobOptions.JobManagerOptions.MaxTaskWallClockTime,
createJobOptions.JobManagerOptions.RetentionTime,
createJobOptions.JobManagerOptions.MaxTaskRetryCount);
unboundJob.JobManagerTask = jobManager;
}
await unboundJob.CommitAsync();
}
public void Bug1433008JobScheduleScheduleNewable()
{
Action test = () =>
{
using (BatchClient batchCli = TestUtilities.OpenBatchClientAsync(TestUtilities.GetCredentialsFromEnvironment()).Result)
{
string jsId = Microsoft.Azure.Batch.Constants.DefaultConveniencePrefix + TestUtilities.GetMyName() + "-Bug1433008JobScheduleScheduleNewable";
try
{
DateTime unboundDNRU = DateTime.UtcNow.AddYears(1);
CloudJobSchedule newJobSchedule = batchCli.JobScheduleOperations.CreateJobSchedule(jsId, null, null);
{
AutoPoolSpecification iaps = new AutoPoolSpecification();
PoolSpecification ips = new PoolSpecification();
JobSpecification jobSpecification = new JobSpecification(new PoolInformation()
{
AutoPoolSpecification = iaps
});
iaps.PoolSpecification = ips;
iaps.AutoPoolIdPrefix = Microsoft.Azure.Batch.Constants.DefaultConveniencePrefix + TestUtilities.GetMyName();
iaps.PoolLifetimeOption = Microsoft.Azure.Batch.Common.PoolLifetimeOption.Job;
iaps.KeepAlive = false;
PoolSpecification ps = iaps.PoolSpecification;
ps.TargetDedicated = 1;
ps.VirtualMachineSize = PoolFixture.VMSize;
ps.CloudServiceConfiguration = new CloudServiceConfiguration(PoolFixture.OSFamily);
Schedule sched = new Schedule();
sched.DoNotRunUntil = unboundDNRU;
newJobSchedule.Schedule = sched;
newJobSchedule.JobSpecification = jobSpecification;
newJobSchedule.Commit();
}
CloudJobSchedule jobSchedule = batchCli.JobScheduleOperations.GetJobSchedule(jsId);
// confirm that the original value(s) are set
TestUtilities.DisplayJobScheduleLong(this.testOutputHelper, jobSchedule);
Assert.Equal(unboundDNRU, jobSchedule.Schedule.DoNotRunUntil);
// now update the schedule and confirm
DateTime boundDNRU = DateTime.UtcNow.AddYears(2);
jobSchedule.Schedule.DoNotRunUntil = boundDNRU;
jobSchedule.Commit();
jobSchedule.Refresh();
// confirm that the new value(s) are set
TestUtilities.DisplayJobScheduleLong(this.testOutputHelper, jobSchedule);
Assert.Equal(boundDNRU, jobSchedule.Schedule.DoNotRunUntil);
}
finally
{
// clean up
TestUtilities.DeleteJobScheduleIfExistsAsync(batchCli, jsId).Wait();
}
}
};
SynchronizationContextHelper.RunTest(test, TestTimeout);
}
/// <summary>
/// Creates a job schedule with the specified options.
/// </summary>
/// <param name="options">The options describing the job schedule to create.</param>
/// <returns></returns>
public async Task CreateJobScheduleAsync(CreateJobScheduleOptions options)
{
CloudJobSchedule unboundJobSchedule = this.Client.JobScheduleOperations.CreateJobSchedule();
unboundJobSchedule.Id = options.JobScheduleId;
PoolInformation poolInformation = new PoolInformation();
if (options.AutoPoolOptions.UseAutoPool.HasValue && options.AutoPoolOptions.UseAutoPool.Value)
{
AutoPoolSpecification autoPoolSpecification = new AutoPoolSpecification()
{
AutoPoolIdPrefix = options.AutoPoolOptions.AutoPoolPrefix,
KeepAlive = options.AutoPoolOptions.KeepAlive,
PoolLifetimeOption = (PoolLifetimeOption)Enum.Parse(typeof(PoolLifetimeOption), options.AutoPoolOptions.LifeTimeOption),
PoolSpecification = new PoolSpecification()
{
CloudServiceConfiguration = new CloudServiceConfiguration(options.AutoPoolOptions.OSFamily),
VirtualMachineSize = options.AutoPoolOptions.VirutalMachineSize,
TargetDedicated = options.AutoPoolOptions.TargetDedicated
}
};
poolInformation.AutoPoolSpecification = autoPoolSpecification;
}
else
{
poolInformation.PoolId = options.PoolId;
}
unboundJobSchedule.JobSpecification = new JobSpecification()
{
Priority = options.Priority,
PoolInformation = poolInformation
};
// TODO: These are read only
unboundJobSchedule.JobSpecification.Constraints = new JobConstraints(options.MaxWallClockTime, options.MaxRetryCount);
Schedule schedule = new Schedule()
{
DoNotRunAfter = options.DoNotRunAfter,
DoNotRunUntil = options.DoNotRunUntil,
RecurrenceInterval = options.RecurrenceInterval,
StartWindow = options.StartWindow
};
unboundJobSchedule.Schedule = schedule;
if (options.CreateJobManager.HasValue && options.CreateJobManager.Value == true)
{
JobManagerTask jobManager = new JobManagerTask()
{
CommandLine = options.JobManagerOptions.CommandLine,
KillJobOnCompletion = options.JobManagerOptions.KillOnCompletion,
Id = options.JobManagerOptions.JobManagerId
};
jobManager.Constraints = new TaskConstraints(options.JobManagerOptions.MaxTaskWallClockTime, options.JobManagerOptions.RetentionTime, options.JobManagerOptions.MaxTaskRetryCount);
unboundJobSchedule.JobSpecification.JobManagerTask = jobManager;
}
await unboundJobSchedule.CommitAsync();
}
/// <summary>
/// Creates a work item with the specified work item options.
/// </summary>
/// <param name="options">The options describing the work item to create.</param>
/// <returns></returns>
public async Task CreateWorkItemAsync(CreateWorkItemOptions options)
{
try
{
using (IWorkItemManager workItemManager = this.Client.OpenWorkItemManager())
{
ICloudWorkItem unboundWorkItem = workItemManager.CreateWorkItem(options.WorkItemName);
IJobExecutionEnvironment jobExecutionEnvironment = new JobExecutionEnvironment();
if (options.UseAutoPool.HasValue && options.UseAutoPool.Value)
{
IAutoPoolSpecification autoPoolSpecification = new AutoPoolSpecification()
{
AutoPoolNamePrefix = options.AutoPoolPrefix,
KeepAlive = options.KeepAlive,
PoolLifeTimeOption = options.LifeTimeOption.Equals("Job", StringComparison.OrdinalIgnoreCase) ? PoolLifeTimeOption.Job : PoolLifeTimeOption.WorkItem
};
jobExecutionEnvironment.AutoPoolSpecification = autoPoolSpecification;
}
else
{
jobExecutionEnvironment.PoolName = options.PoolName;
}
unboundWorkItem.JobExecutionEnvironment = jobExecutionEnvironment;
unboundWorkItem.JobSpecification = new JobSpecification()
{
Priority = options.Priority
};
// TODO: These are read only
unboundWorkItem.JobSpecification.JobConstraints = new JobConstraints(options.MaxWallClockTime, options.MaxRetryCount);
if (options.CreateSchedule.HasValue && options.CreateSchedule.Value == true)
{
IWorkItemSchedule schedule = new WorkItemSchedule()
{
DoNotRunAfter = options.DoNotRunAfter,
DoNotRunUntil = options.DoNotRunUntil,
RecurrenceInterval = options.RecurrenceInterval,
StartWindow = options.StartWindow
};
unboundWorkItem.Schedule = schedule;
}
if (options.CreateJobManager.HasValue && options.CreateJobManager.Value == true)
{
IJobManager jobManager = new JobManager()
{
CommandLine = options.CommandLine,
KillJobOnCompletion = options.KillOnCompletion,
Name = options.JobManagerName
};
jobManager.TaskConstraints = new TaskConstraints(options.MaxTaskWallClockTime, options.RetentionTime, options.MaxTaskRetryCount);
unboundWorkItem.JobSpecification.JobManager = jobManager;
}
await unboundWorkItem.CommitAsync();
}
}
catch
{
throw;
}
}