/// <summary>
/// Create a WorkItemModel backed by an ICloudWorkItem
/// </summary>
public WorkItemModel(ICloudWorkItem workItem)
{
this.WorkItem = workItem;
this.LastUpdatedTime = DateTime.UtcNow;
this.Jobs = new List <JobModel>();
smallestJobId = 0;
}
/// <summary>
/// Builds a PSCloudWorkItem for testing
/// </summary>
public static PSCloudWorkItem CreatePSCloudWorkItem()
{
BatchAccountContext context = CreateBatchContextWithKeys();
using (IWorkItemManager wiManager = context.BatchOMClient.OpenWorkItemManager())
{
ICloudWorkItem workItem = wiManager.CreateWorkItem("testWorkItem");
return(new PSCloudWorkItem(workItem));
}
}
public WorkItemModel(ICloudWorkItem workItem, ICloudJob mostRecentJob)
{
this.WorkItem = workItem;
this.LastUpdatedTime = DateTime.UtcNow;
// create a new job list and add the most recent job to the list
JobModel recentJob = new JobModel(this, mostRecentJob);
this.Jobs = new List <JobModel> {
recentJob
};
smallestJobId = WorkItemModel.GetJobNumberFromJobName(recentJob.Name);
}
/// <summary>
/// Wait for an active job to be created.
/// </summary>
/// <param name="boundWorkItem">The work item to monitor for job creation.</param>
/// <returns>The name of the job.</returns>
public static async Task <string> WaitForActiveJobAsync(ICloudWorkItem boundWorkItem)
{
//Wait for job to be created
while (boundWorkItem.ExecutionInformation == null ||
boundWorkItem.ExecutionInformation.RecentJob == null)
{
await Task.Delay(TimeSpan.FromSeconds(5));
Console.WriteLine("Getting bound work item {0}", boundWorkItem.Name);
await boundWorkItem.RefreshAsync();
}
//Get the job which is running - once this job is complete our work is done
string boundJobName = boundWorkItem.ExecutionInformation.RecentJob.Name;
return(boundJobName);
}
/// <summary>
/// Creates a new WorkItem
/// </summary>
/// <param name="parameters">The parameters to use when creating the WorkItem</param>
public void CreateWorkItem(NewWorkItemParameters parameters)
{
if (parameters == null)
{
throw new ArgumentNullException("parameters");
}
if (string.IsNullOrWhiteSpace(parameters.WorkItemName))
{
throw new ArgumentNullException("WorkItemName");
}
using (IWorkItemManager wiManager = parameters.Context.BatchOMClient.OpenWorkItemManager())
{
ICloudWorkItem workItem = wiManager.CreateWorkItem(parameters.WorkItemName);
if (parameters.Schedule != null)
{
workItem.Schedule = parameters.Schedule.omObject;
}
if (parameters.JobSpecification != null)
{
Utils.Utils.JobSpecificationSyncCollections(parameters.JobSpecification);
workItem.JobSpecification = parameters.JobSpecification.omObject;
}
if (parameters.JobExecutionEnvironment != null)
{
Utils.Utils.JobExecutionEnvironmentSyncCollections(parameters.JobExecutionEnvironment);
workItem.JobExecutionEnvironment = parameters.JobExecutionEnvironment.omObject;
}
if (parameters.Metadata != null)
{
workItem.Metadata = new List <IMetadataItem>();
foreach (DictionaryEntry d in parameters.Metadata)
{
MetadataItem metadata = new MetadataItem(d.Key.ToString(), d.Value.ToString());
workItem.Metadata.Add(metadata);
}
}
WriteVerbose(string.Format(Resources.NBWI_CreatingWorkItem, parameters.WorkItemName));
workItem.Commit(parameters.AdditionalBehaviors);
}
}
/// <summary>
/// Creates a test WorkItem for use in Scenario tests.
/// TODO: Replace with new WorkItem client method when it exists.
/// </summary>
public static void CreateTestWorkItem(BatchAccountContext context, string workItemName, TimeSpan?recurrenceInterval)
{
if (HttpMockServer.Mode == HttpRecorderMode.Record)
{
using (IWorkItemManager wiManager = context.BatchOMClient.OpenWorkItemManager())
{
ICloudWorkItem workItem = wiManager.CreateWorkItem(workItemName);
workItem.JobExecutionEnvironment = new Azure.Batch.JobExecutionEnvironment();
workItem.JobExecutionEnvironment.PoolName = "testPool";
if (recurrenceInterval != null)
{
workItem.Schedule = new Azure.Batch.WorkItemSchedule();
workItem.Schedule.RecurrenceInterval = recurrenceInterval;
}
workItem.Commit();
}
}
}
/// <summary>
/// Lists the workitems matching the specified filter options
/// </summary>
/// <param name="options">The options to use when querying for workitems</param>
/// <returns>The workitems matching the specified filter options</returns>
public IEnumerable <PSCloudWorkItem> ListWorkItems(ListWorkItemOptions options)
{
if (options == null)
{
throw new ArgumentNullException("options");
}
// Get the single WorkItem matching the specified name
if (!string.IsNullOrWhiteSpace(options.WorkItemName))
{
WriteVerbose(string.Format(Resources.GBWI_GetByName, options.WorkItemName));
using (IWorkItemManager wiManager = options.Context.BatchOMClient.OpenWorkItemManager())
{
ICloudWorkItem workItem = wiManager.GetWorkItem(options.WorkItemName, additionalBehaviors: options.AdditionalBehaviors);
PSCloudWorkItem psWorkItem = new PSCloudWorkItem(workItem);
return(new PSCloudWorkItem[] { psWorkItem });
}
}
// List WorkItems using the specified filter
else
{
ODATADetailLevel odata = null;
string verboseLogString = null;
if (!string.IsNullOrEmpty(options.Filter))
{
verboseLogString = Resources.GBWI_GetByOData;
odata = new ODATADetailLevel(filterClause: options.Filter);
}
else
{
verboseLogString = Resources.GBWI_NoFilter;
}
WriteVerbose(verboseLogString);
using (IWorkItemManager wiManager = options.Context.BatchOMClient.OpenWorkItemManager())
{
IEnumerableAsyncExtended <ICloudWorkItem> workItems = wiManager.ListWorkItems(odata, options.AdditionalBehaviors);
Func <ICloudWorkItem, PSCloudWorkItem> mappingFunction = w => { return(new PSCloudWorkItem(w)); };
return(PSAsyncEnumerable <PSCloudWorkItem, ICloudWorkItem> .CreateWithMaxCount(
workItems, mappingFunction, options.MaxCount, () => WriteVerbose(string.Format(Resources.MaxCount, options.MaxCount))));
}
}
}
/// <summary>
/// Populates Azure Storage with the required files, and
/// submits the work item to the Azure Batch service.
/// </summary>
public async Task RunAsync()
{
Console.WriteLine("Running with the following settings: ");
Console.WriteLine("----------------------------------------");
Console.WriteLine(this.configurationSettings.ToString());
//Upload resources if required.
if (this.configurationSettings.ShouldUploadResources)
{
Console.WriteLine("Splitting file: {0} into {1} subfiles",
Constants.TextFilePath,
this.configurationSettings.NumberOfMapperTasks);
//Split the text file into the correct number of files for consumption by the mapper tasks.
FileSplitter splitter = new FileSplitter();
List <string> mapperTaskFiles = await splitter.SplitAsync(
Constants.TextFilePath,
this.configurationSettings.NumberOfMapperTasks);
await this.UploadResourcesAsync(mapperTaskFiles);
}
//Generate a SAS for the container.
string containerSasUrl = Helpers.ConstructContainerSas(
this.configurationSettings.StorageAccountName,
this.configurationSettings.StorageAccountKey,
this.configurationSettings.StorageServiceUrl,
this.configurationSettings.BlobContainer);
//Set up the Batch Service credentials used to authenticate with the Batch Service.
BatchCredentials batchCredentials = new BatchCredentials(
this.configurationSettings.BatchAccountName,
this.configurationSettings.BatchAccountKey);
using (IBatchClient batchClient = BatchClient.Connect(this.configurationSettings.BatchServiceUrl, batchCredentials))
{
using (IWorkItemManager workItemManager = batchClient.OpenWorkItemManager())
{
//Create the unbound work item in local memory. An object which exists only in local memory (and not on the Batch Service) is "unbound".
string workItemName = Environment.GetEnvironmentVariable("USERNAME") + DateTime.UtcNow.ToString("yyyyMMdd-HHmmss");
ICloudWorkItem unboundWorkItem = workItemManager.CreateWorkItem(workItemName);
//
// Construct the work item properties in local memory before commiting them to the Batch Service.
//
//Allow enough VMs in the pool to run each mapper task, and 1 extra to run the job manager.
int numberOfPoolVMs = 1 + this.configurationSettings.NumberOfMapperTasks;
//Define the pool specification for the pool which the work item will run on.
IPoolSpecification poolSpecification = new PoolSpecification()
{
TargetDedicated = numberOfPoolVMs,
VMSize = "small",
//You can learn more about os families and versions at:
//http://msdn.microsoft.com/en-us/library/azure/ee924680.aspx
OSFamily = "4",
TargetOSVersion = "*"
};
//Use the auto pool feature of the Batch Service to create a pool when the work item is created.
//This creates a new pool for each work item which is added.
IAutoPoolSpecification autoPoolSpecification = new AutoPoolSpecification()
{
AutoPoolNamePrefix = "TextSearchPool",
KeepAlive = false,
PoolLifeTimeOption = PoolLifeTimeOption.WorkItem,
PoolSpecification = poolSpecification
};
//Define the execution environment for this work item -- it will run on the pool defined by the auto pool specification above.
unboundWorkItem.JobExecutionEnvironment = new JobExecutionEnvironment()
{
AutoPoolSpecification = autoPoolSpecification
};
//Define the job manager for this work item. This job manager will run when any job is created and will submit the tasks for
//the work item. The job manager is the executable which manages the lifetime of the job
//and all tasks which should run for the job. In this case, the job manager submits the mapper and reducer tasks.
string jobManagerCommandLine = string.Format("{0} -JobManagerTask", Constants.TextSearchExe);
List <IResourceFile> jobManagerResourceFiles = Helpers.GetResourceFiles(containerSasUrl, Constants.RequiredExecutableFiles);
const string jobManagerTaskName = "JobManager";
unboundWorkItem.JobSpecification = new JobSpecification()
{
JobManager = new JobManager()
{
ResourceFiles = jobManagerResourceFiles,
CommandLine = jobManagerCommandLine,
//Determines if the job should terminate when the job manager process exits
KillJobOnCompletion = false,
Name = jobManagerTaskName
}
};
try
{
//Commit the unbound work item to the Batch Service.
Console.WriteLine("Adding work item: {0} to the Batch Service.", unboundWorkItem.Name);
await unboundWorkItem.CommitAsync(); //Issues a request to the Batch Service to add the work item which was defined above.
//
// Wait for the job manager task to complete.
//
//An object which is backed by a corresponding Batch Service object is "bound."
ICloudWorkItem boundWorkItem = await workItemManager.GetWorkItemAsync(workItemName);
//Wait for the job to be created automatically by the Batch Service.
string boundJobName = await Helpers.WaitForActiveJobAsync(boundWorkItem);
ICloudTask boundJobManagerTask = await workItemManager.GetTaskAsync(
workItemName,
boundJobName,
jobManagerTaskName);
TimeSpan maxJobCompletionTimeout = TimeSpan.FromMinutes(30);
IToolbox toolbox = batchClient.OpenToolbox();
ITaskStateMonitor monitor = toolbox.CreateTaskStateMonitor();
bool timedOut = await monitor.WaitAllAsync(new List <ICloudTask> {
boundJobManagerTask
}, TaskState.Completed, maxJobCompletionTimeout);
Console.WriteLine("Done waiting for job manager task.");
await boundJobManagerTask.RefreshAsync();
//Check to ensure the job manager task exited successfully.
await Helpers.CheckForTaskSuccessAsync(boundJobManagerTask, dumpStandardOutOnTaskSuccess : true);
if (timedOut)
{
throw new TimeoutException(string.Format("Timed out waiting for job manager task to complete."));
}
}
catch (AggregateException e)
{
e.Handle(
(innerE) =>
{
//We print all the inner exceptions for debugging purposes.
Console.WriteLine(innerE.ToString());
return(false);
});
throw;
}
catch (Exception e)
{
Console.WriteLine("Hit unexpected exception: {0}", e.ToString());
throw;
}
finally
{
//Delete the work item.
//This will delete the auto pool associated with the work item as long as the pool
//keep alive property is set to false.
if (this.configurationSettings.ShouldDeleteWorkItem)
{
Console.WriteLine("Deleting work item {0}", workItemName);
workItemManager.DeleteWorkItem(workItemName);
}
//Note that there were files uploaded to a container specified in the
//configuration file. This container will not be deleted or cleaned up by this sample.
}
}
}
}
/// <summary>
/// Wait for an active job to be created.
/// </summary>
/// <param name="boundWorkItem">The work item to monitor for job creation.</param>
/// <returns>The name of the job.</returns>
public static async Task<string> WaitForActiveJobAsync(ICloudWorkItem boundWorkItem)
{
//Wait for job to be created
while (boundWorkItem.ExecutionInformation == null ||
boundWorkItem.ExecutionInformation.RecentJob == null)
{
await Task.Delay(TimeSpan.FromSeconds(5));
Console.WriteLine("Getting bound work item {0}", boundWorkItem.Name);
await boundWorkItem.RefreshAsync();
}
//Get the job which is running - once this job is complete our work is done
string boundJobName = boundWorkItem.ExecutionInformation.RecentJob.Name;
return boundJobName;
}
public WorkItemModel(ICloudWorkItem workItem, ICloudJob mostRecentJob)
{
this.WorkItem = workItem;
this.LastUpdatedTime = DateTime.UtcNow;
// create a new job list and add the most recent job to the list
JobModel recentJob = new JobModel(this, mostRecentJob);
this.Jobs = new List<JobModel> { recentJob };
smallestJobId = WorkItemModel.GetJobNumberFromJobName(recentJob.Name);
}
/// <summary>
/// Create a WorkItemModel backed by an ICloudWorkItem
/// </summary>
public WorkItemModel(ICloudWorkItem workItem)
{
this.WorkItem = workItem;
this.LastUpdatedTime = DateTime.UtcNow;
this.Jobs = new List<JobModel>();
smallestJobId = 0;
}
/// <summary>
/// This will take the basic data provided about the account, upload the necessary information to the account, and schedule a job.
/// </summary>
/// <param name="sender"></param>
/// <param name="e"></param>
private void btnStart_Click(object sender, RoutedEventArgs e)
{
BatchCredentials credentials = new BatchCredentials(txtBAccountName.Text, txtBAccountKey.Text);
IBatchClient bClient = BatchClient.Connect(SampleConstants.BatchSvcEndpoint, credentials);
// Setting a retry policy adds robustness against an individual call timing out. When using a policy, by default all recoverable failures are retried.
bClient.CustomBehaviors.Add(new SetRetryPolicy(new ExponentialRetry(TimeSpan.FromSeconds(5), 5)));
// Create a unique workitem name; don't forget to delete these when you're done
string workItemName = SampleConstants.WorkItemNamePrefix + Guid.NewGuid().ToString();
// Identify the pre-existing pool of VMs that will run the tasks. An Autopool specification
// is fine but there is the delay associated with the creation of the pool along with waiting
// for the VMs to reach Idle state before tasks are running. You can use Batch Explorer to
// pre-create the pool and then resize it to the desired size and number of VMs.
JobExecutionEnvironment jee = new JobExecutionEnvironment()
{
PoolName = PoolName
};
// Next, create the JobManager instance describing the environment settings and resources it
// needs to run
JobManager jobMgr = new JobManager()
{
Name = "JM1",
CommandLine = SampleConstants.JobManager,
// NOTE: We do not in general recommend that customers put their secrets on the command line or as environmental variables, as
// these are not a secure locations. This was done for the simplicity of the sample.
EnvironmentSettings = new List <IEnvironmentSetting>()
{
{ new EnvironmentSetting(SampleConstants.EnvWorkItemName, workItemName) },
{ new EnvironmentSetting(SampleConstants.EnvBatchAccountKeyName, txtBAccountKey.Text) }
},
// In many cases you will want KillJobOnCompletion to be set to 'TRUE' - this allows the previous job to finish before
// a recurrence is scheduled. As an alternative, you can set this to 'FALSE' and use MaxWallClockTime as shown below,
// which will instead ensure that every recurrence happens.
KillJobOnCompletion = true
};
// Create a list of resource files that are needed to run JobManager.exe. A shared access signature key specifying
// readonly access is used so the JobManager program will have access to the resource files when it is started
// on a VM.
var sasPrefix = Helpers.ConstructContainerSas(
txtSAccountName.Text,
txtSAccountKey.Text,
"core.windows.net",
txtSContainerName.Text);
jobMgr.ResourceFiles = Helpers.GetResourceFiles(sasPrefix, SampleConstants.JobManagerFiles);
// Create the job specification, identifying that this job has a job manager associated with it
JobSpecification jobSpec = new JobSpecification()
{
JobManager = jobMgr
};
// Set up the desired recurrence or start time schedule.
WorkItemSchedule wiSchedule = new WorkItemSchedule();
if (rdoOnce.IsChecked == true)
{
// Set information if the task is to be run once.
DateTime runOnce = (DateTime)(dpkDate.SelectedDate);
runOnce = runOnce.AddHours(cbxHourO.SelectedIndex);
runOnce = runOnce.AddMinutes(cbxMinuteO.SelectedIndex);
wiSchedule.DoNotRunUntil = runOnce;
}
else
{
// Set information if the task is to be recurring.
TimeSpan recurring = new TimeSpan(cbxHourR.SelectedIndex, cbxMinuteR.SelectedIndex, 0);
wiSchedule.RecurrenceInterval = recurring;
TimeSpan countback = new TimeSpan(0, 0, 30);
jobSpec.JobConstraints = new JobConstraints(recurring.Subtract(countback), null);
}
// Upload files and create workitem.
UploadFiles();
using (IWorkItemManager wiMgr = bClient.OpenWorkItemManager())
{
ICloudWorkItem workItem = wiMgr.CreateWorkItem(workItemName);
workItem.JobExecutionEnvironment = jee;
workItem.Schedule = wiSchedule;
workItem.JobSpecification = jobSpec;
try
{
workItem.Commit();
}
catch (Exception ex)
{
MessageBox.Show(ex.ToString());
}
}
// Remember to clean up your workitems and jobs
}
/// <summary>
/// Submit a large number of tasks to the Batch Service.
/// </summary>
/// <param name="client">The batch client.</param>
private static void SubmitLargeNumberOfTasks(IBatchClient client)
{
const int taskCountToCreate = 5000;
// In order to simulate a "large" task object which has many properties set (such as resource files, environment variables, etc)
// we create a big environment variable so we have a big task object.
char[] env = new char[2048];
for (int i = 0; i < env.Length; i++)
{
env[i] = 'a';
}
string envStr = new string(env);
using (IWorkItemManager wm = client.OpenWorkItemManager())
{
//Create a work item
string workItemName = Environment.GetEnvironmentVariable("USERNAME") + DateTime.Now.ToString("yyyyMMdd-HHmmss");
Console.WriteLine("Creating work item {0}", workItemName);
ICloudWorkItem cloudWorkItem = wm.CreateWorkItem(workItemName);
cloudWorkItem.JobExecutionEnvironment = new JobExecutionEnvironment()
{
PoolName = PoolName
}; //Specify the pool to run on
cloudWorkItem.Commit();
//Wait for an active job
TimeSpan maxJobCreationTimeout = TimeSpan.FromSeconds(90);
DateTime jobCreationStartTime = DateTime.Now;
DateTime jobCreationTimeoutTime = jobCreationStartTime.Add(maxJobCreationTimeout);
cloudWorkItem = wm.GetWorkItem(workItemName);
Console.WriteLine("Waiting for a job to become active...");
while (cloudWorkItem.ExecutionInformation == null || cloudWorkItem.ExecutionInformation.RecentJob == null)
{
cloudWorkItem.Refresh();
if (DateTime.Now > jobCreationTimeoutTime)
{
throw new Exception("Timed out waiting for job.");
}
Thread.Sleep(TimeSpan.FromSeconds(5));
}
string jobName = cloudWorkItem.ExecutionInformation.RecentJob.Name;
Console.WriteLine("Found job {0}. Adding task objects...", jobName);
//Generate a large number of tasks to submit
List <ICloudTask> tasksToSubmit = new List <ICloudTask>();
for (int i = 0; i < taskCountToCreate; i++)
{
ICloudTask task = new CloudTask("echo" + i.ToString("D5"), "echo");
List <IEnvironmentSetting> environmentSettings = new List <IEnvironmentSetting>();
environmentSettings.Add(new EnvironmentSetting("envone", envStr));
task.EnvironmentSettings = environmentSettings;
tasksToSubmit.Add(task);
}
BatchClientParallelOptions parallelOptions = new BatchClientParallelOptions()
{
//This will result in at most 10 simultaneous Bulk Add requests to the Batch Service.
MaxDegreeOfParallelism = 10
};
Console.WriteLine("Submitting {0} tasks to work item: {1}, job: {2}, on pool: {3}",
taskCountToCreate,
cloudWorkItem.Name,
jobName,
cloudWorkItem.JobExecutionEnvironment.PoolName);
Stopwatch stopwatch = new Stopwatch();
stopwatch.Start();
//Use the AddTask overload which supports a list of tasks for best AddTask performence - internally this method performs an
//intelligent submission of tasks in batches in order to limit the number of REST API calls made to the Batch Service.
wm.AddTask(cloudWorkItem.Name, jobName, tasksToSubmit, parallelOptions);
stopwatch.Stop();
Console.WriteLine("Submitted {0} tasks in {1}", taskCountToCreate, stopwatch.Elapsed);
//Delete the work item to ensure the tasks are cleaned up
wm.DeleteWorkItem(workItemName);
}
}
/// <summary>
/// Creates a work item with the specified work item options.
/// </summary>
/// <param name="options">The options describing the work item to create.</param>
/// <returns></returns>
public async Task CreateWorkItemAsync(CreateWorkItemOptions options)
{
try
{
using (IWorkItemManager workItemManager = this.Client.OpenWorkItemManager())
{
ICloudWorkItem unboundWorkItem = workItemManager.CreateWorkItem(options.WorkItemName);
IJobExecutionEnvironment jobExecutionEnvironment = new JobExecutionEnvironment();
if (options.UseAutoPool.HasValue && options.UseAutoPool.Value)
{
IAutoPoolSpecification autoPoolSpecification = new AutoPoolSpecification()
{
AutoPoolNamePrefix = options.AutoPoolPrefix,
KeepAlive = options.KeepAlive,
PoolLifeTimeOption = options.LifeTimeOption.Equals("Job", StringComparison.OrdinalIgnoreCase) ? PoolLifeTimeOption.Job : PoolLifeTimeOption.WorkItem
};
jobExecutionEnvironment.AutoPoolSpecification = autoPoolSpecification;
}
else
{
jobExecutionEnvironment.PoolName = options.PoolName;
}
unboundWorkItem.JobExecutionEnvironment = jobExecutionEnvironment;
unboundWorkItem.JobSpecification = new JobSpecification()
{
Priority = options.Priority
};
// TODO: These are read only
unboundWorkItem.JobSpecification.JobConstraints = new JobConstraints(options.MaxWallClockTime, options.MaxRetryCount);
if (options.CreateSchedule.HasValue && options.CreateSchedule.Value == true)
{
IWorkItemSchedule schedule = new WorkItemSchedule()
{
DoNotRunAfter = options.DoNotRunAfter,
DoNotRunUntil = options.DoNotRunUntil,
RecurrenceInterval = options.RecurrenceInterval,
StartWindow = options.StartWindow
};
unboundWorkItem.Schedule = schedule;
}
if (options.CreateJobManager.HasValue && options.CreateJobManager.Value == true)
{
IJobManager jobManager = new JobManager()
{
CommandLine = options.CommandLine,
KillJobOnCompletion = options.KillOnCompletion,
Name = options.JobManagerName
};
jobManager.TaskConstraints = new TaskConstraints(options.MaxTaskWallClockTime, options.RetentionTime, options.MaxTaskRetryCount);
unboundWorkItem.JobSpecification.JobManager = jobManager;
}
await unboundWorkItem.CommitAsync();
}
}
catch
{
throw;
}
}
