/// <summary>
/// Lists the Tasks matching the specified filter options
/// </summary>
/// <param name="options">The options to use when querying for Tasks</param>
/// <returns>The Tasks matching the specified filter options</returns>
public IEnumerable <PSCloudTask> ListTasks(ListTaskOptions options)
{
if (options == null)
{
throw new ArgumentNullException("options");
}
if ((string.IsNullOrEmpty(options.WorkItemName) || string.IsNullOrEmpty(options.JobName)) && options.Job == null)
{
throw new ArgumentNullException(Resources.GBT_NoJob);
}
// Get the single Task matching the specified name
if (!string.IsNullOrEmpty(options.TaskName))
{
WriteVerbose(string.Format(Resources.GBT_GetByName, options.TaskName, options.JobName, options.WorkItemName));
using (IWorkItemManager wiManager = options.Context.BatchOMClient.OpenWorkItemManager())
{
ICloudTask task = wiManager.GetTask(options.WorkItemName, options.JobName, options.TaskName, additionalBehaviors: options.AdditionalBehaviors);
PSCloudTask psTask = new PSCloudTask(task);
return(new PSCloudTask[] { psTask });
}
}
// List Tasks using the specified filter
else
{
if (options.MaxCount <= 0)
{
options.MaxCount = Int32.MaxValue;
}
string jName = options.Job == null ? options.JobName : options.Job.Name;
ODATADetailLevel odata = null;
if (!string.IsNullOrEmpty(options.Filter))
{
WriteVerbose(string.Format(Resources.GBT_GetByOData, jName, options.MaxCount));
odata = new ODATADetailLevel(filterClause: options.Filter);
}
else
{
WriteVerbose(string.Format(Resources.GBT_GetNoFilter, jName, options.MaxCount));
}
IEnumerableAsyncExtended <ICloudTask> tasks = null;
if (options.Job != null)
{
tasks = options.Job.omObject.ListTasks(odata, options.AdditionalBehaviors);
}
else
{
using (IWorkItemManager wiManager = options.Context.BatchOMClient.OpenWorkItemManager())
{
tasks = wiManager.ListTasks(options.WorkItemName, options.JobName, odata, options.AdditionalBehaviors);
}
}
Func <ICloudTask, PSCloudTask> mappingFunction = t => { return(new PSCloudTask(t)); };
return(new PSAsyncEnumerable <PSCloudTask, ICloudTask>(tasks, mappingFunction).Take(options.MaxCount));
}
}
/// <summary>
/// Populates Azure Storage with the required files, and
/// submits the work item to the Azure Batch service.
/// </summary>
public async Task RunAsync()
{
Console.WriteLine("Running with the following settings: ");
Console.WriteLine("----------------------------------------");
Console.WriteLine(this.configurationSettings.ToString());
//Upload resources if required.
if (this.configurationSettings.ShouldUploadResources)
{
Console.WriteLine("Splitting file: {0} into {1} subfiles",
Constants.TextFilePath,
this.configurationSettings.NumberOfMapperTasks);
//Split the text file into the correct number of files for consumption by the mapper tasks.
FileSplitter splitter = new FileSplitter();
List <string> mapperTaskFiles = await splitter.SplitAsync(
Constants.TextFilePath,
this.configurationSettings.NumberOfMapperTasks);
await this.UploadResourcesAsync(mapperTaskFiles);
}
//Generate a SAS for the container.
string containerSasUrl = Helpers.ConstructContainerSas(
this.configurationSettings.StorageAccountName,
this.configurationSettings.StorageAccountKey,
this.configurationSettings.StorageServiceUrl,
this.configurationSettings.BlobContainer);
//Set up the Batch Service credentials used to authenticate with the Batch Service.
BatchCredentials batchCredentials = new BatchCredentials(
this.configurationSettings.BatchAccountName,
this.configurationSettings.BatchAccountKey);
using (IBatchClient batchClient = BatchClient.Connect(this.configurationSettings.BatchServiceUrl, batchCredentials))
{
using (IWorkItemManager workItemManager = batchClient.OpenWorkItemManager())
{
//Create the unbound work item in local memory. An object which exists only in local memory (and not on the Batch Service) is "unbound".
string workItemName = Environment.GetEnvironmentVariable("USERNAME") + DateTime.UtcNow.ToString("yyyyMMdd-HHmmss");
ICloudWorkItem unboundWorkItem = workItemManager.CreateWorkItem(workItemName);
//
// Construct the work item properties in local memory before commiting them to the Batch Service.
//
//Allow enough VMs in the pool to run each mapper task, and 1 extra to run the job manager.
int numberOfPoolVMs = 1 + this.configurationSettings.NumberOfMapperTasks;
//Define the pool specification for the pool which the work item will run on.
IPoolSpecification poolSpecification = new PoolSpecification()
{
TargetDedicated = numberOfPoolVMs,
VMSize = "small",
//You can learn more about os families and versions at:
//http://msdn.microsoft.com/en-us/library/azure/ee924680.aspx
OSFamily = "4",
TargetOSVersion = "*"
};
//Use the auto pool feature of the Batch Service to create a pool when the work item is created.
//This creates a new pool for each work item which is added.
IAutoPoolSpecification autoPoolSpecification = new AutoPoolSpecification()
{
AutoPoolNamePrefix = "TextSearchPool",
KeepAlive = false,
PoolLifeTimeOption = PoolLifeTimeOption.WorkItem,
PoolSpecification = poolSpecification
};
//Define the execution environment for this work item -- it will run on the pool defined by the auto pool specification above.
unboundWorkItem.JobExecutionEnvironment = new JobExecutionEnvironment()
{
AutoPoolSpecification = autoPoolSpecification
};
//Define the job manager for this work item. This job manager will run when any job is created and will submit the tasks for
//the work item. The job manager is the executable which manages the lifetime of the job
//and all tasks which should run for the job. In this case, the job manager submits the mapper and reducer tasks.
string jobManagerCommandLine = string.Format("{0} -JobManagerTask", Constants.TextSearchExe);
List <IResourceFile> jobManagerResourceFiles = Helpers.GetResourceFiles(containerSasUrl, Constants.RequiredExecutableFiles);
const string jobManagerTaskName = "JobManager";
unboundWorkItem.JobSpecification = new JobSpecification()
{
JobManager = new JobManager()
{
ResourceFiles = jobManagerResourceFiles,
CommandLine = jobManagerCommandLine,
//Determines if the job should terminate when the job manager process exits
KillJobOnCompletion = false,
Name = jobManagerTaskName
}
};
try
{
//Commit the unbound work item to the Batch Service.
Console.WriteLine("Adding work item: {0} to the Batch Service.", unboundWorkItem.Name);
await unboundWorkItem.CommitAsync(); //Issues a request to the Batch Service to add the work item which was defined above.
//
// Wait for the job manager task to complete.
//
//An object which is backed by a corresponding Batch Service object is "bound."
ICloudWorkItem boundWorkItem = await workItemManager.GetWorkItemAsync(workItemName);
//Wait for the job to be created automatically by the Batch Service.
string boundJobName = await Helpers.WaitForActiveJobAsync(boundWorkItem);
ICloudTask boundJobManagerTask = await workItemManager.GetTaskAsync(
workItemName,
boundJobName,
jobManagerTaskName);
TimeSpan maxJobCompletionTimeout = TimeSpan.FromMinutes(30);
IToolbox toolbox = batchClient.OpenToolbox();
ITaskStateMonitor monitor = toolbox.CreateTaskStateMonitor();
bool timedOut = await monitor.WaitAllAsync(new List <ICloudTask> {
boundJobManagerTask
}, TaskState.Completed, maxJobCompletionTimeout);
Console.WriteLine("Done waiting for job manager task.");
await boundJobManagerTask.RefreshAsync();
//Check to ensure the job manager task exited successfully.
await Helpers.CheckForTaskSuccessAsync(boundJobManagerTask, dumpStandardOutOnTaskSuccess : true);
if (timedOut)
{
throw new TimeoutException(string.Format("Timed out waiting for job manager task to complete."));
}
}
catch (AggregateException e)
{
e.Handle(
(innerE) =>
{
//We print all the inner exceptions for debugging purposes.
Console.WriteLine(innerE.ToString());
return(false);
});
throw;
}
catch (Exception e)
{
Console.WriteLine("Hit unexpected exception: {0}", e.ToString());
throw;
}
finally
{
//Delete the work item.
//This will delete the auto pool associated with the work item as long as the pool
//keep alive property is set to false.
if (this.configurationSettings.ShouldDeleteWorkItem)
{
Console.WriteLine("Deleting work item {0}", workItemName);
workItemManager.DeleteWorkItem(workItemName);
}
//Note that there were files uploaded to a container specified in the
//configuration file. This container will not be deleted or cleaned up by this sample.
}
}
}
}
/// <summary>
/// Checks for a tasks success or failure, and optionally dumps the output of the task. In the case that the task hit a scheduler or execution error,
/// dumps that information as well.
/// </summary>
/// <param name="boundTask">The task.</param>
/// <param name="dumpStandardOutOnTaskSuccess">True to log the standard output file of the task even if it succeeded. False to not log anything if the task succeeded.</param>
public static async Task CheckForTaskSuccessAsync(ICloudTask boundTask, bool dumpStandardOutOnTaskSuccess)
{
if (boundTask.State == TaskState.Completed)
{
//Check to see if the task has execution information metadata.
if (boundTask.ExecutionInformation != null)
{
//Dump the task scheduling error if there was one.
if (boundTask.ExecutionInformation.SchedulingError != null)
{
TaskSchedulingError schedulingError = boundTask.ExecutionInformation.SchedulingError;
Console.WriteLine("Task {0} hit scheduling error.", boundTask.Name);
Console.WriteLine("SchedulingError Code: {0}", schedulingError.Code);
Console.WriteLine("SchedulingError Message: {0}", schedulingError.Message);
Console.WriteLine("SchedulingError Category: {0}", schedulingError.Category);
Console.WriteLine("SchedulingError Details:");
foreach (NameValuePair detail in schedulingError.Details)
{
Console.WriteLine("{0} : {1}", detail.Name, detail.Value);
}
throw new TextSearchException(string.Format("Task {0} failed with a scheduling error", boundTask.Name));
}
//Read the content of the output files if the task exited.
if (boundTask.ExecutionInformation.ExitCode.HasValue)
{
Console.WriteLine("Task {0} exit code: {1}", boundTask.Name, boundTask.ExecutionInformation.ExitCode);
if (dumpStandardOutOnTaskSuccess && boundTask.ExecutionInformation.ExitCode.Value == 0 || boundTask.ExecutionInformation.ExitCode.Value != 0)
{
//Dump the standard out file of the task.
ITaskFile taskStandardOut = await boundTask.GetTaskFileAsync(
Batch.Constants.StandardOutFileName);
Console.WriteLine("Task {0} StdOut:", boundTask.Name);
Console.WriteLine("----------------------------------------");
string stdOutString = await taskStandardOut.ReadAsStringAsync();
Console.WriteLine(stdOutString);
}
//Check for nonzero exit code and dump standard error if there was a nonzero exit code.
if (boundTask.ExecutionInformation.ExitCode.Value != 0)
{
ITaskFile taskErrorFile = await boundTask.GetTaskFileAsync(
Batch.Constants.StandardErrorFileName);
Console.WriteLine("Task {0} StdErr:", boundTask.Name);
Console.WriteLine("----------------------------------------");
string stdErrString = await taskErrorFile.ReadAsStringAsync();
Console.WriteLine(stdErrString);
throw new TextSearchException(string.Format("Task {0} failed with a nonzero exit code", boundTask.Name));
}
}
}
else
{
throw new TextSearchException(string.Format("Task {0} is not completed yet. Current state: {1}", boundTask.Name, boundTask.State));
}
}
}
/// <summary>
/// Runs the job manager task.
/// </summary>
public async Task RunAsync()
{
Console.WriteLine("JobManager for account: {0}, work item: {1}, job: {2} has started...",
this.accountName,
this.workItemName,
this.jobName);
Console.WriteLine();
Console.WriteLine("JobManager running with the following settings: ");
Console.WriteLine("----------------------------------------");
Console.WriteLine(this.configurationSettings.ToString());
//Set up the Batch Service credentials used to authenticate with the Batch Service.
BatchCredentials batchCredentials = new BatchCredentials(
this.configurationSettings.BatchAccountName,
this.configurationSettings.BatchAccountKey);
using (IBatchClient batchClient = BatchClient.Connect(this.configurationSettings.BatchServiceUrl, batchCredentials))
{
using (IWorkItemManager workItemManager = batchClient.OpenWorkItemManager())
{
IToolbox toolbox = batchClient.OpenToolbox();
//Construct a container SAS to provide the Batch Service access to the files required to
//run the mapper and reducer tasks.
string containerSas = Helpers.ConstructContainerSas(
this.configurationSettings.StorageAccountName,
this.configurationSettings.StorageAccountKey,
this.configurationSettings.StorageServiceUrl,
this.configurationSettings.BlobContainer);
//
// Submit mapper tasks.
//
Console.WriteLine("Submitting {0} mapper tasks.", this.configurationSettings.NumberOfMapperTasks);
//The collection of tasks to add to the Batch Service.
List <ICloudTask> tasksToAdd = new List <ICloudTask>();
for (int i = 0; i < this.configurationSettings.NumberOfMapperTasks; i++)
{
string taskName = Helpers.GetMapperTaskName(i);
string fileBlobName = Helpers.GetSplitFileName(i);
string fileBlobPath = Helpers.ConstructBlobSource(containerSas, fileBlobName);
string commandLine = string.Format("{0} -MapperTask {1}", Constants.TextSearchExe, fileBlobPath);
ICloudTask unboundMapperTask = new CloudTask(taskName, commandLine);
//The set of files (exe's, dll's and configuration files) required to run the mapper task.
IReadOnlyList <string> mapperTaskRequiredFiles = Constants.RequiredExecutableFiles;
List <IResourceFile> mapperTaskResourceFiles = Helpers.GetResourceFiles(containerSas, mapperTaskRequiredFiles);
unboundMapperTask.ResourceFiles = mapperTaskResourceFiles;
tasksToAdd.Add(unboundMapperTask);
}
//Submit the unbound task collection to the Batch Service.
//Use the AddTask method which takes a collection of ICloudTasks for the best performance.
await workItemManager.AddTaskAsync(this.workItemName, this.jobName, tasksToAdd);
//
// Wait for the mapper tasks to complete.
//
Console.WriteLine("Waiting for the mapper tasks to complete...");
//List all the mapper tasks using a name filter.
DetailLevel mapperTaskNameFilter = new ODATADetailLevel()
{
FilterClause = string.Format("startswith(name, '{0}')", Constants.MapperTaskPrefix)
};
List <ICloudTask> tasksToMonitor = workItemManager.ListTasks(
this.workItemName,
this.jobName,
detailLevel: mapperTaskNameFilter).ToList();
//Use the task state monitor to wait for the tasks to complete.
ITaskStateMonitor taskStateMonitor = toolbox.CreateTaskStateMonitor();
bool timedOut = await taskStateMonitor.WaitAllAsync(tasksToMonitor, TaskState.Completed, TimeSpan.FromMinutes(5));
//Get the list of mapper tasks in order to analyze their state and ensure they completed successfully.
IEnumerableAsyncExtended <ICloudTask> asyncEnumerable = workItemManager.ListTasks(
this.workItemName,
this.jobName,
detailLevel: mapperTaskNameFilter);
IAsyncEnumerator <ICloudTask> asyncEnumerator = asyncEnumerable.GetAsyncEnumerator();
//Dump the status of each mapper task.
while (await asyncEnumerator.MoveNextAsync())
{
ICloudTask cloudTask = asyncEnumerator.Current;
Console.WriteLine("Task {0} is in state: {1}", cloudTask.Name, cloudTask.State);
await Helpers.CheckForTaskSuccessAsync(cloudTask, dumpStandardOutOnTaskSuccess : false);
Console.WriteLine();
}
//If not all the tasks reached the desired state within the timeout then the job manager
//cannot continue.
if (timedOut)
{
const string errorMessage = "Mapper tasks did not complete within expected timeout.";
Console.WriteLine(errorMessage);
throw new TimeoutException(errorMessage);
}
//
// Create the reducer task.
//
string reducerTaskCommandLine = string.Format("{0} -ReducerTask", Constants.TextSearchExe);
Console.WriteLine("Adding the reducer task: {0}", Constants.ReducerTaskName);
ICloudTask unboundReducerTask = new CloudTask(Constants.ReducerTaskName, reducerTaskCommandLine);
//The set of files (exe's, dll's and configuration files) required to run the reducer task.
List <IResourceFile> reducerTaskResourceFiles = Helpers.GetResourceFiles(containerSas, Constants.RequiredExecutableFiles);
unboundReducerTask.ResourceFiles = reducerTaskResourceFiles;
//Send the request to the Batch Service to add the reducer task.
await workItemManager.AddTaskAsync(this.workItemName, this.jobName, unboundReducerTask);
//
//Wait for the reducer task to complete.
//
//Get the bound reducer task and monitor it for completion.
ICloudTask boundReducerTask = await workItemManager.GetTaskAsync(this.workItemName, this.jobName, Constants.ReducerTaskName);
timedOut = await taskStateMonitor.WaitAllAsync(new List <ICloudTask> {
boundReducerTask
}, TaskState.Completed, TimeSpan.FromMinutes(2));
//Refresh the reducer task to get the most recent information about it from the Batch Service.
await boundReducerTask.RefreshAsync();
//Dump the reducer tasks exit code and scheduling error for debugging purposes.
await Helpers.CheckForTaskSuccessAsync(boundReducerTask, dumpStandardOutOnTaskSuccess : true);
//Handle the possibilty that the reducer task did not complete in the expected timeout.
if (timedOut)
{
const string errorMessage = "Reducer task did not complete within expected timeout.";
Console.WriteLine("Task {0} is in state: {1}", boundReducerTask.Name, boundReducerTask.State);
Console.WriteLine(errorMessage);
throw new TimeoutException(errorMessage);
}
//The job manager has completed.
Console.WriteLine("JobManager completed successfully.");
}
}
}
public TaskModel(JobModel parentJob, ICloudTask task)
{
this.ParentJob = parentJob;
this.Task = task;
this.LastUpdatedTime = DateTime.UtcNow;
}
/// <summary>
/// Checks for a tasks success or failure, and optionally dumps the output of the task. In the case that the task hit a scheduler or execution error,
/// dumps that information as well.
/// </summary>
/// <param name="boundTask">The task.</param>
/// <param name="dumpStandardOutOnTaskSuccess">True to log the standard output file of the task even if it succeeded. False to not log anything if the task succeeded.</param>
public static async Task CheckForTaskSuccessAsync(ICloudTask boundTask, bool dumpStandardOutOnTaskSuccess)
{
if (boundTask.State == TaskState.Completed)
{
//Check to see if the task has execution information metadata.
if (boundTask.ExecutionInformation != null)
{
//Dump the task scheduling error if there was one.
if (boundTask.ExecutionInformation.SchedulingError != null)
{
TaskSchedulingError schedulingError = boundTask.ExecutionInformation.SchedulingError;
Console.WriteLine("Task {0} hit scheduling error.", boundTask.Name);
Console.WriteLine("SchedulingError Code: {0}", schedulingError.Code);
Console.WriteLine("SchedulingError Message: {0}", schedulingError.Message);
Console.WriteLine("SchedulingError Category: {0}", schedulingError.Category);
Console.WriteLine("SchedulingError Details:");
foreach (NameValuePair detail in schedulingError.Details)
{
Console.WriteLine("{0} : {1}", detail.Name, detail.Value);
}
throw new TextSearchException(string.Format("Task {0} failed with a scheduling error", boundTask.Name));
}
//Read the content of the output files if the task exited.
if (boundTask.ExecutionInformation.ExitCode.HasValue)
{
Console.WriteLine("Task {0} exit code: {1}", boundTask.Name, boundTask.ExecutionInformation.ExitCode);
if (dumpStandardOutOnTaskSuccess && boundTask.ExecutionInformation.ExitCode.Value == 0 || boundTask.ExecutionInformation.ExitCode.Value != 0)
{
//Dump the standard out file of the task.
ITaskFile taskStandardOut = await boundTask.GetTaskFileAsync(
Microsoft.Azure.Batch.Constants.StandardOutFileName);
Console.WriteLine("Task {0} StdOut:", boundTask.Name);
Console.WriteLine("----------------------------------------");
string stdOutString = await taskStandardOut.ReadAsStringAsync();
Console.WriteLine(stdOutString);
}
//Check for nonzero exit code and dump standard error if there was a nonzero exit code.
if (boundTask.ExecutionInformation.ExitCode.Value != 0)
{
ITaskFile taskErrorFile = await boundTask.GetTaskFileAsync(
Microsoft.Azure.Batch.Constants.StandardErrorFileName);
Console.WriteLine("Task {0} StdErr:", boundTask.Name);
Console.WriteLine("----------------------------------------");
string stdErrString = await taskErrorFile.ReadAsStringAsync();
Console.WriteLine(stdErrString);
throw new TextSearchException(string.Format("Task {0} failed with a nonzero exit code", boundTask.Name));
}
}
}
else
{
throw new TextSearchException(string.Format("Task {0} is not completed yet. Current state: {1}", boundTask.Name, boundTask.State));
}
}
}
public TaskModel(JobModel parentJob, ICloudTask task)
{
this.ParentJob = parentJob;
this.Task = task;
this.LastUpdatedTime = DateTime.UtcNow;
}
