protected void Page_Init(object sender, EventArgs e)
{
if (!Request.GetUser().Browse)
{
Response.StatusCode = 401;
return;
}
try
{
Task = GeneralTask.Create(ID = RouteData.GetRouteString("ID"));
if (Task != null) Title = TaskHelper.GetName(Task.Type) + "中";
Viewer.SetTask(Task);
}
catch
{
Viewer.Never();
}
}
static void AddTasks(BatchClient client)
{
CloudJob job = client.JobOperations.GetJob("testjob1");
ResourceFile programFile = new ResourceFile(
"https://mystorage00.blob.core.windows.net/testcon1/ProcessTaskData.exe",
"ProcessTaskData.exe"
);
ResourceFile assemblyFile = new ResourceFile(
"https://mystorage00.blob.core.windows.net/testcon1/Microsoft.WindowsAzure.Storage.dll",
"Microsoft.WindowsAzure.Storage.dll"
);
for (int i = 1; i < 4; ++i)
{
string blobName = "taskdata" + i;
string taskName = "mytask" + i;
ResourceFile taskData = new ResourceFile("https://mystorage00.blob.core.windows.net/testcon1/" +
blobName, blobName);
CloudTask task = new CloudTask(
taskName,
"ProcessTaskData.exe https://mystorage00.blob.core.windows.net/testcon1/" +
blobName + " 3");
List<ResourceFile> taskFiles = new List<ResourceFile>();
taskFiles.Add(taskData);
taskFiles.Add(programFile);
taskFiles.Add(assemblyFile);
task.ResourceFiles = taskFiles;
job.AddTask(task);
job.Commit();
job.Refresh();
}
client.Utilities.CreateTaskStateMonitor().WaitAll(job.ListTasks(),
TaskState.Completed, new TimeSpan(0, 30, 0));
Console.WriteLine("The tasks completed successfully.");
foreach (CloudTask task in job.ListTasks())
{
Console.WriteLine("Task " + task.Id + " says:\n" + task.GetNodeFile(Constants.StandardOutFileName).ReadAsString());
}
Console.WriteLine("Press Enter to continue.");
Console.ReadLine();
}
private static async Task MainAsync(string[] args)
{
// You may adjust these values to experiment with different compute resource scenarios.
const string nodeSize = "small";
const int nodeCount = 1;
const int maxTasksPerNode = 4;
// Adjust the task count to experiment with different list operation query durations
const int taskCount = 5000;
const string poolId = "EfficientListQueriesSamplePool";
const string jobId = "EfficientListQueriesSampleJob";
// Set up the credentials required by the BatchClient. Configure your AccountSettings in the
// Microsoft.Azure.Batch.Samples.Common project within this solution.
BatchSharedKeyCredentials cred = new BatchSharedKeyCredentials(AccountSettings.Default.BatchServiceUrl,
AccountSettings.Default.BatchAccountName,
AccountSettings.Default.BatchAccountKey);
using (BatchClient batchClient = await BatchClient.OpenAsync(cred))
{
// Create a CloudPool, or obtain an existing pool with the specified ID
CloudPool pool = await ArticleHelpers.CreatePoolIfNotExistAsync(batchClient,
poolId,
nodeSize,
nodeCount,
maxTasksPerNode);
// Create a CloudJob, or obtain an existing job with the specified ID
CloudJob job = await ArticleHelpers.CreateJobIfNotExistAsync(batchClient, poolId, jobId);
// Configure the tasks we'll be querying. Each task simply echoes the node's
// name and then exits. We create "large" tasks by setting an environment
// variable for each that is 2048 bytes in size. This is done simply to
// increase response time when querying the batch service to more clearly
// demonstrate query durations.
List<CloudTask> tasks = new List<CloudTask>();
List<EnvironmentSetting> environmentSettings = new List<EnvironmentSetting>();
environmentSettings.Add(new EnvironmentSetting("BIGENV", GetBigString(2048)));
for (int i = 1; i < taskCount + 1; i++)
{
string taskId = "task" + i.ToString().PadLeft(5, '0');
string taskCommandLine = "cmd /c echo %COMPUTERNAME%";
CloudTask task = new CloudTask(taskId, taskCommandLine);
task.EnvironmentSettings = environmentSettings;
tasks.Add(task);
}
Console.WriteLine();
Console.WriteLine("Adding {0} tasks to job {1}...", taskCount, job.Id);
Stopwatch stopwatch = Stopwatch.StartNew();
// Add the tasks in one API call as opposed to a separate AddTask call for each. Bulk task submission
// helps to ensure efficient underlying API calls to the Batch service.
await batchClient.JobOperations.AddTaskAsync(job.Id, tasks);
stopwatch.Stop();
Console.WriteLine("{0} tasks added in {1}, hit ENTER to query tasks...", taskCount, stopwatch.Elapsed);
Console.ReadLine();
Console.WriteLine();
stopwatch.Reset();
// Obtain the tasks, specifying different detail levels to demonstrate limiting the number of tasks returned
// and the amount of data returned for each. If your job tasks number in the thousands or have "large" properties
// (such as our big environment variable), specifying a DetailLevel is important in reducing the amount of data
// transferred, lowering your query response times (potentially greatly).
// Get a subset of the tasks based on different task states
ODATADetailLevel detail = new ODATADetailLevel();
detail.FilterClause = "state eq 'active'";
detail.SelectClause = "id,state";
await QueryTasksAsync(batchClient, job.Id, detail);
detail.FilterClause = "state eq 'running'";
await QueryTasksAsync(batchClient, job.Id, detail);
detail.FilterClause = "state eq 'completed'";
await QueryTasksAsync(batchClient, job.Id, detail);
// Get all tasks, but limit the properties returned to task id and state only
detail.FilterClause = null;
detail.SelectClause = "id,state";
await QueryTasksAsync(batchClient, job.Id, detail);
// Get all tasks, include id and state, also include the inflated environment settings property
detail.SelectClause = "id,state,environmentSettings";
await QueryTasksAsync(batchClient, job.Id, detail);
// Get all tasks, include all standard properties, and expand the statistics
detail.ExpandClause = "stats";
detail.SelectClause = null;
await QueryTasksAsync(batchClient, job.Id, detail);
Console.WriteLine();
Console.WriteLine("Done!");
Console.WriteLine();
// Clean up the resources we've created in the Batch account
Console.WriteLine("Delete job? [yes] no");
string response = Console.ReadLine().ToLower();
if (response != "n" && response != "no")
{
await batchClient.JobOperations.DeleteJobAsync(job.Id);
}
Console.WriteLine("Delete pool? [yes] no");
response = Console.ReadLine().ToLower();
if (response != "n" && response != "no")
{
await batchClient.PoolOperations.DeletePoolAsync(pool.Id);
}
}
}
private static async Task MainAsync(string[] args)
{
// You may adjust these values to experiment with different compute resource scenarios.
const string nodeSize = "small";
const int nodeCount = 4;
const int maxTasksPerNode = 4;
const int taskCount = 32;
// Ensure there are enough tasks to help avoid hitting some timeout conditions below
const int minimumTaskCount = nodeCount * maxTasksPerNode * 2;
if (taskCount < minimumTaskCount)
{
Console.WriteLine("You must specify at least two tasks per node core for this sample ({0} tasks in this configuration).", minimumTaskCount);
Console.WriteLine();
// Not enough tasks, exit the application
return;
}
// In this sample, the tasks simply ping localhost on the compute nodes; adjust these
// values to simulate variable task duration
const int minPings = 30;
const int maxPings = 60;
const string poolId = "ParallelTasksSamplePool";
const string jobId = "ParallelTasksSampleJob";
// Amount of time to wait before timing out (potentially) long-running tasks
TimeSpan longTaskDurationLimit = TimeSpan.FromMinutes(30);
// Set up access to your Batch account with a BatchClient. Configure your AccountSettings in the
// Microsoft.Azure.Batch.Samples.Common project within this solution.
BatchSharedKeyCredentials cred = new BatchSharedKeyCredentials(AccountSettings.Default.BatchServiceUrl,
AccountSettings.Default.BatchAccountName,
AccountSettings.Default.BatchAccountKey);
using (BatchClient batchClient = await BatchClient.OpenAsync(cred))
{
// Create a CloudPool, or obtain an existing pool with the specified ID
CloudPool pool = await ArticleHelpers.CreatePoolIfNotExistAsync(batchClient,
poolId,
nodeSize,
nodeCount,
maxTasksPerNode);
// Create a CloudJob, or obtain an existing pool with the specified ID
CloudJob job = await ArticleHelpers.CreateJobIfNotExistAsync(batchClient, poolId, jobId);
// The job's tasks ping localhost a random number of times between minPings and maxPings.
// Adjust the minPings/maxPings values above to experiment with different task durations.
Random rand = new Random();
List<CloudTask> tasks = new List<CloudTask>();
for (int i = 1; i <= taskCount; i++)
{
string taskId = "task" + i.ToString().PadLeft(3, '0');
string taskCommandLine = "ping -n " + rand.Next(minPings, maxPings + 1).ToString() + " localhost";
CloudTask task = new CloudTask(taskId, taskCommandLine);
tasks.Add(task);
}
// Pause execution until the pool is steady and its compute nodes are ready to accept jobs.
// NOTE: Such a pause is not necessary within your own code. Tasks can be added to a job at any point and will be
// scheduled to execute on a compute node as soon any node has reached Idle state. Because the focus of this sample
// is the demonstration of running tasks in parallel on multiple compute nodes, we wait for all compute nodes to
// complete initialization and reach the Idle state in order to maximize the number of compute nodes available for
// parallelization.
await ArticleHelpers.WaitForPoolToReachStateAsync(batchClient, pool.Id, AllocationState.Steady, longTaskDurationLimit);
await ArticleHelpers.WaitForNodesToReachStateAsync(batchClient, pool.Id, ComputeNodeState.Idle, longTaskDurationLimit);
// Add the tasks in one API call as opposed to a separate AddTask call for each. Bulk task submission
// helps to ensure efficient underlying API calls to the Batch service.
await batchClient.JobOperations.AddTaskAsync(job.Id, tasks);
// Pause again to wait until *all* nodes are running tasks
await ArticleHelpers.WaitForNodesToReachStateAsync(batchClient, pool.Id, ComputeNodeState.Running, TimeSpan.FromMinutes(2));
Stopwatch stopwatch = Stopwatch.StartNew();
// Print out task assignment information.
Console.WriteLine();
await GettingStartedCommon.PrintNodeTasksAsync(batchClient, pool.Id);
Console.WriteLine();
// Pause execution while we wait for all of the tasks to complete
Console.WriteLine("Waiting for task completion...");
Console.WriteLine();
if (await batchClient.Utilities.CreateTaskStateMonitor().WhenAllAsync(job.ListTasks(),
TaskState.Completed,
longTaskDurationLimit))
{
Console.WriteLine("Operation timed out while waiting for submitted tasks to reach state {0}", TaskState.Completed);
}
stopwatch.Stop();
// Obtain the tasks, specifying a detail level to limit the number of properties returned for each task.
// If you have a large number of tasks, specifying a DetailLevel is extremely important in reducing the
// amount of data transferred, lowering your query response times in increasing performance.
ODATADetailLevel detail = new ODATADetailLevel(selectClause: "id,commandLine,nodeInfo,state");
IPagedEnumerable<CloudTask> allTasks = batchClient.JobOperations.ListTasks(job.Id, detail);
// Get a collection of the completed tasks sorted by the compute nodes on which they executed
List<CloudTask> completedTasks = allTasks
.Where(t => t.State == TaskState.Completed)
.OrderBy(t => t.ComputeNodeInformation.ComputeNodeId)
.ToList();
// Print the completed task information
Console.WriteLine();
Console.WriteLine("Completed tasks:");
string lastNodeId = string.Empty;
foreach (CloudTask task in completedTasks)
{
if (!string.Equals(lastNodeId, task.ComputeNodeInformation.ComputeNodeId))
{
Console.WriteLine();
Console.WriteLine(task.ComputeNodeInformation.ComputeNodeId);
}
lastNodeId = task.ComputeNodeInformation.ComputeNodeId;
Console.WriteLine("\t{0}: {1}", task.Id, task.CommandLine);
}
// Get a collection of the uncompleted tasks which may exist if the TaskMonitor timeout was hit
List<CloudTask> uncompletedTasks = allTasks
.Where(t => t.State != TaskState.Completed)
.OrderBy(t => t.Id)
.ToList();
// Print a list of uncompleted tasks, if any
Console.WriteLine();
Console.WriteLine("Uncompleted tasks:");
Console.WriteLine();
if (uncompletedTasks.Any())
{
foreach (CloudTask task in uncompletedTasks)
{
Console.WriteLine("\t{0}: {1}", task.Id, task.CommandLine);
}
}
else
{
Console.WriteLine("\t<none>");
}
// Print some summary information
Console.WriteLine();
Console.WriteLine(" Nodes: " + nodeCount);
Console.WriteLine(" Node size: " + nodeSize);
Console.WriteLine("Max tasks per node: " + pool.MaxTasksPerComputeNode);
Console.WriteLine(" Tasks: " + tasks.Count);
Console.WriteLine(" Duration: " + stopwatch.Elapsed);
Console.WriteLine();
Console.WriteLine("Done!");
Console.WriteLine();
// Clean up the resources we've created in the Batch account
Console.WriteLine("Delete job? [yes] no");
string response = Console.ReadLine().ToLower();
if (response != "n" && response != "no")
{
await batchClient.JobOperations.DeleteJobAsync(job.Id);
}
Console.WriteLine("Delete pool? [yes] no");
response = Console.ReadLine();
if (response != "n" && response != "no")
{
await batchClient.PoolOperations.DeletePoolAsync(pool.Id);
}
}
}
/// <summary>
/// Checks for a task's success or failure, and optionally dumps the output of the task. In the case that the task hit a scheduler or execution error,
/// dumps that information as well.
/// </summary>
/// <param name="boundTask">The task.</param>
/// <param name="dumpStandardOutOnTaskSuccess">True to log the standard output file of the task even if it succeeded. False to not log anything if the task succeeded.</param>
/// <returns>The string containing the standard out of the file, or null if stdout could not be gathered.</returns>
public static async Task<string> CheckForTaskSuccessAsync(CloudTask boundTask, bool dumpStandardOutOnTaskSuccess)
{
if (boundTask.State == TaskState.Completed)
{
string result = null;
//Check to see if the task has execution information metadata.
if (boundTask.ExecutionInformation != null)
{
//Dump the task scheduling error if there was one.
if (boundTask.ExecutionInformation.SchedulingError != null)
{
TaskSchedulingError schedulingError = boundTask.ExecutionInformation.SchedulingError;
Console.WriteLine("Task {0} hit scheduling error.", boundTask.Id);
Console.WriteLine("SchedulingError Code: {0}", schedulingError.Code);
Console.WriteLine("SchedulingError Message: {0}", schedulingError.Message);
Console.WriteLine("SchedulingError Category: {0}", schedulingError.Category);
Console.WriteLine("SchedulingError Details:");
foreach (NameValuePair detail in schedulingError.Details)
{
Console.WriteLine("{0} : {1}", detail.Name, detail.Value);
}
throw new TextSearchException(String.Format("Task {0} failed with a scheduling error", boundTask.Id));
}
//Read the content of the output files if the task exited.
if (boundTask.ExecutionInformation.ExitCode.HasValue)
{
Console.WriteLine("Task {0} exit code: {1}", boundTask.Id, boundTask.ExecutionInformation.ExitCode);
if (dumpStandardOutOnTaskSuccess && boundTask.ExecutionInformation.ExitCode.Value == 0 || boundTask.ExecutionInformation.ExitCode.Value != 0)
{
//Dump the standard out file of the task.
NodeFile taskStandardOut = await boundTask.GetNodeFileAsync(Batch.Constants.StandardOutFileName);
Console.WriteLine("Task {0} StdOut:", boundTask.Id);
Console.WriteLine("----------------------------------------");
string stdOutString = await taskStandardOut.ReadAsStringAsync();
result = stdOutString;
Console.WriteLine(stdOutString);
}
//Check for nonzero exit code and dump standard error if there was a nonzero exit code.
if (boundTask.ExecutionInformation.ExitCode.Value != 0)
{
NodeFile taskErrorFile = await boundTask.GetNodeFileAsync(Batch.Constants.StandardErrorFileName);
Console.WriteLine("Task {0} StdErr:", boundTask.Id);
Console.WriteLine("----------------------------------------");
string stdErrString = await taskErrorFile.ReadAsStringAsync();
Console.WriteLine(stdErrString);
throw new TextSearchException(String.Format("Task {0} failed with a nonzero exit code", boundTask.Id));
}
}
}
return result;
}
else
{
throw new TextSearchException(String.Format("Task {0} is not completed yet. Current state: {1}", boundTask.Id, boundTask.State));
}
}
public static void Main(string[] args)
{
const int taskCount = 5000;
const string poolId = "poolEffQuery";
const string jobId = "jobEffQuery";
// Set up the credentials required by the BatchClient. Configure your AccountSettings in the
// Microsoft.Azure.Batch.Samples.Common project within this solution.
BatchSharedKeyCredentials cred = new BatchSharedKeyCredentials(AccountSettings.Default.BatchServiceUrl,
AccountSettings.Default.BatchAccountName,
AccountSettings.Default.BatchAccountKey);
using (BatchClient batchClient = BatchClient.Open(cred))
{
// Create a CloudPool, or obtain an existing pool with the specified ID
CreatePool(batchClient, poolId).Wait();
CloudPool pool = batchClient.PoolOperations.GetPool(poolId);
// Create a CloudJob, or obtain an existing job with the specified ID
CloudJob job = ArticleHelpers.CreateJobAsync(batchClient, poolId, jobId).Result;
// Configure the tasks we'll be querying. Each task simply echoes the node's
// name and then exits. We create "large" tasks by setting an environment
// variable for each that is 2048 bytes in size. This is done simply to
// increase response time when querying the batch service to more clearly
// demonstrate query durations.
List<CloudTask> tasks = new List<CloudTask>();
List<EnvironmentSetting> environmentSettings = new List<EnvironmentSetting>();
environmentSettings.Add(new EnvironmentSetting("BIGENV", GetBigString(2048)));
for (int i = 1; i < taskCount + 1; i++)
{
string taskId = "task" + i.ToString().PadLeft(5, '0');
string taskCommandLine = "cmd /c echo %COMPUTERNAME%";
CloudTask task = new CloudTask(taskId, taskCommandLine);
task.EnvironmentSettings = environmentSettings;
tasks.Add(task);
}
Console.WriteLine();
Console.WriteLine("Adding {0} tasks to job {1}...", taskCount, job.Id);
Stopwatch stopwatch = new Stopwatch();
stopwatch.Start();
// To reduce the chances of hitting Batch service throttling limits, we add the tasks in
// one API call as opposed to a separate AddTask call for each. This is crucial if you
// are adding many tasks to your jobs.
batchClient.JobOperations.AddTask(job.Id, tasks);
stopwatch.Stop();
Console.WriteLine("{0} tasks added in {1}, hit ENTER to query tasks...", taskCount, stopwatch.Elapsed);
Console.ReadLine();
Console.WriteLine();
stopwatch.Reset();
// Obtain the tasks, specifying different detail levels to demonstrate limiting the number of tasks returned
// and the amount of data returned for each. If your job tasks number in the thousands or have "large" properties
// (such as our big environment variable), specifying a DetailLevel is important in reducing the amount of data
// transferred, lowering your query response times (potentially greatly).
// Get a subset of the tasks based on different task states
ODATADetailLevel detail = new ODATADetailLevel();
detail.FilterClause = "state eq 'active'";
detail.SelectClause = "id,state";
QueryTasks(batchClient, job.Id, detail);
detail.FilterClause = "state eq 'running'";
QueryTasks(batchClient, job.Id, detail);
detail.FilterClause = "state eq 'completed'";
QueryTasks(batchClient, job.Id, detail);
// Get all tasks, but limit the properties returned to task id and state only
detail.FilterClause = null;
detail.SelectClause = "id,state";
QueryTasks(batchClient, job.Id, detail);
// Get all tasks, include id and state, also include the inflated environment settings property
detail.SelectClause = "id,state,environmentSettings";
QueryTasks(batchClient, job.Id, detail);
// Get all tasks, include all standard properties, and expand the statistics
detail.ExpandClause = "stats";
detail.SelectClause = null;
QueryTasks(batchClient, job.Id, detail);
Console.WriteLine();
Console.WriteLine("Sample complete, hit ENTER to continue...");
Console.ReadLine();
// Clean up the resources we've created in the Batch account
Console.WriteLine("Delete job? [yes] no");
string response = Console.ReadLine().ToLower();
if (response != "n" && response != "no")
{
batchClient.JobOperations.DeleteJob(job.Id);
}
Console.WriteLine("Delete pool? [yes] no");
response = Console.ReadLine().ToLower();
if (response != "n" && response != "no")
{
batchClient.PoolOperations.DeletePool(pool.Id);
}
}
}
/// <summary>
/// Runs the job manager task.
/// </summary>
public async Task RunAsync()
{
Console.WriteLine("JobManager for account: {0}, job: {1} has started...",
this.accountName,
this.jobId);
Console.WriteLine();
Console.WriteLine("JobManager running with the following settings: ");
Console.WriteLine("----------------------------------------");
Console.WriteLine(this.configurationSettings.ToString());
//Set up the Batch Service credentials used to authenticate with the Batch Service.
BatchSharedKeyCredentials batchSharedKeyCredentials = new BatchSharedKeyCredentials(
this.configurationSettings.BatchServiceUrl,
this.configurationSettings.BatchAccountName,
this.configurationSettings.BatchAccountKey);
using (BatchClient batchClient = await BatchClient.OpenAsync(batchSharedKeyCredentials))
{
//Construct a container SAS to provide the Batch Service access to the files required to
//run the mapper and reducer tasks.
string containerSas = Helpers.ConstructContainerSas(
this.configurationSettings.StorageAccountName,
this.configurationSettings.StorageAccountKey,
this.configurationSettings.StorageServiceUrl,
this.configurationSettings.BlobContainer);
//
// Submit mapper tasks.
//
Console.WriteLine("Submitting {0} mapper tasks.", this.configurationSettings.NumberOfMapperTasks);
//The collection of tasks to add to the Batch Service.
List<CloudTask> tasksToAdd = new List<CloudTask>();
for (int i = 0; i < this.configurationSettings.NumberOfMapperTasks; i++)
{
string taskId = Helpers.GetMapperTaskId(i);
string fileBlobName = Helpers.GetSplitFileName(i);
string fileBlobPath = Helpers.ConstructBlobSource(containerSas, fileBlobName);
string commandLine = string.Format("{0} -MapperTask {1}", Constants.TextSearchExe, fileBlobPath);
CloudTask unboundMapperTask = new CloudTask(taskId, commandLine);
//The set of files (exes, dlls and configuration files) required to run the mapper task.
IReadOnlyList<string> mapperTaskRequiredFiles = Constants.RequiredExecutableFiles;
List<ResourceFile> mapperTaskResourceFiles = Helpers.GetResourceFiles(containerSas, mapperTaskRequiredFiles);
unboundMapperTask.ResourceFiles = mapperTaskResourceFiles;
tasksToAdd.Add(unboundMapperTask);
}
//Submit the unbound task collection to the Batch Service.
//Use the AddTask method which takes a collection of CloudTasks for the best performance.
await batchClient.JobOperations.AddTaskAsync(this.jobId, tasksToAdd);
//
// Wait for the mapper tasks to complete.
//
Console.WriteLine("Waiting for the mapper tasks to complete...");
//List all the mapper tasks using an id filter.
DetailLevel mapperTaskIdFilter = new ODATADetailLevel()
{
FilterClause = string.Format("startswith(id, '{0}')", Constants.MapperTaskPrefix)
};
IEnumerable<CloudTask> tasksToMonitor = batchClient.JobOperations.ListTasks(
this.jobId,
detailLevel: mapperTaskIdFilter);
//Use the task state monitor to wait for the tasks to complete.
TaskStateMonitor taskStateMonitor = batchClient.Utilities.CreateTaskStateMonitor();
bool timedOut = await taskStateMonitor.WaitAllAsync(tasksToMonitor, TaskState.Completed, TimeSpan.FromMinutes(5));
//Get the list of mapper tasks in order to analyze their state and ensure they completed successfully.
IPagedEnumerable<CloudTask> asyncEnumerable = batchClient.JobOperations.ListTasks(
this.jobId,
detailLevel: mapperTaskIdFilter);
await asyncEnumerable.ForEachAsync(async cloudTask =>
{
Console.WriteLine("Task {0} is in state: {1}", cloudTask.Id, cloudTask.State);
await Helpers.CheckForTaskSuccessAsync(cloudTask, dumpStandardOutOnTaskSuccess: false);
Console.WriteLine();
});
//If not all the tasks reached the desired state within the timeout then the job manager
//cannot continue.
if (timedOut)
{
const string errorMessage = "Mapper tasks did not complete within expected timeout.";
Console.WriteLine(errorMessage);
throw new TimeoutException(errorMessage);
}
//
// Create the reducer task.
//
string reducerTaskCommandLine = string.Format("{0} -ReducerTask", Constants.TextSearchExe);
Console.WriteLine("Adding the reducer task: {0}", Constants.ReducerTaskId);
CloudTask unboundReducerTask = new CloudTask(Constants.ReducerTaskId, reducerTaskCommandLine);
//The set of files (exes, dlls and configuration files) required to run the reducer task.
List<ResourceFile> reducerTaskResourceFiles = Helpers.GetResourceFiles(containerSas, Constants.RequiredExecutableFiles);
unboundReducerTask.ResourceFiles = reducerTaskResourceFiles;
//Send the request to the Batch Service to add the reducer task.
await batchClient.JobOperations.AddTaskAsync(this.jobId, unboundReducerTask);
//
//Wait for the reducer task to complete.
//
//Get the bound reducer task and monitor it for completion.
CloudTask boundReducerTask = await batchClient.JobOperations.GetTaskAsync(this.jobId, Constants.ReducerTaskId);
timedOut = await taskStateMonitor.WaitAllAsync(new List<CloudTask> {boundReducerTask}, TaskState.Completed, TimeSpan.FromMinutes(2));
//Refresh the reducer task to get the most recent information about it from the Batch Service.
await boundReducerTask.RefreshAsync();
//Dump the reducer tasks exit code and scheduling error for debugging purposes.
await Helpers.CheckForTaskSuccessAsync(boundReducerTask, dumpStandardOutOnTaskSuccess: true);
//Handle the possibilty that the reducer task did not complete in the expected timeout.
if (timedOut)
{
const string errorMessage = "Reducer task did not complete within expected timeout.";
Console.WriteLine("Task {0} is in state: {1}", boundReducerTask.Id, boundReducerTask.State);
Console.WriteLine(errorMessage);
throw new TimeoutException(errorMessage);
}
//The job manager has completed.
Console.WriteLine("JobManager completed successfully.");
}
}
public void Bug1665834TaskStateMonitor()
{
Action test = () =>
{
using (BatchClient batchCli = TestUtilities.OpenBatchClient(TestUtilities.GetCredentialsFromEnvironment()))
{
string jobId = "Bug1665834Job-" + TestUtilities.GetMyName();
try
{
CloudJob unboundJob = batchCli.JobOperations.CreateJob(jobId, new PoolInformation());
unboundJob.PoolInformation.PoolId = this.poolFixture.PoolId;
unboundJob.Commit();
CloudJob boundJob = batchCli.JobOperations.GetJob(jobId);
// add some noise tasks
for (int j = 0; j < 5; j++)
{
CloudTask unboundTaskQuick = new CloudTask((10 + j).ToString(), "cmd /c hostname");
boundJob.AddTask(unboundTaskQuick);
}
System.Threading.Thread.Sleep(5000);
// wait for fast tasks to complete
{
bool repeat = true;
while (repeat)
{
CloudPool boundPool = batchCli.PoolOperations.GetPool(this.poolFixture.PoolId);
repeat = false;
foreach (CloudTask curTask in boundJob.ListTasks())
{
if (curTask.State != Microsoft.Azure.Batch.Common.TaskState.Completed)
{
repeat = true;
this.testOutputHelper.WriteLine("Manual Wait Task Id: " + curTask.Id + ", state = " + curTask.State);
this.testOutputHelper.WriteLine(" poolstate: " + boundPool.State + ", currentdedicated: " + boundPool.CurrentDedicatedComputeNodes);
this.testOutputHelper.WriteLine(" compute nodes:");
foreach (ComputeNode curComputeNode in boundPool.ListComputeNodes())
{
this.testOutputHelper.WriteLine(" computeNode.Id: " + curComputeNode.Id + ", state: " + curComputeNode.State);
}
}
}
}
}
// add some longer running tasks
this.testOutputHelper.WriteLine("Adding longer running tasks");
for (int i = 0; i < 15; i++)
{
CloudTask unboundTask = new CloudTask(i.ToString() + "_a234567890a234567890a234567890a234567890a234567890a234567890", "cmd /c ping 127.0.0.1 -n 4");
boundJob.AddTask(unboundTask);
}
Utilities utilities = batchCli.Utilities;
TaskStateMonitor tsm = utilities.CreateTaskStateMonitor();
IPagedEnumerable <CloudTask> taskList = boundJob.ListTasks();
ODATAMonitorControl odmc = new ODATAMonitorControl();
// try to set really low delay
odmc.DelayBetweenDataFetch = new TimeSpan(0);
// confirm the floor is enforced
Assert.Equal(500, odmc.DelayBetweenDataFetch.Milliseconds);
this.testOutputHelper.WriteLine("Calling TaskStateMonitor.WaitAll(). This will take a while.");
TimeSpan timeToWait = TimeSpan.FromMinutes(5);
Task whenAll = tsm.WhenAll(taskList, Microsoft.Azure.Batch.Common.TaskState.Completed, timeToWait, controlParams: odmc);
//This could throw, if it does the test will fail, which is what we want
whenAll.Wait();
foreach (CloudTask curTask in boundJob.ListTasks())
{
Assert.Equal(TaskState.Completed, curTask.State);
}
}
finally
{
// cleanup
TestUtilities.DeleteJobIfExistsAsync(batchCli, jobId).Wait();
}
}
};
SynchronizationContextHelper.RunTest(test, TestTimeout);
}
/// <summary>
/// Adds a task.
/// </summary>
/// <param name="options">The options describing the task to add.</param>
/// <returns></returns>
public async Task AddTaskAsync(AddTaskOptions options)
{
CloudTask unboundTask = new CloudTask(options.TaskId, options.CommandLine);
unboundTask.ResourceFiles = options.ResourceFiles.ConvertAll(f => new ResourceFile(f.BlobSource, f.FilePath));
await this.Client.JobOperations.AddTaskAsync(options.JobId, unboundTask);
}
static void Main(string[] args)
{
if (String.IsNullOrEmpty(BatchAccountName) ||
String.IsNullOrEmpty(BatchAccountKey) ||
String.IsNullOrEmpty(BatchAccountUrl) ||
String.IsNullOrEmpty(StorageAccountName) ||
String.IsNullOrEmpty(StorageAccountKey))
{
throw new InvalidOperationException("One or more account credential strings have not been populated. Please ensure that your Batch and Storage account credentials have been specified.");
}
try
{
Console.WriteLine("Sample start: {0}", DateTime.Now);
Console.WriteLine();
Stopwatch timer = new Stopwatch();
timer.Start();
// Create the blob client, for use in obtaining references to blob storage containers
CloudBlobClient blobClient = CreateCloudBlobClient(StorageAccountName, StorageAccountKey);
// Use the blob client to create the input container in Azure Storage
const string inputContainerName = "input";
CloudBlobContainer container = blobClient.GetContainerReference(inputContainerName);
container.CreateIfNotExistsAsync().Wait();
// The collection of data files that are to be processed by the tasks
List <string> inputFilePaths = new List <string>
{
"taskdata0.txt",
"taskdata1.txt",
"taskdata2.txt"
};
// Upload the data files to Azure Storage. This is the data that will be processed by each of the tasks that are
// executed on the compute nodes within the pool.
List <ResourceFile> inputFiles = new List <ResourceFile>();
foreach (string filePath in inputFilePaths)
{
inputFiles.Add(UploadFileToContainer(blobClient, inputContainerName, filePath));
}
// Get a Batch client using account creds
BatchSharedKeyCredentials cred = new BatchSharedKeyCredentials(BatchAccountUrl, BatchAccountName, BatchAccountKey);
using (BatchClient batchClient = BatchClient.Open(cred))
{
Console.WriteLine("Creating pool [{0}]...", PoolId);
// Create a Windows Server image, VM configuration, Batch pool
ImageReference imageReference = CreateImageReference();
VirtualMachineConfiguration vmConfiguration = CreateVirtualMachineConfiguration(imageReference);
CreateBatchPool(batchClient, vmConfiguration);
// Create a Batch job
Console.WriteLine("Creating job [{0}]...", JobId);
try
{
CloudJob job = batchClient.JobOperations.CreateJob();
job.Id = JobId;
job.PoolInformation = new PoolInformation {
PoolId = PoolId
};
job.Commit();
}
catch (BatchException be)
{
// Accept the specific error code JobExists as that is expected if the job already exists
if (be.RequestInformation?.BatchError?.Code == BatchErrorCodeStrings.JobExists)
{
Console.WriteLine("The job {0} already existed when we tried to create it", JobId);
}
else
{
throw; // Any other exception is unexpected
}
}
// Create a collection to hold the tasks that we'll be adding to the job
Console.WriteLine("Adding {0} tasks to job [{1}]...", inputFiles.Count, JobId);
List <CloudTask> tasks = new List <CloudTask>();
// Create each of the tasks to process one of the input files.
for (int i = 0; i < inputFiles.Count; i++)
{
string taskId = String.Format("Task{0}", i);
string inputFilename = inputFiles[i].FilePath;
string taskCommandLine = String.Format("cmd /c type {0}", inputFilename);
CloudTask task = new CloudTask(taskId, taskCommandLine);
task.ResourceFiles = new List <ResourceFile> {
inputFiles[i]
};
tasks.Add(task);
}
// Add all tasks to the job.
batchClient.JobOperations.AddTask(JobId, tasks);
// Monitor task success/failure, specifying a maximum amount of time to wait for the tasks to complete.
TimeSpan timeout = TimeSpan.FromMinutes(30);
Console.WriteLine("Monitoring all tasks for 'Completed' state, timeout in {0}...", timeout);
IEnumerable <CloudTask> addedTasks = batchClient.JobOperations.ListTasks(JobId);
batchClient.Utilities.CreateTaskStateMonitor().WaitAll(addedTasks, TaskState.Completed, timeout);
Console.WriteLine("All tasks reached state Completed.");
// Print task output
Console.WriteLine();
Console.WriteLine("Printing task output...");
IEnumerable <CloudTask> completedtasks = batchClient.JobOperations.ListTasks(JobId);
foreach (CloudTask task in completedtasks)
{
string nodeId = String.Format(task.ComputeNodeInformation.ComputeNodeId);
Console.WriteLine("Task: {0}", task.Id);
Console.WriteLine("Node: {0}", nodeId);
Console.WriteLine("Standard out:");
Console.WriteLine(task.GetNodeFile(Constants.StandardOutFileName).ReadAsString());
}
// Print out some timing info
timer.Stop();
Console.WriteLine();
Console.WriteLine("Sample end: {0}", DateTime.Now);
Console.WriteLine("Elapsed time: {0}", timer.Elapsed);
// Clean up Storage resources
container.DeleteIfExistsAsync().Wait();
Console.WriteLine("Container [{0}] deleted.", inputContainerName);
// Clean up Batch resources (if the user so chooses)
Console.WriteLine();
Console.Write("Delete job? [yes] no: ");
string response = Console.ReadLine().ToLower();
if (response != "n" && response != "no")
{
batchClient.JobOperations.DeleteJob(JobId);
}
Console.Write("Delete pool? [yes] no: ");
response = Console.ReadLine().ToLower();
if (response != "n" && response != "no")
{
batchClient.PoolOperations.DeletePool(PoolId);
}
}
}
finally
{
Console.WriteLine();
Console.WriteLine("Sample complete, hit ENTER to exit...");
Console.ReadLine();
}
}
/// <summary>
/// calls the two new get-status REST APIs and asserts their values
///
/// 1: add a single quick task (quick because we don't need it to run very long)
/// 2: this forces a victim compute node to run the JobPrep
/// 3: poll for this compute node, ignore others (sharedPool.size probably > 1)
/// 4: check status of JobPrep
/// 4a: assert as many values as makes sense... this is not a retry test
/// 5: JobPrep succeeds, task runs
/// 6: poll for JobRelease.. it is long running
/// 7: assert as many values as makes sense.
/// </summary>
/// <param name="batchCli"></param>
private void TestGetPrepReleaseStatusCalls(BatchClient batchCli, CloudJobSchedule boundJobSchedule, string sharedPool, IEnumerable <ResourceFile> correctResFiles)
{
// need this often enough lets just pull it out
string jobId = boundJobSchedule.ExecutionInformation.RecentJob.Id;
PoolOperations poolOps = batchCli.PoolOperations;
JobScheduleOperations jobScheduleOperations = batchCli.JobScheduleOperations;
{
DateTime beforeJobPrepRuns = DateTime.UtcNow; // used to test start time
// need a task to force JobPrep
CloudTask sillyTask = new CloudTask("forceJobPrep", "cmd /c hostname");
// add the task
batchCli.JobOperations.AddTask(jobId, sillyTask);
bool keepLooking = true;
while (keepLooking)
{
testOutputHelper.WriteLine("Waiting for task to be scheduled.");
foreach (CloudTask curTask in batchCli.JobOperations.GetJob(jobId).ListTasks())
{
if (curTask.State != TaskState.Active)
{
keepLooking = false;
break;
}
}
Thread.Sleep(1000);
}
List <JobPreparationAndReleaseTaskExecutionInformation> jobPrepStatusList = new List <JobPreparationAndReleaseTaskExecutionInformation>();
while (jobPrepStatusList.Count == 0)
{
jobPrepStatusList = batchCli.JobOperations.ListJobPreparationAndReleaseTaskStatus(jobId).ToList();
}
JobPreparationAndReleaseTaskExecutionInformation jptei = jobPrepStatusList.First();
ComputeNode victimComputeNodeRunningPrepAndRelease = poolOps.GetComputeNode(sharedPool, jptei.ComputeNodeId);
// job prep tests
{
Assert.NotNull(jptei);
Assert.Equal(0, jptei.JobPreparationTaskExecutionInformation.RetryCount);
Assert.True(beforeJobPrepRuns < jptei.JobPreparationTaskExecutionInformation.StartTime + TimeSpan.FromSeconds(10)); // test that the start time is rational -- 10s of wiggle room
Assert.Null(jptei.JobPreparationTaskExecutionInformation.FailureInformation);
testOutputHelper.WriteLine("");
testOutputHelper.WriteLine("listing files for compute node: " + victimComputeNodeRunningPrepAndRelease.Id);
// fiter the list so reduce noise
List <NodeFile> filteredListJobPrep = new List <NodeFile>();
foreach (NodeFile curTF in victimComputeNodeRunningPrepAndRelease.ListNodeFiles(recursive: true))
{
// filter on the jsId since we only run one job per job in this test.
if (curTF.Path.IndexOf(boundJobSchedule.Id, StringComparison.InvariantCultureIgnoreCase) >= 0)
{
testOutputHelper.WriteLine(" name:" + curTF.Path + ", size: " + ((curTF.IsDirectory.HasValue && curTF.IsDirectory.Value) ? "<dir>" : curTF.Properties.ContentLength.ToString()));
filteredListJobPrep.Add(curTF);
}
}
// confirm resource files made it
foreach (ResourceFile curCorrectRF in correctResFiles)
{
bool found = false;
foreach (NodeFile curTF in filteredListJobPrep)
{
// look for the resfile filepath in the taskfile name
found |= curTF.Path.IndexOf(curCorrectRF.FilePath, StringComparison.InvariantCultureIgnoreCase) >= 0;
}
Assert.True(found, "Looking for resourcefile: " + curCorrectRF.FilePath);
}
// poll for completion
while (JobPreparationTaskState.Completed != jptei.JobPreparationTaskExecutionInformation.State)
{
testOutputHelper.WriteLine("waiting for jopPrep to complete");
Thread.Sleep(2000);
// refresh the state info
ODATADetailLevel detailLevel = new ODATADetailLevel()
{
FilterClause = string.Format("nodeId eq '{0}'", victimComputeNodeRunningPrepAndRelease.Id)
};
jobPrepStatusList = batchCli.JobOperations.ListJobPreparationAndReleaseTaskStatus(jobId, detailLevel: detailLevel).ToList();
jptei = jobPrepStatusList.First();
}
// need success
Assert.Equal(0, jptei.JobPreparationTaskExecutionInformation.ExitCode);
// check stdout to confirm prep ran
//Why do I have to use the hardcoded string job-1 here...?
string stdOutFileSpec = Path.Combine("workitems", boundJobSchedule.Id, "job-1", boundJobSchedule.JobSpecification.JobPreparationTask.Id, Constants.StandardOutFileName);
string stdOut = victimComputeNodeRunningPrepAndRelease.GetNodeFile(stdOutFileSpec).ReadAsString();
string stdErrFileSpec = Path.Combine("workitems", boundJobSchedule.Id, "job-1", boundJobSchedule.JobSpecification.JobPreparationTask.Id, Constants.StandardErrorFileName);
string stdErr = string.Empty;
try
{
stdErr = victimComputeNodeRunningPrepAndRelease.GetNodeFile(stdErrFileSpec).ReadAsString();
}
catch (Exception)
{
//Swallow any exceptions here since stderr may not exist
}
testOutputHelper.WriteLine(stdOut);
testOutputHelper.WriteLine(stdErr);
Assert.True(!string.IsNullOrWhiteSpace(stdOut));
Assert.Contains("jobpreparation", stdOut.ToLower());
}
// jobPrep tests completed. let JobPrep complete and task run and wait for JobRelease
TaskStateMonitor tsm = batchCli.Utilities.CreateTaskStateMonitor();
// spam/logging interceptor
Protocol.RequestInterceptor consoleSpammer =
new Protocol.RequestInterceptor((x) =>
{
testOutputHelper.WriteLine("TestGetPrepReleaseStatusCalls: waiting for JobPrep and task to complete");
ODATADetailLevel detailLevel = new ODATADetailLevel()
{
FilterClause = string.Format("nodeId eq '{0}'", victimComputeNodeRunningPrepAndRelease.Id)
};
jobPrepStatusList = batchCli.JobOperations.ListJobPreparationAndReleaseTaskStatus(jobId, detailLevel: detailLevel).ToList();
JobPreparationAndReleaseTaskExecutionInformation jpteiInterceptor =
jobPrepStatusList.First();
testOutputHelper.WriteLine(" JobPrep.State: " + jpteiInterceptor.JobPreparationTaskExecutionInformation.State);
testOutputHelper.WriteLine("");
});
// waiting for the task to complete means so JobRelease is run.
tsm.WaitAll(
batchCli.JobOperations.GetJob(jobId).ListTasks(additionalBehaviors: new[] { consoleSpammer }),
TaskState.Completed,
TimeSpan.FromSeconds(120),
additionalBehaviors: new[] { consoleSpammer });
// trigger JobRelease
batchCli.JobOperations.TerminateJob(jobId, terminateReason: "die! I want JobRelease to run!");
// now that the task has competed, we are racing with the JobRelease... but it is sleeping so we can can catch it
while (true)
{
ODATADetailLevel detailLevel = new ODATADetailLevel()
{
FilterClause = string.Format("nodeId eq '{0}'", victimComputeNodeRunningPrepAndRelease.Id)
};
jobPrepStatusList = batchCli.JobOperations.ListJobPreparationAndReleaseTaskStatus(jobId, detailLevel: detailLevel).ToList();
JobPreparationAndReleaseTaskExecutionInformation jrtei = jobPrepStatusList.FirstOrDefault();
if ((jrtei == null) || (null == jrtei.JobReleaseTaskExecutionInformation))
{
Thread.Sleep(2000);
}
else
{
Assert.NotNull(jrtei);
if (jrtei.JobReleaseTaskExecutionInformation.State != JobReleaseTaskState.Completed)
{
testOutputHelper.WriteLine("JobReleaseTask state is: " + jrtei.JobReleaseTaskExecutionInformation.State);
Thread.Sleep(5000);
}
else
{
testOutputHelper.WriteLine("JobRelease commpleted!");
// we are done
break;
}
}
}
}
}
/// <summary>
/// Creates a job and adds a task to it. The task is a
/// custom executable which has a resource file associated with it.
/// </summary>
/// <param name="batchClient">The BatchClient to use when interacting with the Batch service.</param>
/// <param name="cloudStorageAccount">The storage account to upload the files to.</param>
/// <param name="jobId">The ID of the job.</param>
/// <returns>The set of container names containing the jobs input files.</returns>
private async Task <bool> SubmitJobIfNotExistAsync(BatchClient batchClient, CloudStorageAccount cloudStorageAccount, TestJob testJob)
{
CloudJob batchJob = batchClient.JobOperations.CreateJob();
batchJob.Id = testJob.BatchJobId;
batchJob.PoolInformation = new PoolInformation()
{
PoolId = TestJob.BatchPoolId
};
bool jobExists = false;
try
{
await batchJob.CommitAsync();
}
catch (BatchException e)
{
if (e.RequestInformation != null &&
e.RequestInformation.BatchError != null &&
e.RequestInformation.BatchError.Code == BatchErrorCodeStrings.JobExists)
{
jobExists = true;
}
else
{
throw;
}
}
if (jobExists)
{
// check whether task are there
var job = await batchClient.JobOperations.GetJobAsync(testJob.BatchJobId);
if (job != null && job.ListTasks().Count() > 0)
{
return(true);
}
}
var tasksToRun = new List <CloudTask>();
Messenger.Default.Send($"{DateTime.Now.ToString("T")} - Upload assemblies for batch", "RunningLog");
Console.WriteLine($"{DateTime.Now.ToString("T")} - Upload assemblies for batch");
var resourceFiles = await UploadFilesToContainerAsync(cloudStorageAccount);
Messenger.Default.Send(
new DeployStatusUpdateMessage()
{
Phase = DeployPhase.AssemblyUploaded,
Status = PhaseStatus.Succeeded
},
"DeployStatus"
);
var jobsPerVm = 4 * (int)Math.Pow(2, (int)testJob.SizeOfVM);
for (int i = 0; i < jobsPerVm * testJob.NumofVm; i++)
{
if (string.IsNullOrEmpty(testJob.Message))
{
testJob.Message = string.Empty;
}
var command = string.Format("DeviceLoad.exe {0} {1} {2} {3} {4} {5} {6}-{7} \"{8}\" {9}",
StorageConnectionString,
testJob.DeviceClientEndpoint,
testJob.DevicePerVm / jobsPerVm,
testJob.MessagePerMin,
testJob.DurationInMin,
testJob.BatchJobId,
configurationProvider.GetConfigValue("DeviceIdPrefix"),
i.ToString().PadLeft(4, '0'),
testJob.Message.Replace("\"", "\\\""),
testJob.Transport);
CloudTask taskWithFiles = new CloudTask("deviceTest" + i, command);
taskWithFiles.ResourceFiles = resourceFiles;
await batchClient.JobOperations.AddTaskAsync(testJob.BatchJobId, taskWithFiles);
}
return(true);
}
/// <summary>
/// Adds a task.
/// </summary>
/// <param name="options">The options describing the task to add.</param>
/// <returns></returns>
public async Task AddTaskAsync(AddTaskOptions options)
{
using (IWorkItemManager workItemManager = this.Client.OpenWorkItemManager())
{
ICloudTask unboundTask = new CloudTask(options.TaskName, options.CommandLine);
await workItemManager.AddTaskAsync(options.WorkItemName, options.JobName, unboundTask);
}
}
/// <summary>
/// Returns job preparation and main Batch tasks that represents the given <see cref="TesTask"/>
/// </summary>
/// <param name="task">The <see cref="TesTask"/></param>
/// <returns>Job preparation and main Batch tasks</returns>
private async Task <CloudTask> ConvertTesTaskToBatchTaskAsync(TesTask task)
{
var cromwellPathPrefixWithoutEndSlash = CromwellPathPrefix.TrimEnd('/');
var taskId = task.Id;
var queryStringsToRemoveFromLocalFilePaths = task.Inputs
.Select(i => i.Path)
.Concat(task.Outputs.Select(o => o.Path))
.Where(p => p != null)
.Select(p => queryStringRegex.Match(p).Groups[1].Value)
.Where(qs => !string.IsNullOrEmpty(qs))
.ToList();
var inputFiles = task.Inputs.Distinct();
var cromwellExecutionDirectoryPath = GetParentPath(task.Inputs.FirstOrDefault(IsCromwellCommandScript)?.Path);
if (cromwellExecutionDirectoryPath == null)
{
throw new Exception($"Could not identify Cromwell execution directory path for task {task.Id}. This TES instance supports Cromwell tasks only.");
}
foreach (var output in task.Outputs)
{
if (!output.Path.StartsWith(CromwellPathPrefix, StringComparison.OrdinalIgnoreCase))
{
throw new Exception($"Unsupported output path '{output.Path}' for task Id {task.Id}. Must start with {CromwellPathPrefix}");
}
}
var batchExecutionDirectoryPath = $"{cromwellExecutionDirectoryPath}/{BatchExecutionDirectoryName}";
// TODO: Cromwell bug: Cromwell command write_tsv() generates a file in the execution directory, for example execution/write_tsv_3922310b441805fc43d52f293623efbc.tmp. These are not passed on to TES inputs.
// WORKAROUND: Get the list of files in the execution directory and add them to task inputs.
var executionDirectoryUri = new Uri(await MapLocalPathToSasUrlAsync(cromwellExecutionDirectoryPath, getContainerSas: true));
var blobsInExecutionDirectory = (await azureProxy.ListBlobsAsync(executionDirectoryUri)).Where(b => !b.EndsWith($"/{CromwellScriptFileName}")).Where(b => !b.Contains($"/{BatchExecutionDirectoryName}/"));
var additionalInputFiles = blobsInExecutionDirectory.Select(b => $"{CromwellPathPrefix}{b}").Select(b => new TesInput {
Content = null, Path = b, Url = b, Name = Path.GetFileName(b), Type = TesFileType.FILEEnum
});
var filesToDownload = await Task.WhenAll(inputFiles.Union(additionalInputFiles).Select(async f => await GetTesInputFileUrl(f, task.Id, queryStringsToRemoveFromLocalFilePaths)));
var downloadFilesScriptContent = string.Join(" && ", filesToDownload.Select(f =>
f.Url.Contains(".blob.core.")
? $"blobxfer download --storage-url '{f.Url}' --local-path '{f.Path}' --chunk-size-bytes 104857600 --rename --no-recursive"
: $"mkdir -p {GetParentPath(f.Path)} && wget -O '{f.Path}' '{f.Url}'"));
var downloadFilesScriptPath = $"{batchExecutionDirectoryPath}/{DownloadFilesScriptFileName}";
var writableDownloadFilesScriptUrl = new Uri(await MapLocalPathToSasUrlAsync(downloadFilesScriptPath, getContainerSas: true));
var downloadFilesScriptUrl = await MapLocalPathToSasUrlAsync(downloadFilesScriptPath);
await azureProxy.UploadBlobAsync(writableDownloadFilesScriptUrl, downloadFilesScriptContent);
var filesToUpload = await Task.WhenAll(
task.Outputs.Select(async f =>
new TesOutput {
Path = f.Path, Url = await MapLocalPathToSasUrlAsync(f.Path, getContainerSas: true), Name = f.Name, Type = f.Type
}));
var uploadFilesScriptContent = string.Join(" && ", filesToUpload.Select(f =>
{
// Ignore missing stdout/stderr files. CWL workflows have an issue where if the stdout/stderr are redirected, they are still listed in the TES outputs
// Syntax is: If file doesn't exist, run a noop (":") operator , otherwise run the upload command
var fileExistsCheck = f.Path.EndsWith("/stdout") || f.Path.EndsWith("/stderr") ? $"[ ! -f '{f.Path}' ] && : || " : "";
return($"{fileExistsCheck}blobxfer upload --storage-url '{f.Url}' --local-path '{f.Path}' --one-shot-bytes 104857600 {(f.Type == TesFileType.FILEEnum ? "--rename --no-recursive" : "")}");
}));
var uploadFilesScriptPath = $"{batchExecutionDirectoryPath}/{UploadFilesScriptFileName}";
var writableUploadFilesScriptUrl = new Uri(await MapLocalPathToSasUrlAsync(uploadFilesScriptPath, getContainerSas: true));
var uploadFilesScriptUrl = await MapLocalPathToSasUrlAsync(uploadFilesScriptPath);
await azureProxy.UploadBlobAsync(writableUploadFilesScriptUrl, uploadFilesScriptContent);
var executor = task.Executors.First();
var volumeMountsOption = $"-v /mnt{cromwellPathPrefixWithoutEndSlash}:{cromwellPathPrefixWithoutEndSlash}";
var executorImageIsPublic = (await azureProxy.GetContainerRegistryInfoAsync(executor.Image)) == null;
var taskCommand = $@"
docker pull --quiet {BlobxferImageName} && \
{(executorImageIsPublic ? $"docker pull --quiet {executor.Image} &&" : "")} \
docker run --rm {volumeMountsOption} --entrypoint=/bin/sh {BlobxferImageName} {downloadFilesScriptPath} && \
chmod -R o+rwx /mnt{cromwellPathPrefixWithoutEndSlash} && \
docker run --rm {volumeMountsOption} --entrypoint= --workdir / {executor.Image} {executor.Command[0]} -c '{ string.Join(" && ", executor.Command.Skip(1))}' && \
docker run --rm {volumeMountsOption} --entrypoint=/bin/sh {BlobxferImageName} {uploadFilesScriptPath}
";
var batchExecutionDirectoryUrl = await MapLocalPathToSasUrlAsync($"{batchExecutionDirectoryPath}", getContainerSas : true);
var cloudTask = new CloudTask(taskId, $"/bin/sh -c \"{taskCommand.Trim()}\"")
{
UserIdentity = new UserIdentity(new AutoUserSpecification(elevationLevel: ElevationLevel.Admin, scope: AutoUserScope.Pool)),
ResourceFiles = new List <ResourceFile> {
ResourceFile.FromUrl(downloadFilesScriptUrl, $"/mnt{downloadFilesScriptPath}"), ResourceFile.FromUrl(uploadFilesScriptUrl, $"/mnt{uploadFilesScriptPath}")
},
OutputFiles = new List <OutputFile> {
new OutputFile(
"../std*.txt",
new OutputFileDestination(new OutputFileBlobContainerDestination(batchExecutionDirectoryUrl)),
new OutputFileUploadOptions(OutputFileUploadCondition.TaskFailure))
}
};
if (!executorImageIsPublic)
{
// If the executor image is private, and in order to run multiple containers in the main task, the image has to be downloaded via pool ContainerConfiguration.
// This also requires that the main task runs inside a container. So we run the "docker" container that in turn runs other containers.
// If the executor image is public, there is no need for pool ContainerConfiguration and task can run normally, without being wrapped in a docker container.
// Volume mapping for docker.sock below allows the docker client in the container to access host's docker daemon.
var containerRunOptions = $"--rm -v /var/run/docker.sock:/var/run/docker.sock -v /mnt{cromwellPathPrefixWithoutEndSlash}:/mnt{cromwellPathPrefixWithoutEndSlash} ";
cloudTask.ContainerSettings = new TaskContainerSettings(DockerInDockerImageName, containerRunOptions);
}
return(cloudTask);
}
public static void Submit()
{
Log("Start submission process.");
state = null;
BatchSharedKeyCredentials cred = new BatchSharedKeyCredentials(Settings.batchEndpoint, Settings.batchAccount, Settings.batchKey);
using (BatchClient client = BatchClient.Open(cred)) // <-- connect to the cluster
{
#region job submission
string jobname = prefix + Environment.GetEnvironmentVariable("USERNAME") + "_" + DateTime.Now.ToString("yyyyMMdd-HHmmss");
PoolInformation pool = new PoolInformation();
pool.PoolId = Settings.poolname;
CloudJob job = client.JobOperations.CreateJob(jobname, pool); // <-- create a workitem that runs on pool "trdemo"
Log("Submitting...");
job.Commit();
jobName = jobname;
Log(string.Format("Job {0} created.", jobname));
job = client.JobOperations.GetJob(jobname);
Log("Analyzing input blobs...");
string inputcontainersas = StorageHelper.GetContainerSAS(Settings.inputContainer);
string outputcontainersas = StorageHelper.GetContainerSAS(Settings.outputContainer);
foreach (string blob in StorageHelper.ListBlobs(Settings.inputContainer))
{
string filename = System.IO.Path.GetFileName((new Uri(blob)).LocalPath);
string taskname = "task_" + System.IO.Path.GetFileNameWithoutExtension(filename);
// prepare the command line
string cli;
cli = ". robocopy.exe ${env:WATASK_TVM_ROOT_DIR}\\shared\\ . *.*;";
cli += "ffmpeg.exe -i {0} -vf 'movie=microsoft.png [watermark]; [in][watermark] overlay=10:main_h-overlay_h-10 [out]' {0}.output.avi;".Replace("{0}", filename);
cli += "azcopy.exe . {0} *.output.avi /destsas:'{1}' /y".Replace("{0}", Settings.outputContainer).Replace("{1}", outputcontainersas);
cli = string.Format("powershell -command \"{0}\"", cli);
// prepare task object
CloudTask task = new CloudTask(taskname, cli);
task.ResourceFiles = new List<ResourceFile>();
task.ResourceFiles.Add(new ResourceFile(blob + inputcontainersas, filename));
job.AddTask(task); // <-- add Task
}
#endregion job submission
ThreadPool.QueueUserWorkItem((x) => { Monitor(); });
client.Utilities.CreateTaskStateMonitor().WaitAll(client.JobOperations.ListTasks(jobname), TaskState.Completed, new TimeSpan(0, 60, 0));
client.JobOperations.GetJob(jobname).Terminate();
}
}
/// <summary>
/// Creates a job and adds a task to it. The task is a
/// custom executable which has a resource file associated with it.
/// </summary>
/// <param name="batchClient">The BatchClient to use when interacting with the Batch service.</param>
/// <param name="cloudStorageAccount">The storage account to upload the files to.</param>
/// <param name="jobId">The ID of the job.</param>
/// <returns>The set of container names containing the jobs input files.</returns>
private async Task<HashSet<string>> SubmitJobAsync(BatchClient batchClient, CloudStorageAccount cloudStorageAccount, string jobId)
{
// create an empty unbound Job
CloudJob unboundJob = batchClient.JobOperations.CreateJob();
unboundJob.Id = jobId;
unboundJob.PoolInformation = new PoolInformation() { PoolId = this.poolsAndResourceFileSettings.PoolId };
// Commit Job to create it in the service
await unboundJob.CommitAsync();
List<CloudTask> tasksToRun = new List<CloudTask>();
// Create a task which requires some resource files
CloudTask taskWithFiles = new CloudTask("task_with_file1", SimpleTaskExe);
// Set up a collection of files to be staged -- these files will be uploaded to Azure Storage
// when the tasks are submitted to the Azure Batch service.
taskWithFiles.FilesToStage = new List<IFileStagingProvider>();
// generate a local file in temp directory
string localSampleFile = Path.Combine(Environment.GetEnvironmentVariable("TEMP"), "HelloWorld.txt");
File.WriteAllText(localSampleFile, "hello from Batch PoolsAndResourceFiles sample!");
StagingStorageAccount fileStagingStorageAccount = new StagingStorageAccount(
storageAccount: this.accountSettings.StorageAccountName,
storageAccountKey: this.accountSettings.StorageAccountKey,
blobEndpoint: cloudStorageAccount.BlobEndpoint.ToString());
// add the files as a task dependency so they will be uploaded to storage before the task
// is submitted and downloaded to the node before the task starts execution.
FileToStage helloWorldFile = new FileToStage(localSampleFile, fileStagingStorageAccount);
FileToStage simpleTaskFile = new FileToStage(SimpleTaskExe, fileStagingStorageAccount);
// When this task is added via JobOperations.AddTaskAsync below, the FilesToStage are uploaded to storage once.
// The Batch service does not automatically delete content from your storage account, so files added in this
// way must be manually removed when they are no longer used.
taskWithFiles.FilesToStage.Add(helloWorldFile);
taskWithFiles.FilesToStage.Add(simpleTaskFile);
tasksToRun.Add(taskWithFiles);
var fileStagingArtifacts = new ConcurrentBag<ConcurrentDictionary<Type, IFileStagingArtifact>>();
// Use the AddTask method which takes an enumerable of tasks for best performance, as it submits up to 100
// tasks at once in a single request. If the list of tasks is N where N > 100, this will correctly parallelize
// the requests and return when all N tasks have been added.
await batchClient.JobOperations.AddTaskAsync(jobId, tasksToRun, fileStagingArtifacts: fileStagingArtifacts);
// Extract the names of the blob containers from the file staging artifacts
HashSet<string> blobContainerNames = GettingStartedCommon.ExtractBlobContainerNames(fileStagingArtifacts);
return blobContainerNames;
}
public static void JobMain(string[] args)
{
//Load the configuration
TopNWordsConfiguration configuration = TopNWordsConfiguration.LoadConfigurationFromAppConfig();
StagingStorageAccount stagingStorageAccount = new StagingStorageAccount(
configuration.StorageAccountName,
configuration.StorageAccountKey,
configuration.StorageAccountBlobEndpoint);
using (BatchClient client = BatchClient.Open(new BatchSharedKeyCredentials(configuration.BatchServiceUrl, configuration.BatchAccountName, configuration.BatchAccountKey)))
{
string stagingContainer = null;
//Create a pool (if user hasn't provided one)
if (configuration.ShouldCreatePool)
{
//OSFamily 4 == OS 2012 R2. You can learn more about os families and versions at:
//http://msdn.microsoft.com/en-us/library/azure/ee924680.aspx
CloudPool pool = client.PoolOperations.CreatePool(configuration.PoolId, targetDedicated: configuration.PoolSize, osFamily: "4", virtualMachineSize: "small");
Console.WriteLine("Adding pool {0}", configuration.PoolId);
try
{
pool.Commit();
}
catch (AggregateException ae)
{
// Go through all exceptions and dump useful information
ae.Handle(x =>
{
Console.Error.WriteLine("Creating pool ID {0} failed", configuration.PoolId);
if (x is BatchException)
{
BatchException be = x as BatchException;
Console.WriteLine(be.ToString());
Console.WriteLine();
}
else
{
Console.WriteLine(x);
}
// can't continue without a pool
return(false);
});
}
}
try
{
Console.WriteLine("Creating job: " + configuration.JobId);
// get an empty unbound Job
CloudJob unboundJob = client.JobOperations.CreateJob();
unboundJob.Id = configuration.JobId;
unboundJob.PoolInformation = new PoolInformation()
{
PoolId = configuration.PoolId
};
// Commit Job to create it in the service
unboundJob.Commit();
// create file staging objects that represent the executable and its dependent assembly to run as the task.
// These files are copied to every node before the corresponding task is scheduled to run on that node.
FileToStage topNWordExe = new FileToStage(TopNWordsExeName, stagingStorageAccount);
FileToStage storageDll = new FileToStage(StorageClientDllName, stagingStorageAccount);
// In this sample, the input data is copied separately to Storage and its URI is passed to the task as an argument.
// This approach is appropriate when the amount of input data is large such that copying it to every node via FileStaging
// is not desired and the number of tasks is small since a large number of readers of the blob might get throttled
// by Storage which will lengthen the overall processing time.
//
// You'll need to observe the behavior and use published techniques for finding the right balance of performance versus
// complexity.
string bookFileUri = UploadBookFileToCloudBlob(configuration, configuration.BookFileName);
Console.WriteLine("{0} uploaded to cloud", configuration.BookFileName);
// initialize a collection to hold the tasks that will be submitted in their entirety
List <CloudTask> tasksToRun = new List <CloudTask>(configuration.NumberOfTasks);
for (int i = 1; i <= configuration.NumberOfTasks; i++)
{
CloudTask task = new CloudTask("task_no_" + i, String.Format("{0} --Task {1} {2} {3} {4}",
TopNWordsExeName,
bookFileUri,
configuration.NumberOfTopWords,
configuration.StorageAccountName,
configuration.StorageAccountKey));
//This is the list of files to stage to a container -- for each job, one container is created and
//files all resolve to Azure Blobs by their name (so two tasks with the same named file will create just 1 blob in
//the container).
task.FilesToStage = new List <IFileStagingProvider>
{
topNWordExe,
storageDll
};
tasksToRun.Add(task);
}
// Commit all the tasks to the Batch Service. Ask AddTask to return information about the files that were staged.
// The container information is used later on to remove these files from Storage.
ConcurrentBag <ConcurrentDictionary <Type, IFileStagingArtifact> > fsArtifactBag = new ConcurrentBag <ConcurrentDictionary <Type, IFileStagingArtifact> >();
client.JobOperations.AddTask(configuration.JobId, tasksToRun, fileStagingArtifacts: fsArtifactBag);
// loop through the bag of artifacts, looking for the one that matches our staged files. Once there,
// capture the name of the container holding the files so they can be deleted later on if that option
// was configured in the settings.
foreach (var fsBagItem in fsArtifactBag)
{
IFileStagingArtifact fsValue;
if (fsBagItem.TryGetValue(typeof(FileToStage), out fsValue))
{
SequentialFileStagingArtifact stagingArtifact = fsValue as SequentialFileStagingArtifact;
if (stagingArtifact != null)
{
stagingContainer = stagingArtifact.BlobContainerCreated;
Console.WriteLine(
"Uploaded files to container: {0} -- you will be charged for their storage unless you delete them.",
stagingArtifact.BlobContainerCreated);
}
}
}
//Get the job to monitor status.
CloudJob job = client.JobOperations.GetJob(configuration.JobId);
Console.Write("Waiting for tasks to complete ... ");
// Wait 20 minutes for all tasks to reach the completed state. The long timeout is necessary for the first
// time a pool is created in order to allow nodes to be added to the pool and initialized to run tasks.
IPagedEnumerable <CloudTask> ourTasks = job.ListTasks(new ODATADetailLevel(selectClause: "id"));
client.Utilities.CreateTaskStateMonitor().WaitAll(ourTasks, TaskState.Completed, TimeSpan.FromMinutes(20));
Console.WriteLine("tasks are done.");
foreach (CloudTask t in ourTasks)
{
Console.WriteLine("Task " + t.Id);
Console.WriteLine("stdout:" + Environment.NewLine + t.GetNodeFile(Constants.StandardOutFileName).ReadAsString());
Console.WriteLine();
Console.WriteLine("stderr:" + Environment.NewLine + t.GetNodeFile(Constants.StandardErrorFileName).ReadAsString());
}
}
finally
{
//Delete the pool that we created
if (configuration.ShouldCreatePool)
{
Console.WriteLine("Deleting pool: {0}", configuration.PoolId);
client.PoolOperations.DeletePool(configuration.PoolId);
}
//Delete the job that we created
if (configuration.ShouldDeleteJob)
{
Console.WriteLine("Deleting job: {0}", configuration.JobId);
client.JobOperations.DeleteJob(configuration.JobId);
}
//Delete the containers we created
if (configuration.ShouldDeleteContainer)
{
DeleteContainers(configuration, stagingContainer);
}
}
}
}
private static async Task TaskCreation(BatchClient p_batchClient, string p_jobId)
{
Console.WriteLine("Creating the Task");
string taskId = "demotask";
string in_container_name = "input";
string out_container_name = "output";
string l_blobName = "sample.mp4";
string outputfile = "audio.aac";
string storageConnectionString = String.Format("DefaultEndpointsProtocol=https;AccountName={0};AccountKey={1}",
demo_storageAccountName, demo_storageAccountKey);
CloudStorageAccount l_storageAccount = CloudStorageAccount.Parse(storageConnectionString);
CloudBlobClient l_blobClient = l_storageAccount.CreateCloudBlobClient();
CloudBlobContainer in_container = l_blobClient.GetContainerReference(in_container_name);
CloudBlobContainer out_container = l_blobClient.GetContainerReference(out_container_name);
SharedAccessBlobPolicy i_sasConstraints = new SharedAccessBlobPolicy
{
SharedAccessExpiryTime = DateTime.UtcNow.AddHours(2),
Permissions = SharedAccessBlobPermissions.Read | SharedAccessBlobPermissions.List
};
SharedAccessBlobPolicy o_sasConstraints = new SharedAccessBlobPolicy
{
SharedAccessExpiryTime = DateTime.UtcNow.AddHours(2),
Permissions = SharedAccessBlobPermissions.Write
};
string in_sasToken = in_container.GetSharedAccessSignature(i_sasConstraints);
string in_containerSasUrl = String.Format("{0}{1}", in_container.Uri, in_sasToken);
string out_sasToken = out_container.GetSharedAccessSignature(o_sasConstraints);
string out_containerSasUrl = String.Format("{0}{1}", out_container.Uri, out_sasToken);
ResourceFile inputFile = ResourceFile.FromStorageContainerUrl(in_containerSasUrl);
List <ResourceFile> file = new List <ResourceFile>();
file.Add(inputFile);
string appPath = String.Format("%AZ_BATCH_APP_PACKAGE_{0}#{1}%", demo_packageid, demo_packageversion);
string taskCommandLine = String.Format("cmd /c {0}\\ffmpeg.exe -i {1} -vn -acodec copy audio.aac", appPath, l_blobName);
CloudTask task = new CloudTask(taskId, taskCommandLine);
task.ResourceFiles = file;
// Setting the output file location
List <OutputFile> outputFileList = new List <OutputFile>();
OutputFileBlobContainerDestination outputContainer = new OutputFileBlobContainerDestination(out_containerSasUrl);
OutputFile outputFile = new OutputFile(outputfile,
new OutputFileDestination(outputContainer),
new OutputFileUploadOptions(OutputFileUploadCondition.TaskSuccess));
outputFileList.Add(outputFile);
task.OutputFiles = outputFileList;
await p_batchClient.JobOperations.AddTaskAsync(p_jobId, task);
}
public void LongRunning_Bug1965363Wat7OSVersionFeaturesQuickJobWithAutoPool()
{
Action test = () =>
{
using (BatchClient batchCli = TestUtilities.OpenBatchClient(TestUtilities.GetCredentialsFromEnvironment()))
{
string jobId = "Bug1965363Job-" + TestUtilities.GetMyName();
try
{
PoolInformation poolInfo = new PoolInformation()
{
AutoPoolSpecification = new AutoPoolSpecification()
{
PoolLifetimeOption = PoolLifetimeOption.Job,
PoolSpecification = new PoolSpecification()
{
CloudServiceConfiguration = new CloudServiceConfiguration(PoolFixture.OSFamily),
VirtualMachineSize = PoolFixture.VMSize,
TargetDedicatedComputeNodes = 1
}
}
};
CloudJob unboundJob = batchCli.JobOperations.CreateJob(jobId, poolInfo);
this.testOutputHelper.WriteLine("Commiting quickjob");
unboundJob.Commit();
CloudTask task = new CloudTask("Bug1965363Wat7OSVersionFeaturesQuickJobWithAutoPoolTask", "cmd /c echo Bug1965363");
CloudJob boundJob = batchCli.JobOperations.GetJob(jobId);
boundJob.AddTask(task);
this.testOutputHelper.WriteLine("Getting pool name: {0}", boundJob.ExecutionInformation.PoolId);
CloudPool boundPool = batchCli.PoolOperations.GetPool(boundJob.ExecutionInformation.PoolId);
TaskStateMonitor tsm = batchCli.Utilities.CreateTaskStateMonitor();
ODATAMonitorControl odControl = new ODATAMonitorControl();
// we know that the autopool compute nodes will take a long time to become scheduleable so we slow down polling/spam
odControl.DelayBetweenDataFetch = TimeSpan.FromSeconds(5);
this.testOutputHelper.WriteLine("Invoking TaskStateMonitor");
tsm.WaitAll(
boundJob.ListTasks(),
TaskState.Completed,
TimeSpan.FromMinutes(15),
odControl,
new[] {
// spam/logging interceptor
new Microsoft.Azure.Batch.Protocol.RequestInterceptor((x) =>
{
this.testOutputHelper.WriteLine("Issuing request type: " + x.GetType().ToString());
// print out the compute node states... we are actually waiting on the compute nodes
List <ComputeNode> allComputeNodes = boundPool.ListComputeNodes().ToList();
this.testOutputHelper.WriteLine(" #comnpute nodes: " + allComputeNodes.Count);
allComputeNodes.ForEach((icn) => { this.testOutputHelper.WriteLine(" computeNode.id: " + icn.Id + ", state: " + icn.State); });
this.testOutputHelper.WriteLine("");
})
});
// confirm the task ran by inspecting the stdOut
string stdOut = boundJob.ListTasks().ToList()[0].GetNodeFile(Constants.StandardOutFileName).ReadAsString();
Assert.Contains("Bug1965363", stdOut);
}
finally
{
TestUtilities.DeleteJobIfExistsAsync(batchCli, jobId).Wait();
}
}
};
SynchronizationContextHelper.RunTest(test, LongTestTimeout);
}
public async Task UnboundTaskRefreshFails()
{
var task = new CloudTask("Foo", "Bar");
await Assert.ThrowsAsync <InvalidOperationException>(async() => await task.RefreshAsync());
}
/// <summary>
/// Submit a large number of tasks to the Batch Service.
/// </summary>
/// <param name="client">The batch client.</param>
/// <param name="sharedPoolId">The ID of the pool to use for the job</param>
private static void SubmitLargeNumberOfTasks(BatchClient client, string sharedPoolId)
{
const int taskCountToCreate = 5000;
// In order to simulate a "large" task object which has many properties set (such as resource files, environment variables, etc)
// we create a big environment variable so we have a big task object.
char[] env = new char[2048];
for (int i = 0; i < env.Length; i++)
{
env[i] = 'a';
}
string envStr = new string(env);
string jobId = CreateJobId("HelloWorldLargeTaskCountJob");
Console.WriteLine("Creating job: " + jobId);
CloudJob boundJob = CreateBoundJob(client.JobOperations, sharedPoolId, jobId);
//Generate a large number of tasks to submit
List<CloudTask> tasksToSubmit = new List<CloudTask>(taskCountToCreate);
for (int i = 0; i < taskCountToCreate; i++)
{
CloudTask task = new CloudTask("echo" + i.ToString("D5"), "echo");
List<EnvironmentSetting> environmentSettings = new List<EnvironmentSetting>();
environmentSettings.Add(new EnvironmentSetting("envone", envStr));
task.EnvironmentSettings = environmentSettings;
tasksToSubmit.Add(task);
}
BatchClientParallelOptions parallelOptions = new BatchClientParallelOptions()
{
//This will result in at most 10 simultaneous Bulk Add requests to the Batch Service.
MaxDegreeOfParallelism = 10
};
Console.WriteLine("Submitting {0} tasks to job: {1}, on pool: {2}",
taskCountToCreate,
boundJob.Id,
boundJob.ExecutionInformation.PoolId);
Stopwatch stopwatch = new Stopwatch();
stopwatch.Start();
// Use the AddTask overload which supports a list of tasks for best AddTask performence - internally this method performs a
// submission of multiple tasks in one REST API request in order to limit the number of calls made to the Batch Service.
client.JobOperations.AddTask(boundJob.Id, tasksToSubmit, parallelOptions);
stopwatch.Stop();
Console.WriteLine("Submitted {0} tasks in {1}", taskCountToCreate, stopwatch.Elapsed);
//Delete the job to ensure the tasks are cleaned up
Console.WriteLine("Deleting job: {0}", boundJob.Id);
client.JobOperations.DeleteJob(boundJob.Id);
}
/// <summary>
/// Creates a new task.
/// </summary>
/// <param name="parameters">The parameters to use when creating the task.</param>
public void CreateTask(NewTaskParameters parameters)
{
if (parameters == null)
{
throw new ArgumentNullException("parameters");
}
CloudTask task = new CloudTask(parameters.TaskId, parameters.CommandLine);
task.DisplayName = parameters.DisplayName;
if (parameters.AuthenticationTokenSettings != null)
{
task.AuthenticationTokenSettings = parameters.AuthenticationTokenSettings.omObject;
}
if (parameters.UserIdentity != null)
{
task.UserIdentity = parameters.UserIdentity.omObject;
}
if (parameters.EnvironmentSettings != null)
{
task.EnvironmentSettings = new List <EnvironmentSetting>();
foreach (DictionaryEntry d in parameters.EnvironmentSettings)
{
EnvironmentSetting setting = new EnvironmentSetting(d.Key.ToString(), d.Value.ToString());
task.EnvironmentSettings.Add(setting);
}
}
if (parameters.ResourceFiles != null)
{
task.ResourceFiles = parameters.ResourceFiles.Select(file => file.omObject).ToList();
}
if (parameters.AffinityInformation != null)
{
task.AffinityInformation = parameters.AffinityInformation.omObject;
}
if (parameters.Constraints != null)
{
task.Constraints = parameters.Constraints.omObject;
}
if (parameters.DependsOn != null)
{
task.DependsOn = parameters.DependsOn;
}
if (parameters.MultiInstanceSettings != null)
{
Utils.Utils.MultiInstanceSettingsSyncCollections(parameters.MultiInstanceSettings);
task.MultiInstanceSettings = parameters.MultiInstanceSettings.omObject;
}
if (parameters.ApplicationPackageReferences != null)
{
task.ApplicationPackageReferences = parameters.ApplicationPackageReferences.ToList().ConvertAll(apr => apr.omObject);
}
if (parameters.ExitConditions != null)
{
Utils.Utils.ExitConditionsSyncCollections(parameters.ExitConditions);
task.ExitConditions = parameters.ExitConditions.omObject;
}
if (parameters.OutputFiles != null)
{
task.OutputFiles = parameters.OutputFiles.ToList().ConvertAll(outputFile => outputFile.omObject);
}
task.ContainerSettings = parameters.ContainerSettings?.omObject;
WriteVerbose(string.Format(Resources.CreatingTask, parameters.TaskId));
if (parameters.Job != null)
{
parameters.Job.omObject.AddTask(task, parameters.AdditionalBehaviors);
}
else
{
JobOperations jobOperations = parameters.Context.BatchOMClient.JobOperations;
jobOperations.AddTask(parameters.JobId, task, parameters.AdditionalBehaviors);
}
}
static void AddTasks(BatchClient client, CloudJob cloudJob, string jobId, IEnumerable<IComputeTask> computeTasks)
{
foreach (var computeTask in computeTasks)
{
var definition = computeTask.Definition;
var executable = new ResourceFile($"{definition.StorageUri}/{definition.ExecutableName}", definition.ExecutableName);
var resources = definition.Resources.Select(resource => new ResourceFile($"{definition.StorageUri}/{resource}", resource));
var inputs = computeTask.Inputs.Select(input => new ResourceFile($"{definition.StorageUri}/{input}", input));
var resourceFiles = new List<ResourceFile> { executable };
resourceFiles.AddRange(resources);
resourceFiles.AddRange(inputs);
var task = client.JobOperations.ListTasks(jobId).SingleOrDefault(t => t.Id == computeTask.Id);
if (task == null)
{
task = new CloudTask(computeTask.Id, computeTask.CommandLine)
{
ResourceFiles = resourceFiles
};
cloudJob.AddTask(task);
cloudJob.Commit();
cloudJob.Refresh();
}
}
client.Utilities.CreateTaskStateMonitor().WaitAll(cloudJob.ListTasks(), TaskState.Completed, new TimeSpan(0, 30, 0));
}
public static void JobMain(string[] args)
{
//Load the configuration
TopNWordsConfiguration configuration = TopNWordsConfiguration.LoadConfigurationFromAppConfig();
StagingStorageAccount stagingStorageAccount = new StagingStorageAccount(
configuration.StorageAccountName,
configuration.StorageAccountKey,
configuration.StorageAccountBlobEndpoint);
IBatchClient client = BatchClient.Connect(configuration.BatchServiceUrl, new BatchCredentials(configuration.BatchAccountName, configuration.BatchAccountKey));
string stagingContainer = null;
//Create a pool (if user hasn't provided one)
if (configuration.ShouldCreatePool)
{
using (IPoolManager pm = client.OpenPoolManager())
{
//OSFamily 4 == OS 2012 R2
//You can learn more about os families and versions at:
//http://msdn.microsoft.com/en-us/library/azure/ee924680.aspx
ICloudPool pool = pm.CreatePool(configuration.PoolName, targetDedicated: configuration.PoolSize, osFamily: "4", vmSize: "small");
Console.WriteLine("Adding pool {0}", configuration.PoolName);
pool.Commit();
}
}
try
{
using (IWorkItemManager wm = client.OpenWorkItemManager())
{
IToolbox toolbox = client.OpenToolbox();
//Use the TaskSubmissionHelper to help us create a WorkItem and add tasks to it.
ITaskSubmissionHelper taskSubmissionHelper = toolbox.CreateTaskSubmissionHelper(wm, configuration.PoolName);
taskSubmissionHelper.WorkItemName = configuration.WorkItemName;
FileToStage topNWordExe = new FileToStage(TopNWordsExeName, stagingStorageAccount);
FileToStage storageDll = new FileToStage(StorageClientDllName, stagingStorageAccount);
string bookFileUri = UploadBookFileToCloudBlob(configuration, configuration.BookFileName);
Console.WriteLine("{0} uploaded to cloud", configuration.BookFileName);
for (int i = 1; i <= configuration.NumberOfTasks; i++)
{
ICloudTask task = new CloudTask("task_no_" + i, String.Format("{0} --Task {1} {2} {3} {4}",
TopNWordsExeName,
bookFileUri,
configuration.NumberOfTopWords,
configuration.StorageAccountName,
configuration.StorageAccountKey));
//This is the list of files to stage to a container -- for each TaskSubmissionHelper one container is created and
//files all resolve to Azure Blobs by their name (so two tasks with the same named file will create just 1 blob in
//the TaskSubmissionHelper's container).
task.FilesToStage = new List<IFileStagingProvider>
{
topNWordExe,
storageDll
};
taskSubmissionHelper.AddTask(task);
}
//Commit all the tasks to the Batch Service.
IJobCommitUnboundArtifacts artifacts = taskSubmissionHelper.Commit() as IJobCommitUnboundArtifacts;
foreach (var fileStagingArtifact in artifacts.FileStagingArtifacts)
{
SequentialFileStagingArtifact stagingArtifact = fileStagingArtifact.Value as SequentialFileStagingArtifact;
if (stagingArtifact != null)
{
stagingContainer = stagingArtifact.BlobContainerCreated;
Console.WriteLine("Uploaded files to container: {0} -- you will be charged for their storage unless you delete them.",
stagingArtifact.BlobContainerCreated);
}
}
//Get the job to monitor status.
ICloudJob job = wm.GetJob(artifacts.WorkItemName, artifacts.JobName);
Console.Write("Waiting for tasks to complete ...");
// Wait 1 minute for all tasks to reach the completed state
client.OpenToolbox().CreateTaskStateMonitor().WaitAll(job.ListTasks(), TaskState.Completed, TimeSpan.FromMinutes(20));
Console.WriteLine("Done.");
foreach (ICloudTask task in job.ListTasks())
{
Console.WriteLine("Task " + task.Name + " says:\n" + task.GetTaskFile(Constants.StandardOutFileName).ReadAsString());
Console.WriteLine(task.GetTaskFile(Constants.StandardErrorFileName).ReadAsString());
}
}
}
finally
{
//Delete the pool that we created
if (configuration.ShouldCreatePool)
{
using (IPoolManager pm = client.OpenPoolManager())
{
Console.WriteLine("Deleting pool: {0}", configuration.PoolName);
pm.DeletePool(configuration.PoolName);
}
}
//Delete the workitem that we created
if (configuration.ShouldDeleteWorkItem)
{
using (IWorkItemManager wm = client.OpenWorkItemManager())
{
Console.WriteLine("Deleting work item: {0}", configuration.WorkItemName);
wm.DeleteWorkItem(configuration.WorkItemName);
}
}
//Delete the containers we created
if(configuration.ShouldDeleteContainer)
{
DeleteContainers(configuration, stagingContainer);
}
}
}
/// <summary>
/// Adds a task.
/// </summary>
/// <param name="options">The options describing the task to add.</param>
/// <returns></returns>
public async Task AddTaskAsync(AddTaskOptions options)
{
CloudTask unboundTask = new CloudTask(options.TaskId, options.CommandLine);
if (options.IsMultiInstanceTask)
{
unboundTask.MultiInstanceSettings = new MultiInstanceSettings(options.InstanceNumber);
unboundTask.MultiInstanceSettings.CoordinationCommandLine = options.BackgroundCommand;
unboundTask.MultiInstanceSettings.CommonResourceFiles = options.CommonResourceFiles.ConvertAll(f => new ResourceFile(f.BlobSource, f.FilePath));
}
unboundTask.RunElevated = options.RunElevated;
unboundTask.Constraints = new TaskConstraints(null, null, options.MaxTaskRetryCount);
unboundTask.ResourceFiles = options.ResourceFiles.ConvertAll(f => new ResourceFile(f.BlobSource, f.FilePath));
await this.Client.JobOperations.AddTaskAsync(options.JobId, unboundTask);
}
/// <summary>
/// Populates Azure Storage with the required files, and
/// submits the job to the Azure Batch service.
/// </summary>
public async Task RunAsync()
{
Console.WriteLine("Running with the following settings: ");
Console.WriteLine("-------------------------------------");
Console.WriteLine(this.configurationSettings.ToString());
// Set up the Batch Service credentials used to authenticate with the Batch Service.
BatchSharedKeyCredentials credentials = new BatchSharedKeyCredentials(
this.configurationSettings.BatchServiceUrl,
this.configurationSettings.BatchAccountName,
this.configurationSettings.BatchAccountKey);
CloudStorageAccount cloudStorageAccount = new CloudStorageAccount(
new StorageCredentials(this.configurationSettings.StorageAccountName,
this.configurationSettings.StorageAccountKey),
new Uri(this.configurationSettings.StorageBlobEndpoint),
null,
null,
null);
// Get an instance of the BatchClient for a given Azure Batch account.
using (BatchClient batchClient = await BatchClient.OpenAsync(credentials))
{
// add a retry policy. The built-in policies are No Retry (default), Linear Retry, and Exponential Retry
batchClient.CustomBehaviors.Add(RetryPolicyProvider.LinearRetryProvider(TimeSpan.FromSeconds(10), 3));
string jobId = null;
try
{
// Allocate a pool
await this.CreatePoolIfNotExistAsync(batchClient, cloudStorageAccount);
// Submit the job
jobId = GettingStartedCommon.CreateJobId("SimpleJob");
await this.SubmitJobAsync(batchClient, cloudStorageAccount, jobId);
// Print out the status of the pools/jobs under this account
await GettingStartedCommon.PrintJobsAsync(batchClient);
await GettingStartedCommon.PrintPoolsAsync(batchClient);
// Wait for the job manager to complete
CloudTask jobManagerTask = await batchClient.JobOperations.GetTaskAsync(jobId, JobManagerTaskId);
await GettingStartedCommon.WaitForTasksAndPrintOutputAsync(batchClient, new List <CloudTask> {
jobManagerTask
}, TimeSpan.FromMinutes(10));
}
finally
{
// TODO: In C# 6 we can await here instead of .Wait()
// Delete Azure Batch resources
List <string> jobIdsToDelete = new List <string>();
List <string> poolIdsToDelete = new List <string>();
if (this.configurationSettings.ShouldDeleteJob)
{
jobIdsToDelete.Add(jobId);
}
if (this.configurationSettings.ShouldDeletePool)
{
poolIdsToDelete.Add(this.configurationSettings.PoolId);
}
SampleHelpers.DeleteBatchResourcesAsync(batchClient, jobIdsToDelete, poolIdsToDelete).Wait();
}
}
}
/// <summary>
/// This is the client that creates workitem and submits tasks.
/// </summary>
/// <param name="args"></param>
public static void SubmitTasks(string[] args)
{
config = Config.ParseConfig();
//Upload resources if specified
if (config.UploadResources)
{
//Upload ImgProc.exe, Batch.dll and the Storage Client
ImgProcUtils.UploadFileToBlob(Constants.StorageClientDllName, config.ResourceContainerSAS);
ImgProcUtils.UploadFileToBlob(Constants.ImgProcExeName, config.ResourceContainerSAS);
ImgProcUtils.UploadFileToBlob(Constants.BatchClientDllName, config.ResourceContainerSAS);
Console.WriteLine("Done uploading files to blob");
}
try
{
using (IWorkItemManager wm = config.Client.OpenWorkItemManager())
{
IToolbox toolbox = config.Client.OpenToolbox();
//Use the task submission helper to ease creation of workitem and addition of tasks, as well as management of resource file staging.
ITaskSubmissionHelper taskSubmissionHelper = toolbox.CreateTaskSubmissionHelper(wm, config.PoolName);
taskSubmissionHelper.WorkItemName = config.WorkitemName;
//Compute the number of images each task should process
int numImgsPerTask = (int)Math.Round(config.NumInputBlobs / (decimal)config.NumTasks);
for (int i = 0; i < config.NumTasks; i++)
{
ICloudTask task = new CloudTask(
name: "task_no_" + i,
commandline: string.Format("{0} --Task {1} thumb{2}", Constants.ImgProcExeName, config.OutputContainerSAS, i));
Console.WriteLine("Generating task: {0}", task.Name);
task.FilesToStage = new List<IFileStagingProvider>();
int start = i * numImgsPerTask;
int end;
if (i < config.NumTasks - 1)
{
end = ((i + 1) * numImgsPerTask) - 1;
}
else
{
end = config.NumInputBlobs - 1;
}
//Generate and set up the list of files to be processed by this task
for (int j = start; j < end; j++)
{
string input = GetTempFilePath(j);
ImgProcUtils.GenerateImages(input, string.Format("{0}", j));
task.FilesToStage.Add(new FileToStage(input, new StagingStorageAccount(config.StorageAccount, config.StorageKey, config.StorageBlobEndpoint)));
}
task.ResourceFiles = ImgProcUtils.GetResourceFiles(config.ResourceContainerSAS);
taskSubmissionHelper.AddTask(task);
}
IJobCommitUnboundArtifacts artifacts = null;
try
{
Console.WriteLine("Submitting {0} tasks to the Batch Service", config.NumTasks);
//Submit the tasks to the Batch Service
artifacts = taskSubmissionHelper.Commit() as IJobCommitUnboundArtifacts;
}
catch (AggregateException ae)
{
// Go through all exceptions and dump useful information
ae.Handle(x =>
{
if (x is BatchException)
{
BatchException be = x as BatchException;
if (null != be.RequestInformation && null != be.RequestInformation.AzureError)
{
// Write the server side error information
Console.Error.WriteLine(be.RequestInformation.AzureError.Code);
Console.Error.WriteLine(be.RequestInformation.AzureError.Message.Value);
if (null != be.RequestInformation.AzureError.Values)
{
foreach (var v in be.RequestInformation.AzureError.Values)
{
Console.Error.WriteLine(v.Key + " : " + v.Value);
}
}
}
}
// Indicate that the error has been handled
return true;
});
}
DateTime starttime = DateTime.Now;
//Wait for the job to complete
if (config.WaitForCompletion)
{
ICloudJob job = wm.GetJob(artifacts.WorkItemName, artifacts.JobName);
Console.WriteLine("Waiting for tasks to complete...");
// Wait up to 15 minutes for all tasks to reach the completed state
config.Client.OpenToolbox().CreateTaskStateMonitor().WaitAll(job.ListTasks(), TaskState.Completed, new TimeSpan(0, 15, 0));
DateTime endtime = DateTime.Now;
Console.WriteLine("Time taken for processing the images : {0} sec", endtime.Subtract(starttime).TotalSeconds);
}
}
}
finally
{
//Delete the workitem that we created
if (config.DeleteWorkitem &&
config.WaitForCompletion)
{
Console.WriteLine("Press any key to delete the workitem . . .");
Console.ReadKey();
config.Client.OpenWorkItemManager().DeleteWorkItem(config.WorkitemName);
}
}
}
// AddTasks(): Creates tasks to process each of the specified input files, and submits them
// to the specified job for execution.
// batchClient: A BatchClient object.
// jobId: The ID of the job to which the tasks are added.
// inputFiles: A collection of ResourceFile objects representing the input files
// to be processed by the tasks executed on the compute nodes.
// outputContainerSasUrl: The shared access signature URL for the Azure Storage
// container that will hold the output files that the tasks create.
// Returns: A collection of the submitted cloud tasks.
private static List <CloudTask> AddTasks(BatchClient batchClient, string jobId, List <ResourceFile> inputFiles, string outputContainerSasUrl)
{
Console.WriteLine("Adding {0} tasks to job [{1}]...", inputFiles.Count, jobId);
// Create a collection to hold the tasks added to the job:
List <CloudTask> tasks = new List <CloudTask>();
// Create each task. The start task copies the application executable (ffmpeg.exe) to the
// node's shared directory, so the cloud tasks can access this application via the shared
// directory on whichever node each task runs.
foreach (ResourceFile inputFile in inputFiles)
{
// Assign a task ID for each iteration
string taskId = "task_" + inputFiles.IndexOf(inputFile);
// Define task command line to convert the video format from MP4 to MPEG-1 using ffmpeg.
// Note that ffmpeg syntax specifies the format conversion using the file extension of
// the input file and the output file respectively.
/*
* // %% REPLACE THIS BLOCK WITH DAN'S OUTPUT FILE FORMATTING
* string inputFilePath = inputFile.FilePath;
* string outputFileName = String.Format("{0}{1}",
* System.IO.Path.GetFileNameWithoutExtension(inputFilePath),
* ".mpeg");
* string outputFilePath = outputContainerSasUrl + outputFileName;
*/
// %% NEW OUTPUT FILE FORMATTING:
// Define paths for input and output files
string inputMediaFile = inputFile.FilePath;
string outputMediaFile = String.Format("{0}{1}",
System.IO.Path.GetFileNameWithoutExtension(inputMediaFile),
".mpeg");
// Format the task command line
string taskCommandLine = String.Format("cmd /c %AZ_BATCH_NODE_SHARED_DIR%\\ffmpeg.exe -i {0} {1}", inputMediaFile, outputMediaFile);
// Create a cloud task (with the task ID and command line) and give it a list of input files
CloudTask task = new CloudTask(taskId, taskCommandLine);
task.ResourceFiles = new List <ResourceFile> {
inputFile
};
// Send the result to the task's list of output files
List <OutputFile> outputFileList = new List <OutputFile>();
OutputFileBlobContainerDestination outputContainer = new OutputFileBlobContainerDestination(outputContainerSasUrl);
OutputFile outputFile = new OutputFile(outputMediaFile,
new OutputFileDestination(outputContainer),
new OutputFileUploadOptions(OutputFileUploadCondition.TaskSuccess));
outputFileList.Add(outputFile);
task.OutputFiles = outputFileList;
// Add the cloud task to the task list
tasks.Add(task);
}
// Call BatchClient.JobOperations.AddTask() to add the tasks as a collection rather than making a
// separate call for each. Bulk task submission helps to ensure efficient underlying API
// calls to the Batch service. calls AddTaskAsync() so the add operation doesn't hang up program execution.
batchClient.JobOperations.AddTaskAsync(jobId, tasks).Wait();
return(tasks);
}
private async Task SubmitReducerTaskAsync(BatchClient batchClient, string containerSas)
{
Console.WriteLine("Adding the reducer task: {0}", Constants.ReducerTaskId);
CloudTask unboundReducerTask = new CloudTask(Constants.ReducerTaskId, Constants.ReducerTaskExecutable);
//The set of files (exes, dlls and configuration files) required to run the reducer task.
List<ResourceFile> reducerTaskResourceFiles = SampleHelpers.GetResourceFiles(containerSas, Constants.RequiredExecutableFiles);
unboundReducerTask.ResourceFiles = reducerTaskResourceFiles;
//Send the request to the Batch Service to add the reducer task.
await batchClient.JobOperations.AddTaskAsync(this.jobId, unboundReducerTask);
}
public static void HelloWorld(
BatchClient batchCli,
ITestOutputHelper testOutputHelper,
CloudPool sharedPool,
out string jobId,
out string taskId,
bool deleteJob = true,
bool isLinux = false)
{
jobId = "HelloWorldJob-" + GetMyName() + "-" + GetTimeStamp();
try
{
// here we show how to use an unbound Job + Commit() to run a simple "Hello World" task
// get an empty unbound Job
CloudJob quickJob = batchCli.JobOperations.CreateJob();
quickJob.Id = jobId;
quickJob.PoolInformation = new PoolInformation()
{
PoolId = sharedPool.Id
};
// Commit Job
quickJob.Commit();
// get an empty unbound Task
taskId = "dwsHelloWorldTask";
const string winPaasHWTaskCmdLine = "cmd /c echo Hello World";
const string linuxIaasHWTaskCmdLine = "echo Hello World";
string winnerTaskCmdLine = isLinux ? linuxIaasHWTaskCmdLine : winPaasHWTaskCmdLine;
CloudTask hwTask = new CloudTask(id: taskId, commandline: winnerTaskCmdLine);
// Open the new Job as bound.
CloudJob boundJob = batchCli.JobOperations.GetJob(jobId);
// add Task to Job
boundJob.AddTask(hwTask);
// wait for the task to complete
Utilities utilities = batchCli.Utilities;
TaskStateMonitor taskStateMonitor = utilities.CreateTaskStateMonitor();
taskStateMonitor.WaitAll(
boundJob.ListTasks(),
TaskState.Completed,
TimeSpan.FromMinutes(3));
CloudTask myCompletedTask = new List <CloudTask>(boundJob.ListTasks(null))[0];
string stdOut = myCompletedTask.GetNodeFile(Constants.StandardOutFileName).ReadAsString();
string stdErr = myCompletedTask.GetNodeFile(Constants.StandardErrorFileName).ReadAsString();
// confirm that stdout includes correct value
Assert.Contains("Hello World", stdOut);
testOutputHelper.WriteLine("StdOut: ");
testOutputHelper.WriteLine(stdOut);
testOutputHelper.WriteLine("StdErr: ");
testOutputHelper.WriteLine(stdErr);
}
finally
{
// delete the job to free the Pool compute nodes.
if (deleteJob)
{
TestUtilities.DeleteJobIfExistsAsync(batchCli, jobId).Wait();
}
}
}
/// <summary>
/// Creates a new Azure Batch job
/// </summary>
/// <param name="jobId"></param>
/// <param name="jobPreparationTask"></param>
/// <param name="cloudTask"></param>
/// <param name="poolInformation"></param>
/// <returns></returns>
public async Task CreateBatchJobAsync(string jobId, JobPreparationTask jobPreparationTask, CloudTask cloudTask, PoolInformation poolInformation)
{
var job = batchClient.JobOperations.CreateJob(jobId, poolInformation);
job.JobPreparationTask = jobPreparationTask;
await job.CommitAsync();
try
{
job = await batchClient.JobOperations.GetJobAsync(job.Id); // Retrieve the "bound" version of the job
job.PoolInformation = poolInformation; // Redoing this since the container registry password is not retrieved by GetJobAsync()
job.OnAllTasksComplete = OnAllTasksComplete.TerminateJob;
await job.AddTaskAsync(cloudTask);
await job.CommitAsync();
}
catch (Exception ex)
{
var batchError = JsonConvert.SerializeObject((ex as BatchException)?.RequestInformation?.BatchError);
logger.LogError(ex, $"Deleting {job.Id} because adding task to it failed. Batch error: {batchError}");
await batchClient.JobOperations.DeleteJobAsync(job.Id);
throw;
}
}
public Task CreateBatchJobAsync(string jobId, CloudTask cloudTask, PoolInformation poolInformation) => azureProxy.CreateBatchJobAsync(jobId, cloudTask, poolInformation);
public static async Task MainAsync()
{
const string poolId = "MultiInstanceSamplePool";
const string jobId = "MultiInstanceSampleJob";
const string taskId = "MultiInstanceSampleTask";
const int numberOfNodes = 3;
// The application package and version to deploy to the compute nodes.
// It should contain your MPIHelloWorld sample MS-MPI program:
// https://blogs.technet.microsoft.com/windowshpc/2015/02/02/how-to-compile-and-run-a-simple-ms-mpi-program/
// And the MSMpiSetup.exe installer:
// https://www.microsoft.com/download/details.aspx?id=52981
// Then upload it as an application package:
// https://azure.microsoft.com/documentation/articles/batch-application-packages/
const string appPackageId = "MPIHelloWorld";
const string appPackageVersion = "1.0";
TimeSpan timeout = TimeSpan.FromMinutes(30);
AccountSettings accountSettings = SampleHelpers.LoadAccountSettings();
// Configure your AccountSettings in the Microsoft.Azure.Batch.Samples.Common project within this solution
BatchSharedKeyCredentials cred = new BatchSharedKeyCredentials(
accountSettings.BatchServiceUrl,
accountSettings.BatchAccountName,
accountSettings.BatchAccountKey);
using (BatchClient batchClient = BatchClient.Open(cred))
{
// Create the pool of compute nodes and the job to which we add the multi-instance task.
await CreatePoolAsync(batchClient, poolId, numberOfNodes, appPackageId, appPackageVersion);
await CreateJobAsync(batchClient, jobId, poolId);
// Create the multi-instance task. The MultiInstanceSettings property (configured
// below) tells Batch to create one primary and several subtasks, the total number
// of which matches the number of instances you specify in the MultiInstanceSettings.
// This main task's command line is the "application command," and is executed *only*
// by the primary, and only after the primary and all subtasks have executed the
// "coordination command" (the MultiInstanceSettings.CoordinationCommandLine).
CloudTask multiInstanceTask = new CloudTask(id: taskId,
commandline: $"cmd /c mpiexec.exe -c 1 -wdir %AZ_BATCH_TASK_SHARED_DIR% %AZ_BATCH_APP_PACKAGE_{appPackageId.ToUpper()}#{appPackageVersion}%\\MPIHelloWorld.exe");
// Configure the task's MultiInstanceSettings. Specify the number of nodes
// to allocate to the multi-instance task, and the "coordination command".
// The CoordinationCommandLine is run by the primary and subtasks, and is
// used in this sample to start SMPD on the compute nodes.
multiInstanceTask.MultiInstanceSettings =
new MultiInstanceSettings(@"cmd /c start cmd /c smpd.exe -d", numberOfNodes);
// Submit the task to the job. Batch will take care of creating one primary and
// enough subtasks to match the total number of nodes allocated to the task,
// and schedule them for execution on the nodes.
Console.WriteLine($"Adding task [{taskId}] to job [{jobId}]...");
await batchClient.JobOperations.AddTaskAsync(jobId, multiInstanceTask);
// Get the "bound" version of the multi-instance task.
CloudTask mainTask = await batchClient.JobOperations.GetTaskAsync(jobId, taskId);
// We use a TaskStateMonitor to monitor the state of our tasks. In this case,
// we will wait for the task to reach the Completed state.
Console.WriteLine($"Awaiting task completion, timeout in {timeout}...");
TaskStateMonitor taskStateMonitor = batchClient.Utilities.CreateTaskStateMonitor();
await taskStateMonitor.WhenAll(new List <CloudTask> {
mainTask
}, TaskState.Completed, timeout);
// Refresh the task to obtain up-to-date property values from Batch, such as
// its current state and information about the node on which it executed.
await mainTask.RefreshAsync();
string stdOut = mainTask.GetNodeFile(Constants.StandardOutFileName).ReadAsString();
string stdErr = mainTask.GetNodeFile(Constants.StandardErrorFileName).ReadAsString();
Console.WriteLine();
Console.WriteLine($"Main task [{mainTask.Id}] is in state [{mainTask.State}] and ran on compute node [{mainTask.ComputeNodeInformation.ComputeNodeId}]:");
Console.WriteLine("---- stdout.txt ----");
Console.WriteLine(stdOut);
Console.WriteLine("---- stderr.txt ----");
Console.WriteLine(stdErr);
// Need to delay a bit to allow the Batch service to mark the subtasks as Complete
TimeSpan subtaskTimeout = TimeSpan.FromSeconds(10);
Console.WriteLine($"Main task completed, waiting {subtaskTimeout} for subtasks to complete...");
System.Threading.Thread.Sleep(subtaskTimeout);
Console.WriteLine();
Console.WriteLine("---- Subtask information ----");
// Obtain the collection of subtasks for the multi-instance task, and print
// some information about each.
IPagedEnumerable <SubtaskInformation> subtasks = mainTask.ListSubtasks();
await subtasks.ForEachAsync(async (subtask) =>
{
Console.WriteLine("subtask: " + subtask.Id);
Console.WriteLine("\texit code: " + subtask.ExitCode);
if (subtask.State == SubtaskState.Completed)
{
// Obtain the file from the node on which the subtask executed. For normal CloudTasks,
// we could simply call CloudTask.GetNodeFile(Constants.StandardOutFileName), but the
// subtasks are not "normal" tasks in Batch, and thus must be handled differently.
ComputeNode node =
await batchClient.PoolOperations.GetComputeNodeAsync(subtask.ComputeNodeInformation.PoolId,
subtask.ComputeNodeInformation.ComputeNodeId);
string outPath = subtask.ComputeNodeInformation.TaskRootDirectory + "\\" + Constants.StandardOutFileName;
string errPath = subtask.ComputeNodeInformation.TaskRootDirectory + "\\" + Constants.StandardErrorFileName;
NodeFile stdOutFile = await node.GetNodeFileAsync(outPath.Trim('\\'));
NodeFile stdErrFile = await node.GetNodeFileAsync(errPath.Trim('\\'));
stdOut = await stdOutFile.ReadAsStringAsync();
stdErr = await stdErrFile.ReadAsStringAsync();
Console.WriteLine($"\tnode: " + node.Id);
Console.WriteLine("\tstdout.txt: " + stdOut);
Console.WriteLine("\tstderr.txt: " + stdErr);
}
else
{
Console.WriteLine($"\tSubtask {subtask.Id} is in state {subtask.State}");
}
});
// Clean up the resources we've created in the Batch account
Console.WriteLine();
Console.Write("Delete job? [yes] no: ");
string response = Console.ReadLine().ToLower();
if (response != "n" && response != "no")
{
await batchClient.JobOperations.DeleteJobAsync(jobId);
}
Console.Write("Delete pool? [yes] no: ");
response = Console.ReadLine().ToLower();
if (response != "n" && response != "no")
{
await batchClient.PoolOperations.DeletePoolAsync(poolId);
}
}
}
/// <summary>
/// Populates Azure Storage with the required files, and
/// submits the job to the Azure Batch service.
/// </summary>
public async Task RunAsync()
{
Console.WriteLine("Running with the following settings: ");
Console.WriteLine("----------------------------------------");
Console.WriteLine(this.textSearchSettings.ToString());
Console.WriteLine(this.accountSettings.ToString());
CloudStorageAccount cloudStorageAccount = new CloudStorageAccount(
new StorageCredentials(
this.accountSettings.StorageAccountName,
this.accountSettings.StorageAccountKey),
this.accountSettings.StorageServiceUrl,
useHttps: true);
//Upload resources if required.
if (this.textSearchSettings.ShouldUploadResources)
{
Console.WriteLine("Splitting file: {0} into {1} subfiles",
Constants.TextFilePath,
this.textSearchSettings.NumberOfMapperTasks);
//Split the text file into the correct number of files for consumption by the mapper tasks.
FileSplitter splitter = new FileSplitter();
List <string> mapperTaskFiles = await splitter.SplitAsync(
Constants.TextFilePath,
this.textSearchSettings.NumberOfMapperTasks);
List <string> files = Constants.RequiredExecutableFiles.Union(mapperTaskFiles).ToList();
await SampleHelpers.UploadResourcesAsync(
cloudStorageAccount,
this.textSearchSettings.BlobContainer,
files);
}
//Generate a SAS for the container.
string containerSasUrl = SampleHelpers.ConstructContainerSas(
cloudStorageAccount,
this.textSearchSettings.BlobContainer);
//Set up the Batch Service credentials used to authenticate with the Batch Service.
BatchSharedKeyCredentials credentials = new BatchSharedKeyCredentials(
this.accountSettings.BatchServiceUrl,
this.accountSettings.BatchAccountName,
this.accountSettings.BatchAccountKey);
using (BatchClient batchClient = BatchClient.Open(credentials))
{
//
// Construct the job properties in local memory before commiting them to the Batch Service.
//
//Allow enough compute nodes in the pool to run each mapper task, and 1 extra to run the job manager.
int numberOfPoolComputeNodes = 1 + this.textSearchSettings.NumberOfMapperTasks;
//Define the pool specification for the pool which the job will run on.
PoolSpecification poolSpecification = new PoolSpecification()
{
TargetDedicatedComputeNodes = numberOfPoolComputeNodes,
VirtualMachineSize = "standard_d1_v2",
//You can learn more about os families and versions at:
//http://azure.microsoft.com/documentation/articles/cloud-services-guestos-update-matrix
CloudServiceConfiguration = new CloudServiceConfiguration(osFamily: "5")
};
//Use the auto pool feature of the Batch Service to create a pool when the job is created.
//This creates a new pool for each job which is added.
AutoPoolSpecification autoPoolSpecification = new AutoPoolSpecification()
{
AutoPoolIdPrefix = "TextSearchPool",
KeepAlive = false,
PoolLifetimeOption = PoolLifetimeOption.Job,
PoolSpecification = poolSpecification
};
//Define the pool information for this job -- it will run on the pool defined by the auto pool specification above.
PoolInformation poolInformation = new PoolInformation()
{
AutoPoolSpecification = autoPoolSpecification
};
//Define the job manager for this job. This job manager will run first and will submit the tasks for
//the job. The job manager is the executable which manages the lifetime of the job
//and all tasks which should run for the job. In this case, the job manager submits the mapper and reducer tasks.
List <ResourceFile> jobManagerResourceFiles = SampleHelpers.GetResourceFiles(containerSasUrl, Constants.RequiredExecutableFiles);
const string jobManagerTaskId = "JobManager";
JobManagerTask jobManagerTask = new JobManagerTask()
{
ResourceFiles = jobManagerResourceFiles,
CommandLine = Constants.JobManagerExecutable,
//Determines if the job should terminate when the job manager process exits.
KillJobOnCompletion = true,
Id = jobManagerTaskId
};
//Create the unbound job in local memory. An object which exists only in local memory (and not on the Batch Service) is "unbound".
string jobId = Environment.GetEnvironmentVariable("USERNAME") + DateTime.UtcNow.ToString("yyyyMMdd-HHmmss");
CloudJob unboundJob = batchClient.JobOperations.CreateJob(jobId, poolInformation);
unboundJob.JobManagerTask = jobManagerTask; //Assign the job manager task to this job
try
{
//Commit the unbound job to the Batch Service.
Console.WriteLine("Adding job: {0} to the Batch Service.", unboundJob.Id);
await unboundJob.CommitAsync(); //Issues a request to the Batch Service to add the job which was defined above.
//
// Wait for the job manager task to complete.
//
//An object which is backed by a corresponding Batch Service object is "bound."
CloudJob boundJob = await batchClient.JobOperations.GetJobAsync(jobId);
CloudTask boundJobManagerTask = await boundJob.GetTaskAsync(jobManagerTaskId);
TimeSpan maxJobCompletionTimeout = TimeSpan.FromMinutes(30);
// Monitor the current tasks to see when they are done.
// Occasionally a task may get killed and requeued during an upgrade or hardware failure, including the job manager
// task. The job manager will be re-run in this case. Robustness against this was not added into the sample for
// simplicity, but should be added into any production code.
Console.WriteLine("Waiting for job's tasks to complete");
TaskStateMonitor taskStateMonitor = batchClient.Utilities.CreateTaskStateMonitor();
try
{
await taskStateMonitor.WhenAll(new List <CloudTask> {
boundJobManagerTask
}, TaskState.Completed, maxJobCompletionTimeout);
}
finally
{
Console.WriteLine("Done waiting for job manager task.");
await boundJobManagerTask.RefreshAsync();
//Check to ensure the job manager task exited successfully.
await Helpers.CheckForTaskSuccessAsync(boundJobManagerTask, dumpStandardOutOnTaskSuccess : false);
}
//
// Download and write out the reducer tasks output
//
string reducerText = await SampleHelpers.DownloadBlobTextAsync(cloudStorageAccount, this.textSearchSettings.BlobContainer, Constants.ReducerTaskResultBlobName);
Console.WriteLine("Reducer reuslts:");
Console.WriteLine(reducerText);
}
finally
{
//Delete the job.
//This will delete the auto pool associated with the job as long as the pool
//keep alive property is set to false.
if (this.textSearchSettings.ShouldDeleteJob)
{
Console.WriteLine("Deleting job {0}", jobId);
await batchClient.JobOperations.DeleteJobAsync(jobId);
}
//Note that there were files uploaded to a container specified in the
//configuration file. This container will not be deleted or cleaned up by this sample.
}
}
}
public void Bug1996130_JobTaskVerbsFailAfterDoubleRefresh()
{
Action test = () =>
{
using (BatchClient batchCli = TestUtilities.OpenBatchClient(TestUtilities.GetCredentialsFromEnvironment()))
{
string jobId = "Bug1996130Job-" + TestUtilities.GetMyName();
try
{
// get a job/task to test. use workflow
CloudJob boundJob = null;
{
// need a bound job/task for the tests so set one up
CloudJob tsh = batchCli.JobOperations.CreateJob(jobId, new PoolInformation());
tsh.PoolInformation.PoolId = this.poolFixture.PoolId;
tsh.Commit();
boundJob = batchCli.JobOperations.GetJob(jobId);
boundJob.AddTask(new CloudTask("Bug1996130_task", "cmd /c hostname"));
}
// test task double refresh
{
// get the task
CloudTask boundTask = batchCli.JobOperations.ListTasks(jobId).First();
// double refresh
boundTask.Refresh();
boundTask.Refresh(); // this branch of the bug actually fixed in the other doublerefesh checkin by matthchr
// do verbs
boundTask.Refresh();
boundTask.Delete();
Thread.Sleep(5000); // give server time to do its deed
List <CloudTask> tasks = batchCli.JobOperations.ListTasks(jobId).ToList();
// confirm delete suceeded
Assert.Empty(tasks);
}
// test job double refresh and verbs
{
boundJob = batchCli.JobOperations.GetJob(jobId);
// double refresh to taint the instance... lost path variable
boundJob.Refresh();
boundJob.Refresh(); // this used to fail/throw
boundJob.Refresh(); // this should fail but does not
boundJob.Delete(); // yet another verb that suceeds
CloudJob job = batchCli.JobOperations.ListJobs().ToList().FirstOrDefault(j => j.Id == jobId);
// confirm job delete suceeded
Assert.True(job == null || (JobState.Deleting == job.State));
}
}
finally
{
TestUtilities.DeleteJobIfExistsAsync(batchCli, jobId).Wait();
}
}
};
SynchronizationContextHelper.RunTest(test, TestTimeout);
}
public static void JobMain(string[] args)
{
//Load the configuration
TopNWordsConfiguration configuration = TopNWordsConfiguration.LoadConfigurationFromAppConfig();
StagingStorageAccount stagingStorageAccount = new StagingStorageAccount(
configuration.StorageAccountName,
configuration.StorageAccountKey,
configuration.StorageAccountBlobEndpoint);
using (BatchClient client = BatchClient.Open(new BatchSharedKeyCredentials(configuration.BatchServiceUrl, configuration.BatchAccountName, configuration.BatchAccountKey)))
{
string stagingContainer = null;
//Create a pool (if user hasn't provided one)
if (configuration.ShouldCreatePool)
{
//OSFamily 4 == OS 2012 R2. You can learn more about os families and versions at:
//http://msdn.microsoft.com/en-us/library/azure/ee924680.aspx
CloudPool pool = client.PoolOperations.CreatePool(configuration.PoolId, targetDedicated: configuration.PoolSize, osFamily: "4", virtualMachineSize: "small");
Console.WriteLine("Adding pool {0}", configuration.PoolId);
try
{
pool.Commit();
}
catch (AggregateException ae)
{
// Go through all exceptions and dump useful information
ae.Handle(x =>
{
Console.Error.WriteLine("Creating pool ID {0} failed", configuration.PoolId);
if (x is BatchException)
{
BatchException be = x as BatchException;
Console.WriteLine(be.ToString());
Console.WriteLine();
}
else
{
Console.WriteLine(x);
}
// can't continue without a pool
return false;
});
}
}
try
{
Console.WriteLine("Creating job: " + configuration.JobId);
// get an empty unbound Job
CloudJob unboundJob = client.JobOperations.CreateJob();
unboundJob.Id = configuration.JobId;
unboundJob.PoolInformation = new PoolInformation() { PoolId = configuration.PoolId };
// Commit Job to create it in the service
unboundJob.Commit();
// create file staging objects that represent the executable and its dependent assembly to run as the task.
// These files are copied to every node before the corresponding task is scheduled to run on that node.
FileToStage topNWordExe = new FileToStage(TopNWordsExeName, stagingStorageAccount);
FileToStage storageDll = new FileToStage(StorageClientDllName, stagingStorageAccount);
// In this sample, the input data is copied separately to Storage and its URI is passed to the task as an argument.
// This approach is appropriate when the amount of input data is large such that copying it to every node via FileStaging
// is not desired and the number of tasks is small since a large number of readers of the blob might get throttled
// by Storage which will lengthen the overall processing time.
//
// You'll need to observe the behavior and use published techniques for finding the right balance of performance versus
// complexity.
string bookFileUri = UploadBookFileToCloudBlob(configuration, configuration.BookFileName);
Console.WriteLine("{0} uploaded to cloud", configuration.BookFileName);
// initialize a collection to hold the tasks that will be submitted in their entirety
List<CloudTask> tasksToRun = new List<CloudTask>(configuration.NumberOfTasks);
for (int i = 1; i <= configuration.NumberOfTasks; i++)
{
CloudTask task = new CloudTask("task_no_" + i, String.Format("{0} --Task {1} {2} {3} {4}",
TopNWordsExeName,
bookFileUri,
configuration.NumberOfTopWords,
configuration.StorageAccountName,
configuration.StorageAccountKey));
//This is the list of files to stage to a container -- for each job, one container is created and
//files all resolve to Azure Blobs by their name (so two tasks with the same named file will create just 1 blob in
//the container).
task.FilesToStage = new List<IFileStagingProvider>
{
topNWordExe,
storageDll
};
tasksToRun.Add(task);
}
// Commit all the tasks to the Batch Service. Ask AddTask to return information about the files that were staged.
// The container information is used later on to remove these files from Storage.
ConcurrentBag<ConcurrentDictionary<Type, IFileStagingArtifact>> fsArtifactBag = new ConcurrentBag<ConcurrentDictionary<Type, IFileStagingArtifact>>();
client.JobOperations.AddTask(configuration.JobId, tasksToRun, fileStagingArtifacts: fsArtifactBag);
// loop through the bag of artifacts, looking for the one that matches our staged files. Once there,
// capture the name of the container holding the files so they can be deleted later on if that option
// was configured in the settings.
foreach (var fsBagItem in fsArtifactBag)
{
IFileStagingArtifact fsValue;
if (fsBagItem.TryGetValue(typeof(FileToStage), out fsValue))
{
SequentialFileStagingArtifact stagingArtifact = fsValue as SequentialFileStagingArtifact;
if (stagingArtifact != null)
{
stagingContainer = stagingArtifact.BlobContainerCreated;
Console.WriteLine(
"Uploaded files to container: {0} -- you will be charged for their storage unless you delete them.",
stagingArtifact.BlobContainerCreated);
}
}
}
//Get the job to monitor status.
CloudJob job = client.JobOperations.GetJob(configuration.JobId);
Console.Write("Waiting for tasks to complete ... ");
// Wait 20 minutes for all tasks to reach the completed state. The long timeout is necessary for the first
// time a pool is created in order to allow nodes to be added to the pool and initialized to run tasks.
IPagedEnumerable<CloudTask> ourTasks = job.ListTasks(new ODATADetailLevel(selectClause: "id"));
client.Utilities.CreateTaskStateMonitor().WaitAll(ourTasks, TaskState.Completed, TimeSpan.FromMinutes(20));
Console.WriteLine("tasks are done.");
foreach (CloudTask t in ourTasks)
{
Console.WriteLine("Task " + t.Id);
Console.WriteLine("stdout:" + Environment.NewLine + t.GetNodeFile(Constants.StandardOutFileName).ReadAsString());
Console.WriteLine();
Console.WriteLine("stderr:" + Environment.NewLine + t.GetNodeFile(Constants.StandardErrorFileName).ReadAsString());
}
}
finally
{
//Delete the pool that we created
if (configuration.ShouldCreatePool)
{
Console.WriteLine("Deleting pool: {0}", configuration.PoolId);
client.PoolOperations.DeletePool(configuration.PoolId);
}
//Delete the job that we created
if (configuration.ShouldDeleteJob)
{
Console.WriteLine("Deleting job: {0}", configuration.JobId);
client.JobOperations.DeleteJob(configuration.JobId);
}
//Delete the containers we created
if (configuration.ShouldDeleteContainer)
{
DeleteContainers(configuration, stagingContainer);
}
}
}
}
public void TestExitConditionsAreBeingRoundTrippedCorrectly()
{
Action test = () =>
{
using (BatchClient client = TestUtilities.OpenBatchClient(TestUtilities.GetCredentialsFromEnvironment()))
{
//Create a job
string jobId = Constants.DefaultConveniencePrefix + TestUtilities.GetMyName() + "-TestExitConditionsAreBeingRoundTrippedCorrectly";
string taskId = "task-id-1";
try
{
CloudJob boundJob = null;
{
// need a bound job/task for the tests so set one up
boundJob = CreateBoundJob(client, jobId, j => { j.OnTaskFailure = OnTaskFailure.PerformExitOptionsJobAction; });
CloudTask cloudTask = new CloudTask(taskId, "cmd /c exit 2");
cloudTask.ExitConditions = new ExitConditions
{
ExitCodes = new List <ExitCodeMapping> {
new ExitCodeMapping(1, new ExitOptions {
JobAction = JobAction.None
})
},
ExitCodeRanges = new List <ExitCodeRangeMapping>
{
new ExitCodeRangeMapping(2, 4, new ExitOptions {
JobAction = JobAction.Disable
})
},
PreProcessingError = new ExitOptions {
JobAction = JobAction.Terminate
},
FileUploadError = new ExitOptions {
JobAction = JobAction.Terminate
},
Default = new ExitOptions {
JobAction = JobAction.Terminate
},
};
boundJob.AddTask(cloudTask);
boundJob.Refresh();
Assert.Equal(OnTaskFailure.PerformExitOptionsJobAction, boundJob.OnTaskFailure);
CloudTask boundTask = client.JobOperations.GetTask(jobId, taskId);
Assert.Equal(JobAction.None, boundTask.ExitConditions.ExitCodes.First().ExitOptions.JobAction);
Assert.Equal(1, boundTask.ExitConditions.ExitCodes.First().Code);
var exitCodeRangeMappings = boundTask.ExitConditions.ExitCodeRanges;
Assert.Equal(2, exitCodeRangeMappings.First().Start);
Assert.Equal(4, exitCodeRangeMappings.First().End);
Assert.Equal(JobAction.Disable, exitCodeRangeMappings.First().ExitOptions.JobAction);
Assert.Equal(JobAction.Terminate, boundTask.ExitConditions.PreProcessingError.JobAction);
Assert.Equal(JobAction.Terminate, boundTask.ExitConditions.FileUploadError.JobAction);
Assert.Equal(JobAction.Terminate, boundTask.ExitConditions.Default.JobAction);
}
}
finally
{
TestUtilities.DeleteJobIfExistsAsync(client, jobId).Wait();
}
}
};
SynchronizationContextHelper.RunTest(test, TestTimeout);
}
/// <summary>
/// Submit a work item with tasks which have dependant files.
/// The files are automatically uploaded to Azure Storage using the FileStaging feature of the Azure.Batch client library.
/// </summary>
/// <param name="client"></param>
private static void AddWorkWithFileStaging(IBatchClient client)
{
using (IWorkItemManager wm = client.OpenWorkItemManager())
{
IToolbox toolbox = client.OpenToolbox();
ITaskSubmissionHelper taskSubmissionHelper = toolbox.CreateTaskSubmissionHelper(wm, Program.PoolName);
taskSubmissionHelper.WorkItemName = Environment.GetEnvironmentVariable("USERNAME") + DateTime.Now.ToString("yyyyMMdd-HHmmss");
Console.WriteLine("Creating work item: {0}", taskSubmissionHelper.WorkItemName);
ICloudTask taskToAdd1 = new CloudTask("task_with_file1", "cmd /c type *.txt");
ICloudTask taskToAdd2 = new CloudTask("task_with_file2", "cmd /c dir /s");
//Set up a collection of files to be staged -- these files will be uploaded to Azure Storage
//when the tasks are submitted to the Azure Batch service.
taskToAdd1.FilesToStage = new List <IFileStagingProvider>();
taskToAdd2.FilesToStage = new List <IFileStagingProvider>();
// generate a local file in temp directory
Process cur = Process.GetCurrentProcess();
string path = Path.Combine(Environment.GetEnvironmentVariable("TEMP"), cur.Id.ToString() + ".txt");
System.IO.File.WriteAllText(path, "hello from " + cur.Id.ToString());
// add file as task dependency so it'll be uploaded to storage before task
// is submitted and download onto the VM before task starts execution
FileToStage file = new FileToStage(path, new StagingStorageAccount(Program.StorageAccount, Program.StorageKey, Program.StorageBlobEndpoint));
taskToAdd1.FilesToStage.Add(file);
taskToAdd2.FilesToStage.Add(file); // filetostage object can be reused
taskSubmissionHelper.AddTask(taskToAdd1);
taskSubmissionHelper.AddTask(taskToAdd2);
IJobCommitUnboundArtifacts artifacts = null;
bool errors = false;
try
{
//Stage the files to Azure Storage and add the tasks to Azure Batch.
artifacts = taskSubmissionHelper.Commit() as IJobCommitUnboundArtifacts;
}
catch (AggregateException ae)
{
errors = true;
// Go through all exceptions and dump useful information
ae.Handle((x) =>
{
if (x is BatchException)
{
BatchException be = x as BatchException;
if (null != be.RequestInformation && null != be.RequestInformation.AzureError)
{
// Write the server side error information
Console.Error.WriteLine(be.RequestInformation.AzureError.Code);
Console.Error.WriteLine(be.RequestInformation.AzureError.Message.Value);
if (null != be.RequestInformation.AzureError.Values)
{
foreach (var v in be.RequestInformation.AzureError.Values)
{
Console.Error.WriteLine(v.Key + " : " + v.Value);
}
}
}
}
else
{
Console.WriteLine(x);
}
// Indicate that the error has been handled
return(true);
});
}
// if there is no exception, wait for job response
if (!errors)
{
List <ICloudTask> tasksToMonitorForCompletion = wm.ListTasks(artifacts.WorkItemName, artifacts.JobName).ToList();
Console.WriteLine("Waiting for all tasks to complete on work item: {0}, Job: {1} ...", artifacts.WorkItemName, artifacts.JobName);
client.OpenToolbox().CreateTaskStateMonitor().WaitAll(tasksToMonitorForCompletion, TaskState.Completed, TimeSpan.FromMinutes(30));
foreach (ICloudTask task in wm.ListTasks(artifacts.WorkItemName, artifacts.JobName))
{
Console.WriteLine("Task " + task.Name + " says:\n" + task.GetTaskFile(Constants.StandardOutFileName).ReadAsString());
Console.WriteLine(task.GetTaskFile(Constants.StandardErrorFileName).ReadAsString());
}
}
Console.WriteLine("Deleting work item: {0}", artifacts.WorkItemName);
wm.DeleteWorkItem(artifacts.WorkItemName); //Don't forget to delete the work item before you exit
}
}
private static void CreatePool(string poolName)
{
var vmSize = ConfigurationManager.AppSettings["VirtualMachineSize"];
var dedicatedVmCount = Int32.Parse(ConfigurationManager.AppSettings["VirtualMachineCount"]);
var lowPriorityVmCount = 0;
if (Boolean.Parse(ConfigurationManager.AppSettings["UseLowPriority"]))
{
lowPriorityVmCount = dedicatedVmCount;
dedicatedVmCount = 0;
}
var pool = Client.PoolOperations.CreatePool(
poolName,
vmSize,
new VirtualMachineConfiguration(
new ImageReference(
ConfigurationManager.AppSettings["ImageOffer"],
ConfigurationManager.AppSettings["ImagePublisher"],
ConfigurationManager.AppSettings["ImageSku"],
ConfigurationManager.AppSettings["ImageVersion"]),
ConfigurationManager.AppSettings["NodeAgentSku"]),
dedicatedVmCount,
lowPriorityVmCount);
SetupPoolNetworking(pool);
pool.ApplicationLicenses = new List <string> {
"3dsmax", "vray"
};
pool.Commit();
Console.WriteLine("Created pool {0} with {1} compute node(s)", poolName, dedicatedVmCount == 0 ? lowPriorityVmCount : dedicatedVmCount);
var job = Client.JobOperations.CreateJob("vray-dr-" + poolName, new PoolInformation {
PoolId = poolName
});
job.Commit();
job.Refresh();
var vmCount = Math.Max(lowPriorityVmCount, dedicatedVmCount);
var cmd = string.Format("cd .. & vray-adv-dr.cmd {0}",
ConfigurationManager.AppSettings["VRayServerPort"]);
var task = new CloudTask("setup-vray-dr", "set");
task.MultiInstanceSettings = new MultiInstanceSettings(cmd, vmCount);
task.MultiInstanceSettings.CommonResourceFiles = new List <ResourceFile>
{
new ResourceFile("https://raw.githubusercontent.com/Azure/azure-hpc/master/Rendering/VRayPoolManager/Scripts/vray-adv-dr.cmd", "vray-adv-dr.cmd")
};
task.Constraints = new TaskConstraints(maxTaskRetryCount: -1);
task.UserIdentity = new UserIdentity(new AutoUserSpecification(AutoUserScope.Pool, elevationLevel: ElevationLevel.Admin));
job.AddTask(task);
Console.WriteLine("Waiting for pool nodes to start...");
pool.Refresh();
while (pool.AllocationState.Value != AllocationState.Steady)
{
Thread.Sleep(10000);
pool.Refresh();
}
if (pool.CurrentDedicatedComputeNodes != dedicatedVmCount)
{
Console.WriteLine("Failed to allocate enough dedicated nodes: {0} or {1}", pool.TargetDedicatedComputeNodes, dedicatedVmCount);
Environment.Exit(1);
}
if (pool.CurrentLowPriorityComputeNodes != lowPriorityVmCount)
{
Console.WriteLine("Failed to allocate enough low priority nodes: {0} or {1}", pool.TargetDedicatedComputeNodes, dedicatedVmCount);
Environment.Exit(1);
}
WriteVRayConfig(pool);
Console.WriteLine("Waiting for VRay spawner to start...");
task = Client.JobOperations.GetTask(job.Id, task.Id);
while (task.State.Value == TaskState.Active)
{
Thread.Sleep(15000);
task = Client.JobOperations.GetTask(job.Id, task.Id);
}
Thread.Sleep(15000);
}
/// <summary>
/// Submit a large number of tasks to the Batch Service.
/// </summary>
/// <param name="client">The batch client.</param>
private static void SubmitLargeNumberOfTasks(IBatchClient client)
{
const int taskCountToCreate = 5000;
// In order to simulate a "large" task object which has many properties set (such as resource files, environment variables, etc)
// we create a big environment variable so we have a big task object.
char[] env = new char[2048];
for (int i = 0; i < env.Length; i++)
{
env[i] = 'a';
}
string envStr = new string(env);
using (IWorkItemManager wm = client.OpenWorkItemManager())
{
//Create a work item
string workItemName = Environment.GetEnvironmentVariable("USERNAME") + DateTime.Now.ToString("yyyyMMdd-HHmmss");
Console.WriteLine("Creating work item {0}", workItemName);
ICloudWorkItem cloudWorkItem = wm.CreateWorkItem(workItemName);
cloudWorkItem.JobExecutionEnvironment = new JobExecutionEnvironment()
{
PoolName = PoolName
}; //Specify the pool to run on
cloudWorkItem.Commit();
//Wait for an active job
TimeSpan maxJobCreationTimeout = TimeSpan.FromSeconds(90);
DateTime jobCreationStartTime = DateTime.Now;
DateTime jobCreationTimeoutTime = jobCreationStartTime.Add(maxJobCreationTimeout);
cloudWorkItem = wm.GetWorkItem(workItemName);
Console.WriteLine("Waiting for a job to become active...");
while (cloudWorkItem.ExecutionInformation == null || cloudWorkItem.ExecutionInformation.RecentJob == null)
{
cloudWorkItem.Refresh();
if (DateTime.Now > jobCreationTimeoutTime)
{
throw new Exception("Timed out waiting for job.");
}
Thread.Sleep(TimeSpan.FromSeconds(5));
}
string jobName = cloudWorkItem.ExecutionInformation.RecentJob.Name;
Console.WriteLine("Found job {0}. Adding task objects...", jobName);
//Generate a large number of tasks to submit
List <ICloudTask> tasksToSubmit = new List <ICloudTask>();
for (int i = 0; i < taskCountToCreate; i++)
{
ICloudTask task = new CloudTask("echo" + i.ToString("D5"), "echo");
List <IEnvironmentSetting> environmentSettings = new List <IEnvironmentSetting>();
environmentSettings.Add(new EnvironmentSetting("envone", envStr));
task.EnvironmentSettings = environmentSettings;
tasksToSubmit.Add(task);
}
BatchClientParallelOptions parallelOptions = new BatchClientParallelOptions()
{
//This will result in at most 10 simultaneous Bulk Add requests to the Batch Service.
MaxDegreeOfParallelism = 10
};
Console.WriteLine("Submitting {0} tasks to work item: {1}, job: {2}, on pool: {3}",
taskCountToCreate,
cloudWorkItem.Name,
jobName,
cloudWorkItem.JobExecutionEnvironment.PoolName);
Stopwatch stopwatch = new Stopwatch();
stopwatch.Start();
//Use the AddTask overload which supports a list of tasks for best AddTask performence - internally this method performs an
//intelligent submission of tasks in batches in order to limit the number of REST API calls made to the Batch Service.
wm.AddTask(cloudWorkItem.Name, jobName, tasksToSubmit, parallelOptions);
stopwatch.Stop();
Console.WriteLine("Submitted {0} tasks in {1}", taskCountToCreate, stopwatch.Elapsed);
//Delete the work item to ensure the tasks are cleaned up
wm.DeleteWorkItem(workItemName);
}
}
/// <summary>
/// Submit tasks which have dependant files.
/// The files are automatically uploaded to Azure Storage using the FileStaging feature of the Azure.Batch client library.
/// </summary>
private static void AddTasksWithFileStaging(BatchClient client, string sharedPoolId)
{
string jobId = CreateJobId("HelloWorldFileStagingJob");
Console.WriteLine("Creating job: " + jobId);
CloudJob boundJob = CreateBoundJob(client.JobOperations, sharedPoolId, jobId);
CloudTask taskToAdd1 = new CloudTask("task_with_file1", "cmd /c type *.txt");
CloudTask taskToAdd2 = new CloudTask("task_with_file2", "cmd /c dir /s");
//Set up a collection of files to be staged -- these files will be uploaded to Azure Storage
//when the tasks are submitted to the Azure Batch service.
taskToAdd1.FilesToStage = new List<IFileStagingProvider>();
taskToAdd2.FilesToStage = new List<IFileStagingProvider>();
// generate a local file in temp directory
Process cur = Process.GetCurrentProcess();
string path = Path.Combine(Environment.GetEnvironmentVariable("TEMP"), cur.Id + ".txt");
File.WriteAllText(path, "hello from " + cur.Id);
// add the files as a task dependency so they will be uploaded to storage before the task
// is submitted and downloaded to the VM before the task starts execution on the node
FileToStage file = new FileToStage(path, new StagingStorageAccount(StorageAccount, StorageKey, StorageBlobEndpoint));
taskToAdd1.FilesToStage.Add(file);
taskToAdd2.FilesToStage.Add(file); // filetostage object can be reused
// create a list of the tasks to add.
List<CloudTask> tasksToRun = new List<CloudTask> {taskToAdd1, taskToAdd2};
bool errors = false;
try
{
client.JobOperations.AddTask(boundJob.Id, tasksToRun);
}
catch (AggregateException ae)
{
errors = true;
// Go through all exceptions and dump useful information
ae.Handle(x =>
{
Console.Error.WriteLine("Adding tasks for job {0} failed", boundJob.Id);
if (x is BatchException)
{
BatchException be = x as BatchException;
if (null != be.RequestInformation && null != be.RequestInformation.AzureError)
{
// Write the server side error information
Console.Error.WriteLine(" AzureError.Code: " + be.RequestInformation.AzureError.Code);
Console.Error.WriteLine(" AzureError.Message.Value: " + be.RequestInformation.AzureError.Message.Value);
if (null != be.RequestInformation.AzureError.Values)
{
Console.Error.WriteLine(" AzureError.Values");
foreach (var v in be.RequestInformation.AzureError.Values)
{
Console.Error.WriteLine(" {0} : {1}", v.Key, v.Value);
}
}
Console.Error.WriteLine();
}
}
else
{
Console.WriteLine(x);
}
// Indicate that the error has been handled
return true;
});
}
// if there is no exception, wait for job response
if (!errors)
{
Console.WriteLine("Waiting for all tasks to complete on job: {0}...", boundJob.Id);
IPagedEnumerable<CloudTask> ourTasks = boundJob.ListTasks();
client.Utilities.CreateTaskStateMonitor().WaitAll(ourTasks, TaskState.Completed, TimeSpan.FromMinutes(30));
foreach (CloudTask task in ourTasks)
{
Console.WriteLine("Task " + task.Id);
Console.WriteLine("stdout:\n" + task.GetNodeFile(Constants.StandardOutFileName).ReadAsString());
Console.WriteLine("\nstderr:\n" + task.GetNodeFile(Constants.StandardErrorFileName).ReadAsString());
}
}
//Delete the job to ensure the tasks are cleaned up
Console.WriteLine("Deleting job: {0}", boundJob.Id);
client.JobOperations.DeleteJob(boundJob.Id);
}
public async Task <string> GetTaskState(Guid guid)
{
CloudTask task = await GetTask(guid);
return(task.State.ToString());
}
public TaskModel(JobModel parentJob, CloudTask task)
{
this.ParentJob = parentJob;
this.Task = task;
this.LastUpdatedTime = DateTime.UtcNow;
this.SubtasksInfo = null;
this.OutputFiles = task.ListNodeFiles();
}
public async Task <bool> GetTaskFailed(Guid guid)
{
CloudTask task = await GetTask(guid);
return(task.ExecutionInformation.Result == TaskExecutionResult.Failure);
}
static void Main(string[] args)
{
BatchSharedKeyCredentials cred = new BatchSharedKeyCredentials(endpoint, account, key);
using (BatchClient client = BatchClient.Open(cred)) // <-- connect to the cluster
{
List<ResourceFile> resources = new List<ResourceFile>();
foreach (string blob in StorageHelper.ListBlobs(resourceContainer))
{
string filename = System.IO.Path.GetFileName((new Uri(blob)).LocalPath);
resources.Add(new ResourceFile(StorageHelper.GetBlobSASURL(blob), filename));
}
CloudPool pool = client.PoolOperations.CreatePool(poolname, "4", "medium", 10);
pool.StartTask = new StartTask();
pool.StartTask.ResourceFiles = resources;
pool.StartTask.CommandLine = @"cmd /c copy *.* %WATASK_TVM_ROOT_DIR%\shared\";
pool.StartTask.WaitForSuccess = true;
//pool.Commit(); // <-- Create demo pool
// Submit Job
string jobname = "MVP_" + Environment.GetEnvironmentVariable("USERNAME") + "_" + DateTime.Now.ToString("yyyyMMdd-HHmmss");
PoolInformation poolinfo = new PoolInformation();
poolinfo.PoolId = poolname;
CloudJob job = client.JobOperations.CreateJob(jobname, poolinfo); // <-- create a job that runs on demo pool
Console.WriteLine("Creating job..." + jobname);
job.Commit();
job = client.JobOperations.GetJob(jobname);
string inputcontainersas = StorageHelper.GetContainerSAS(inputContainer);
string outputcontainersas = StorageHelper.GetContainerSAS(outputContainer);
List<CloudTask> tasks = new List<CloudTask>();
Console.WriteLine("Analyzing blobs...");
foreach (string blob in StorageHelper.ListBlobs(inputContainer)) // <-- Going through blobs
{
string filename = System.IO.Path.GetFileName((new Uri(blob)).LocalPath);
string taskname = "task_" + System.IO.Path.GetFileNameWithoutExtension(filename);
// prepare the command line
string cli;
cli = ". robocopy.exe ${env:WATASK_TVM_ROOT_DIR}\\shared\\ . *.*;";
cli += "ffmpeg.exe -i {0} -vf 'movie=microsoft.png [watermark]; [in][watermark] overlay=10:main_h-overlay_h-10 [out]' {0}.output.avi;".Replace("{0}", filename);
cli += "azcopy.exe . {0} *.output.avi /destsas:'{1}' /y".Replace("{0}", outputContainer).Replace("{1}", outputcontainersas);
cli = string.Format("powershell -command \"{0}\"", cli);
// prepare task object
CloudTask task = new CloudTask(taskname, cli);
task.ResourceFiles = new List<ResourceFile>();
task.ResourceFiles.Add(new ResourceFile(blob + inputcontainersas, filename));
tasks.Add(task); // <-- prepare 1 task for 1 blob
}
Console.WriteLine("Submit tasks...");
client.JobOperations.AddTask(jobname, tasks); // <-- Submit all 100 tasks with 1 API call
Console.WriteLine("Waiting for tasks to finish...");
client.Utilities.CreateTaskStateMonitor().WaitAll(client.JobOperations.ListTasks(jobname), TaskState.Completed, new TimeSpan(0, 60, 0));
Console.WriteLine("Closing job...");
client.JobOperations.TerminateJob(jobname);
Console.WriteLine("All done. Press Enter to exit.");
Console.ReadLine();
}
}
public async void GetTaskResult(Guid guid)
{
CloudTask task = await GetTask(guid);
TaskOutputStorage output = task.OutputStorage(DatabaseConnection.account);
}
/// <summary>
/// Creates tasks to process each of the specified input files, and submits them to the
/// specified job for execution.
/// </summary>
/// <param name="batchClient">A <see cref="BatchClient"/>.</param>
/// <param name="jobId">The id of the job to which the tasks should be added.</param>
/// <param name="inputFiles">A collection of <see cref="ResourceFile"/> objects representing the input files to be
/// processed by the tasks executed on the compute nodes.</param>
/// <param name="outputContainerSasUrl">The shared access signature URL for the container within Azure Storage that
/// will receive the output files created by the tasks.</param>
/// <returns>A collection of the submitted tasks.</returns>
private static async Task<List<CloudTask>> AddTasksAsync(BatchClient batchClient, string jobId, List<ResourceFile> inputFiles, string outputContainerSasUrl)
{
Console.WriteLine("Adding {0} tasks to job [{1}]...", inputFiles.Count, jobId);
// Create a collection to hold the tasks that we'll be adding to the job
List<CloudTask> tasks = new List<CloudTask>();
// Create each of the tasks. Because we copied the task application to the
// node's shared directory with the pool's StartTask, we can access it via
// the shared directory on whichever node each task will run.
foreach (ResourceFile inputFile in inputFiles)
{
string taskId = "topNtask" + inputFiles.IndexOf(inputFile);
string taskCommandLine = String.Format("cmd /c %AZ_BATCH_NODE_SHARED_DIR%\\TaskApplication.exe {0} 3 \"{1}\"", inputFile.FilePath, outputContainerSasUrl);
CloudTask task = new CloudTask(taskId, taskCommandLine);
task.ResourceFiles = new List<ResourceFile> { inputFile };
tasks.Add(task);
}
// Add the tasks as a collection opposed to a separate AddTask call for each. Bulk task submission
// helps to ensure efficient underlying API calls to the Batch service.
await batchClient.JobOperations.AddTaskAsync(jobId, tasks);
return tasks;
}
public void RunTaskAndUploadFiles_FilesAreSuccessfullyUploaded()
{
Action test = async() =>
{
string containerName = "runtaskanduploadfiles";
StagingStorageAccount storageAccount = TestUtilities.GetStorageCredentialsFromEnvironment();
CloudStorageAccount cloudStorageAccount = new CloudStorageAccount(
new StorageCredentials(storageAccount.StorageAccount, storageAccount.StorageAccountKey),
blobEndpoint: storageAccount.BlobUri,
queueEndpoint: null,
tableEndpoint: null,
fileEndpoint: null);
CloudBlobClient blobClient = cloudStorageAccount.CreateCloudBlobClient();
using (BatchClient batchCli = TestUtilities.OpenBatchClient(TestUtilities.GetCredentialsFromEnvironment()))
{
string jobId = "RunTaskAndUploadFiles-" + TestUtilities.GetMyName();
try
{
// Create container and writeable SAS
var container = blobClient.GetContainerReference(containerName);
await container.CreateIfNotExistsAsync();
var sas = container.GetSharedAccessSignature(new SharedAccessBlobPolicy()
{
Permissions = SharedAccessBlobPermissions.Write,
SharedAccessExpiryTime = DateTime.UtcNow.AddDays(1)
});
var fullSas = container.Uri + sas;
CloudJob createJob = batchCli.JobOperations.CreateJob(jobId, new PoolInformation()
{
PoolId = this.poolFixture.PoolId
});
createJob.Commit();
const string blobPrefix = "foo/bar";
const string taskId = "simpletask";
CloudTask unboundTask = new CloudTask(taskId, "echo test")
{
OutputFiles = new List <OutputFile>
{
new OutputFile(
filePattern: @"../*.txt",
destination: new OutputFileDestination(new OutputFileBlobContainerDestination(fullSas, blobPrefix)),
uploadOptions: new OutputFileUploadOptions(uploadCondition: OutputFileUploadCondition.TaskCompletion))
}
};
batchCli.JobOperations.AddTask(jobId, unboundTask);
var tasks = batchCli.JobOperations.ListTasks(jobId);
var monitor = batchCli.Utilities.CreateTaskStateMonitor();
monitor.WaitAll(tasks, TaskState.Completed, TimeSpan.FromMinutes(1));
// Ensure that the correct files got uploaded
var blobs = await BlobStorageExtensions.ListBlobs(container, useFlatBlobListing : true);
blobs = blobs.ToList();
Assert.Equal(4, blobs.Count()); //There are 4 .txt files created, stdout, stderr, fileuploadout, and fileuploaderr
foreach (var blob in blobs)
{
var blockBlob = blob as CloudBlockBlob;
Assert.StartsWith(blobPrefix, blockBlob.Name);
}
}
finally
{
TestUtilities.DeleteJobIfExistsAsync(batchCli, jobId).Wait();
var container = blobClient.GetContainerReference(containerName);
await container.DeleteIfExistsAsync();
}
}
};
SynchronizationContextHelper.RunTest(test, TestTimeout);
}
public TaskModel(JobModel parentJob, CloudTask task)
{
this.ParentJob = parentJob;
this.Task = task;
this.LastUpdatedTime = DateTime.UtcNow;
}
private static async Task MainAsync(string[] args)
{
const string poolId = "JobPrepReleaseSamplePool";
const string jobId = "JobPrepReleaseSampleJob";
// Location of the file that the job tasks will work with, a text file in the
// node's "shared" directory.
const string taskOutputFile = "%AZ_BATCH_NODE_SHARED_DIR%\\job_prep_and_release.txt";
// The job prep task will write the node ID to the text file in the shared directory
const string jobPrepCmdLine = "cmd /c echo %AZ_BATCH_NODE_ID% tasks: >" + taskOutputFile;
// Each task then echoes its ID to the same text file
const string taskCmdLine = "cmd /c echo %AZ_BATCH_TASK_ID% >> " + taskOutputFile;
// The job release task will then delete the text file from the shared directory
const string jobReleaseCmdLine = "cmd /c del " + taskOutputFile;
// Configure your AccountSettings in the Microsoft.Azure.Batch.Samples.Common project within this solution
BatchSharedKeyCredentials cred = new BatchSharedKeyCredentials(AccountSettings.Default.BatchServiceUrl,
AccountSettings.Default.BatchAccountName,
AccountSettings.Default.BatchAccountKey);
// Initialize the BatchClient for access to your Batch account
using (BatchClient batchClient = await BatchClient.OpenAsync(cred))
{
// Create a CloudPool (or obtain an existing pool with the specified ID)
CloudPool pool = await ArticleHelpers.CreatePoolIfNotExistAsync(batchClient,
poolId,
"small",
2,
1);
// Create a CloudJob (or obtain an existing job with the specified ID)
CloudJob job = await SampleHelpers.GetJobIfExistAsync(batchClient, jobId);
if (job == null)
{
Console.WriteLine("Job {0} not found, creating...", jobId);
CloudJob unboundJob = batchClient.JobOperations.CreateJob(jobId, new PoolInformation() { PoolId = poolId });
// Configure and assign the job preparation task
unboundJob.JobPreparationTask = new JobPreparationTask { CommandLine = jobPrepCmdLine };
// Configure and assign the job release task
unboundJob.JobReleaseTask = new JobReleaseTask { CommandLine = jobReleaseCmdLine };
await unboundJob.CommitAsync();
// Get the bound version of the job with all of its properties populated
job = await batchClient.JobOperations.GetJobAsync(jobId);
}
// Create the tasks that the job will execute
List<CloudTask> tasks = new List<CloudTask>();
for (int i = 1; i <= 8; i++)
{
string taskId = "task" + i.ToString().PadLeft(3, '0');
string taskCommandLine = taskCmdLine;
CloudTask task = new CloudTask(taskId, taskCommandLine);
tasks.Add(task);
}
// Add the tasks in one API call as opposed to a separate AddTask call for each. Bulk task
// submission helps to ensure efficient underlying API calls to the Batch service.
Console.WriteLine("Submitting tasks and awaiting completion...");
await batchClient.JobOperations.AddTaskAsync(job.Id, tasks);
// Wait for the tasks to complete before proceeding. The long timeout here is to allow time
// for the nodes within the pool to be created and started if the pool had not yet been created.
if (await batchClient.Utilities.CreateTaskStateMonitor().WhenAllAsync(job.ListTasks(),
TaskState.Completed,
TimeSpan.FromMinutes(30)))
{
Console.WriteLine("Operation timed out while waiting for submitted tasks to reach state {0}", TaskState.Completed);
return;
}
else
{
Console.WriteLine("All tasks completed.");
Console.WriteLine();
}
// Print the contents of the shared text file modified by the job preparation and other tasks.
ODATADetailLevel nodeDetail = new ODATADetailLevel(selectClause: "id, state");
IPagedEnumerable<ComputeNode> nodes = batchClient.PoolOperations.ListComputeNodes(pool.Id, nodeDetail);
await nodes.ForEachAsync(async (node) =>
{
// Check to ensure that the node is Idle before attempting to pull the text file.
// If the pool was just created, there is a chance that another node completed all
// of the tasks prior to the other node(s) completing their startup procedure.
if (node.State == ComputeNodeState.Idle)
{
NodeFile sharedTextFile = await node.GetNodeFileAsync("shared\\job_prep_and_release.txt");
Console.WriteLine("Contents of {0} on {1}:", sharedTextFile.Name, node.Id);
Console.WriteLine("-------------------------------------------");
Console.WriteLine(await sharedTextFile.ReadAsStringAsync());
}
});
// Terminate the job to mark it as Completed; this will initiate the Job Release Task on any node
// that executed job tasks. Note that the Job Release Task is also executed when a job is deleted,
// thus you need not call Terminate if you typically delete your jobs upon task completion.
await batchClient.JobOperations.TerminateJobAsync(job.Id);
// Wait for the job to reach state "Completed." Note that this wait is not typically necessary in
// production code, but is done here to enable the checking of the release tasks exit code below.
await ArticleHelpers.WaitForJobToReachStateAsync(batchClient, job.Id, JobState.Completed, TimeSpan.FromMinutes(2));
// Print the exit codes of the prep and release tasks by obtaining their execution info
List<JobPreparationAndReleaseTaskExecutionInformation> prepReleaseInfo = await batchClient.JobOperations.ListJobPreparationAndReleaseTaskStatus(job.Id).ToListAsync();
foreach (JobPreparationAndReleaseTaskExecutionInformation info in prepReleaseInfo)
{
Console.WriteLine();
Console.WriteLine("{0}: ", info.ComputeNodeId);
// If no tasks were scheduled to run on the node, the JobPreparationTaskExecutionInformation will be null
if (info.JobPreparationTaskExecutionInformation != null)
{
Console.WriteLine(" Prep task exit code: {0}", info.JobPreparationTaskExecutionInformation.ExitCode);
}
// If no tasks were scheduled to run on the node, the JobReleaseTaskExecutionInformation will be null
if (info.JobReleaseTaskExecutionInformation != null)
{
Console.WriteLine(" Release task exit code: {0}", info.JobReleaseTaskExecutionInformation.ExitCode);
}
}
// Clean up the resources we've created in the Batch account
Console.WriteLine();
Console.WriteLine("Delete job? [yes] no");
string response = Console.ReadLine().ToLower();
if (response != "n" && response != "no")
{
// Note that deleting the job will execute the job release task if the job was not previously terminated
await batchClient.JobOperations.DeleteJobAsync(job.Id);
}
Console.WriteLine("Delete pool? [yes] no");
response = Console.ReadLine();
if (response != "n" && response != "no")
{
await batchClient.PoolOperations.DeletePoolAsync(pool.Id);
}
}
}
/// <summary>
/// Submits a set of tasks to the job
/// </summary>
/// <param name="batchClient">The batch client to use.</param>
/// <returns>The set of blob artifacts created by file staging.</returns>
private async Task<HashSet<string>> SubmitTasks(BatchClient batchClient)
{
List<CloudTask> tasksToRun = new List<CloudTask>();
// Create a task which requires some resource files
CloudTask taskWithFiles = new CloudTask("task_with_file1", SimpleTaskExe);
// Set up a collection of files to be staged -- these files will be uploaded to Azure Storage
// when the tasks are submitted to the Azure Batch service.
taskWithFiles.FilesToStage = new List<IFileStagingProvider>();
// generate a local file in temp directory
string localSampleFilePath = GettingStartedCommon.GenerateTemporaryFile("HelloWorld.txt", "hello from Batch JobManager sample!");
StagingStorageAccount fileStagingStorageAccount = new StagingStorageAccount(
storageAccount: this.configurationSettings.StorageAccountName,
storageAccountKey: this.configurationSettings.StorageAccountKey,
blobEndpoint: this.configurationSettings.StorageBlobEndpoint);
// add the files as a task dependency so they will be uploaded to storage before the task
// is submitted and downloaded to the node before the task starts execution.
FileToStage helloWorldFile = new FileToStage(localSampleFilePath, fileStagingStorageAccount);
FileToStage simpleTaskFile = new FileToStage(SimpleTaskExe, fileStagingStorageAccount);
// When this task is added via JobOperations.AddTaskAsync below, the FilesToStage are uploaded to storage once.
// The Batch service does not automatically delete content from your storage account, so files added in this
// way must be manually removed when they are no longer used.
taskWithFiles.FilesToStage.Add(helloWorldFile);
taskWithFiles.FilesToStage.Add(simpleTaskFile);
tasksToRun.Add(taskWithFiles);
var fileStagingArtifacts = new ConcurrentBag<ConcurrentDictionary<Type, IFileStagingArtifact>>();
// Use the AddTask method which takes an enumerable of tasks for best performance, as it submits up to 100
// tasks at once in a single request. If the list of tasks is N where N > 100, this will correctly parallelize
// the requests and return when all N tasks have been added.
await batchClient.JobOperations.AddTaskAsync(jobId, tasksToRun, fileStagingArtifacts: fileStagingArtifacts);
// Extract the names of the blob containers from the file staging artifacts
HashSet<string> blobContainerNames = GettingStartedCommon.ExtractBlobContainerNames(fileStagingArtifacts);
return blobContainerNames;
}
/// <summary>
/// Performs a simple AddTask test, adding the specified task count using the specified parallelOptions and resultHandlerFunc
/// </summary>
/// <returns></returns>
private async System.Threading.Tasks.Task AddTasksSimpleTestAsync(
BatchClient batchCli,
string testName,
int taskCount,
BatchClientParallelOptions parallelOptions = null,
Func <AddTaskResult, CancellationToken, AddTaskResultStatus> resultHandlerFunc = null,
StagingStorageAccount storageCredentials = null,
IEnumerable <string> localFilesToStage = null,
ConcurrentBag <ConcurrentDictionary <Type, IFileStagingArtifact> > fileStagingArtifacts = null,
TimeSpan?timeout = null,
bool useJobOperations = true,
List <ResourceFile> resourceFiles = null)
{
JobOperations jobOperations = batchCli.JobOperations;
string jobId = "Bulk-" + TestUtilities.GetMyName() + "-" + testName + "-" + useJobOperations;
try
{
CloudJob unboundJob = jobOperations.CreateJob();
this.testOutputHelper.WriteLine("Initial job commit for job: {0}", jobId);
unboundJob.PoolInformation = new PoolInformation()
{
PoolId = "DummyPool"
};
unboundJob.Id = jobId;
await unboundJob.CommitAsync().ConfigureAwait(false);
CloudJob boundJob = await jobOperations.GetJobAsync(jobId).ConfigureAwait(false);
//
// Add a simple set of tasks
//
IEnumerable <string> taskNames = GenerateTaskIds(taskCount);
List <CloudTask> tasksToAdd = new List <CloudTask>();
List <CloudTask> tasksToValidateWith = new List <CloudTask>();
IList <IFileStagingProvider> lastFilesToStageList = null;
foreach (string taskName in taskNames)
{
CloudTask myTask = new CloudTask(taskName, "cmd /c echo hello world");
CloudTask duplicateReadableTask = new CloudTask(taskName, "cmd /c echo hello world");
myTask.ResourceFiles = resourceFiles;
duplicateReadableTask.ResourceFiles = resourceFiles;
if (localFilesToStage != null && storageCredentials != null)
{
myTask.FilesToStage = new List <IFileStagingProvider>();
lastFilesToStageList = myTask.FilesToStage;
duplicateReadableTask.FilesToStage = new List <IFileStagingProvider>();
foreach (string fileToStage in localFilesToStage)
{
duplicateReadableTask.FilesToStage.Add(new FileToStage(fileToStage, storageCredentials));
myTask.FilesToStage.Add(new FileToStage(fileToStage, storageCredentials));
}
}
tasksToAdd.Add(myTask);
tasksToValidateWith.Add(duplicateReadableTask);
}
List <BatchClientBehavior> behaviors = new List <BatchClientBehavior>();
if (resultHandlerFunc != null)
{
behaviors.Add(new AddTaskCollectionResultHandler(resultHandlerFunc));
}
//Add the tasks
Stopwatch stopwatch = new Stopwatch();
this.testOutputHelper.WriteLine("Starting task add");
stopwatch.Start();
if (useJobOperations)
{
await jobOperations.AddTaskAsync(
jobId,
tasksToAdd,
parallelOptions : parallelOptions,
fileStagingArtifacts : fileStagingArtifacts,
timeout : timeout,
additionalBehaviors : behaviors).ConfigureAwait(continueOnCapturedContext: false);
}
else
{
await boundJob.AddTaskAsync(
tasksToAdd,
parallelOptions : parallelOptions,
fileStagingArtifacts : fileStagingArtifacts,
timeout : timeout,
additionalBehaviors : behaviors).ConfigureAwait(continueOnCapturedContext: false);
}
stopwatch.Stop();
this.testOutputHelper.WriteLine("Task add finished, took: {0}", stopwatch.Elapsed);
if (lastFilesToStageList != null)
{
TestUtilities.AssertThrows <InvalidOperationException>(() => lastFilesToStageList.Add(new FileToStage("test", null)));
}
//Ensure the task lists match
List <CloudTask> tasksFromService = await jobOperations.ListTasks(jobId).ToListAsync().ConfigureAwait(false);
EnsureTasksListsMatch(tasksToValidateWith, tasksFromService);
}
catch (Exception e)
{
this.testOutputHelper.WriteLine("Exception: {0}", e.ToString());
throw;
}
finally
{
TestUtilities.DeleteJobIfExistsAsync(batchCli, jobId).Wait();
}
}
/// <summary>
/// Creates a new task.
/// </summary>
/// <param name="parameters">The parameters to use when creating the task.</param>
public void CreateTask(NewTaskParameters parameters)
{
if (parameters == null)
{
throw new ArgumentNullException("parameters");
}
CloudTask task = new CloudTask(parameters.TaskId, parameters.CommandLine);
task.DisplayName = parameters.DisplayName;
task.RunElevated = parameters.RunElevated;
if (parameters.EnvironmentSettings != null)
{
task.EnvironmentSettings = new List<EnvironmentSetting>();
foreach (DictionaryEntry d in parameters.EnvironmentSettings)
{
EnvironmentSetting setting = new EnvironmentSetting(d.Key.ToString(), d.Value.ToString());
task.EnvironmentSettings.Add(setting);
}
}
if (parameters.ResourceFiles != null)
{
task.ResourceFiles = new List<ResourceFile>();
foreach (DictionaryEntry d in parameters.ResourceFiles)
{
ResourceFile file = new ResourceFile(d.Value.ToString(), d.Key.ToString());
task.ResourceFiles.Add(file);
}
}
if (parameters.AffinityInformation != null)
{
task.AffinityInformation = parameters.AffinityInformation.omObject;
}
if (parameters.Constraints != null)
{
task.Constraints = parameters.Constraints.omObject;
}
WriteVerbose(string.Format(Resources.CreatingTask, parameters.TaskId));
if (parameters.Job != null)
{
parameters.Job.omObject.AddTask(task, parameters.AdditionalBehaviors);
}
else
{
JobOperations jobOperations = parameters.Context.BatchOMClient.JobOperations;
jobOperations.AddTask(parameters.JobId, task, parameters.AdditionalBehaviors);
}
}
private static async Task MainAsync(string[] args)
{
// You may adjust these values to experiment with different compute resource scenarios.
const string nodeSize = "standard_d1_v2";
const int nodeCount = 4;
const int maxTasksPerNode = 4;
const int taskCount = 32;
// Ensure there are enough tasks to help avoid hitting some timeout conditions below
int minimumTaskCount = nodeCount * maxTasksPerNode * 2;
if (taskCount < minimumTaskCount)
{
Console.WriteLine("You must specify at least two tasks per node core for this sample ({0} tasks in this configuration).", minimumTaskCount);
Console.WriteLine();
// Not enough tasks, exit the application
return;
}
// In this sample, the tasks simply ping localhost on the compute nodes; adjust these
// values to simulate variable task duration
const int minPings = 30;
const int maxPings = 60;
const string poolId = "ParallelTasksSamplePool";
const string jobId = "ParallelTasksSampleJob";
// Amount of time to wait before timing out (potentially) long-running tasks
TimeSpan longTaskDurationLimit = TimeSpan.FromMinutes(30);
// Set up access to your Batch account with a BatchClient. Configure your AccountSettings in the
// Microsoft.Azure.Batch.Samples.Common project within this solution.
AccountSettings accountSettings = SampleHelpers.LoadAccountSettings();
BatchSharedKeyCredentials cred = new BatchSharedKeyCredentials(
accountSettings.BatchServiceUrl,
accountSettings.BatchAccountName,
accountSettings.BatchAccountKey);
using (BatchClient batchClient = BatchClient.Open(cred))
{
// Create a CloudPool, or obtain an existing pool with the specified ID
CloudPool pool = await ArticleHelpers.CreatePoolIfNotExistAsync(
batchClient,
poolId,
nodeSize,
nodeCount,
maxTasksPerNode);
// Create a CloudJob, or obtain an existing pool with the specified ID
CloudJob job = await ArticleHelpers.CreateJobIfNotExistAsync(batchClient, poolId, jobId);
// The job's tasks ping localhost a random number of times between minPings and maxPings.
// Adjust the minPings/maxPings values above to experiment with different task durations.
Random rand = new Random();
List <CloudTask> tasks = new List <CloudTask>();
for (int i = 1; i <= taskCount; i++)
{
string taskId = "task" + i.ToString().PadLeft(3, '0');
string taskCommandLine = "ping -n " + rand.Next(minPings, maxPings + 1).ToString() + " localhost";
CloudTask task = new CloudTask(taskId, taskCommandLine);
tasks.Add(task);
}
// Pause execution until the pool is steady and its compute nodes are ready to accept jobs.
// NOTE: Such a pause is not necessary within your own code. Tasks can be added to a job at any point and will be
// scheduled to execute on a compute node as soon any node has reached Idle state. Because the focus of this sample
// is the demonstration of running tasks in parallel on multiple compute nodes, we wait for all compute nodes to
// complete initialization and reach the Idle state in order to maximize the number of compute nodes available for
// parallelization.
await ArticleHelpers.WaitForPoolToReachStateAsync(batchClient, pool.Id, AllocationState.Steady, longTaskDurationLimit);
await ArticleHelpers.WaitForNodesToReachStateAsync(batchClient, pool.Id, ComputeNodeState.Idle, longTaskDurationLimit);
// Add the tasks in one API call as opposed to a separate AddTask call for each. Bulk task submission
// helps to ensure efficient underlying API calls to the Batch service.
await batchClient.JobOperations.AddTaskAsync(job.Id, tasks);
// Pause again to wait until *all* nodes are running tasks
await ArticleHelpers.WaitForNodesToReachStateAsync(batchClient, pool.Id, ComputeNodeState.Running, TimeSpan.FromMinutes(2));
Stopwatch stopwatch = Stopwatch.StartNew();
// Print out task assignment information.
Console.WriteLine();
await GettingStartedCommon.PrintNodeTasksAsync(batchClient, pool.Id);
Console.WriteLine();
// Pause execution while we wait for all of the tasks to complete
Console.WriteLine("Waiting for task completion...");
Console.WriteLine();
try
{
await batchClient.Utilities.CreateTaskStateMonitor().WhenAll(
job.ListTasks(),
TaskState.Completed,
longTaskDurationLimit);
}
catch (TimeoutException e)
{
Console.WriteLine(e.ToString());
}
stopwatch.Stop();
// Obtain the tasks, specifying a detail level to limit the number of properties returned for each task.
// If you have a large number of tasks, specifying a DetailLevel is extremely important in reducing the
// amount of data transferred, lowering your query response times in increasing performance.
ODATADetailLevel detail = new ODATADetailLevel(selectClause: "id,commandLine,nodeInfo,state");
IPagedEnumerable <CloudTask> allTasks = batchClient.JobOperations.ListTasks(job.Id, detail);
// Get a collection of the completed tasks sorted by the compute nodes on which they executed
List <CloudTask> completedTasks = allTasks
.Where(t => t.State == TaskState.Completed)
.OrderBy(t => t.ComputeNodeInformation.ComputeNodeId)
.ToList();
// Print the completed task information
Console.WriteLine();
Console.WriteLine("Completed tasks:");
string lastNodeId = string.Empty;
foreach (CloudTask task in completedTasks)
{
if (!string.Equals(lastNodeId, task.ComputeNodeInformation.ComputeNodeId))
{
Console.WriteLine();
Console.WriteLine(task.ComputeNodeInformation.ComputeNodeId);
}
lastNodeId = task.ComputeNodeInformation.ComputeNodeId;
Console.WriteLine("\t{0}: {1}", task.Id, task.CommandLine);
}
// Get a collection of the uncompleted tasks which may exist if the TaskMonitor timeout was hit
List <CloudTask> uncompletedTasks = allTasks
.Where(t => t.State != TaskState.Completed)
.OrderBy(t => t.Id)
.ToList();
// Print a list of uncompleted tasks, if any
Console.WriteLine();
Console.WriteLine("Uncompleted tasks:");
Console.WriteLine();
if (uncompletedTasks.Any())
{
foreach (CloudTask task in uncompletedTasks)
{
Console.WriteLine("\t{0}: {1}", task.Id, task.CommandLine);
}
}
else
{
Console.WriteLine("\t<none>");
}
// Print some summary information
Console.WriteLine();
Console.WriteLine(" Nodes: " + nodeCount);
Console.WriteLine(" Node size: " + nodeSize);
Console.WriteLine("Max tasks per node: " + pool.MaxTasksPerComputeNode);
Console.WriteLine(" Tasks: " + tasks.Count);
Console.WriteLine(" Duration: " + stopwatch.Elapsed);
Console.WriteLine();
Console.WriteLine("Done!");
Console.WriteLine();
// Clean up the resources we've created in the Batch account
Console.WriteLine("Delete job? [yes] no");
string response = Console.ReadLine().ToLower();
if (response != "n" && response != "no")
{
await batchClient.JobOperations.DeleteJobAsync(job.Id);
}
Console.WriteLine("Delete pool? [yes] no");
response = Console.ReadLine();
if (response != "n" && response != "no")
{
await batchClient.PoolOperations.DeletePoolAsync(pool.Id);
}
}
}
/// <summary>
/// Adds a task.
/// </summary>
/// <param name="options">The options describing the task to add.</param>
/// <returns></returns>
public async Task AddTaskAsync(AddTaskOptions options)
{
CloudTask unboundTask = new CloudTask(options.TaskId, options.CommandLine);
await this.Client.JobOperations.AddTaskAsync(options.JobId, unboundTask);
}
private static async Task MainAsync(string[] args)
{
const string poolId = "JobPrepReleaseSamplePool";
const string jobId = "JobPrepReleaseSampleJob";
// Location of the file that the job tasks will work with, a text file in the
// node's "shared" directory.
const string taskOutputFile = "%AZ_BATCH_NODE_SHARED_DIR%\\job_prep_and_release.txt";
// The job prep task will write the node ID to the text file in the shared directory
const string jobPrepCmdLine = "cmd /c echo %AZ_BATCH_NODE_ID% tasks: >" + taskOutputFile;
// Each task then echoes its ID to the same text file
const string taskCmdLine = "cmd /c echo %AZ_BATCH_TASK_ID% >> " + taskOutputFile;
// The job release task will then delete the text file from the shared directory
const string jobReleaseCmdLine = "cmd /c del " + taskOutputFile;
var accountSettings = SampleHelpers.LoadAccountSettings();
// Configure your AccountSettings in the Microsoft.Azure.Batch.Samples.Common project within this solution
BatchSharedKeyCredentials cred = new BatchSharedKeyCredentials(
accountSettings.BatchServiceUrl,
accountSettings.BatchAccountName,
accountSettings.BatchAccountKey);
// Initialize the BatchClient for access to your Batch account
using (BatchClient batchClient = await BatchClient.OpenAsync(cred))
{
// Create a CloudPool (or obtain an existing pool with the specified ID)
CloudPool pool = await ArticleHelpers.CreatePoolIfNotExistAsync(
batchClient,
poolId,
"standard_d1_v2",
2,
1);
// Create a CloudJob (or obtain an existing job with the specified ID)
CloudJob job = await SampleHelpers.GetJobIfExistAsync(batchClient, jobId);
if (job == null)
{
Console.WriteLine("Job {0} not found, creating...", jobId);
CloudJob unboundJob = batchClient.JobOperations.CreateJob(jobId, new PoolInformation()
{
PoolId = poolId
});
// Configure and assign the job preparation task
unboundJob.JobPreparationTask = new JobPreparationTask {
CommandLine = jobPrepCmdLine
};
// Configure and assign the job release task
unboundJob.JobReleaseTask = new JobReleaseTask {
CommandLine = jobReleaseCmdLine
};
await unboundJob.CommitAsync();
// Get the bound version of the job with all of its properties populated
job = await batchClient.JobOperations.GetJobAsync(jobId);
}
// Create the tasks that the job will execute
List <CloudTask> tasks = new List <CloudTask>();
for (int i = 1; i <= 8; i++)
{
string taskId = "task" + i.ToString().PadLeft(3, '0');
string taskCommandLine = taskCmdLine;
CloudTask task = new CloudTask(taskId, taskCommandLine);
tasks.Add(task);
}
// Add the tasks in one API call as opposed to a separate AddTask call for each. Bulk task
// submission helps to ensure efficient underlying API calls to the Batch service.
Console.WriteLine("Submitting tasks and awaiting completion...");
await batchClient.JobOperations.AddTaskAsync(job.Id, tasks);
// Wait for the tasks to complete before proceeding. The long timeout here is to allow time
// for the nodes within the pool to be created and started if the pool had not yet been created.
await batchClient.Utilities.CreateTaskStateMonitor().WhenAll(
job.ListTasks(),
TaskState.Completed,
TimeSpan.FromMinutes(30));
Console.WriteLine("All tasks completed.");
Console.WriteLine();
// Print the contents of the shared text file modified by the job preparation and other tasks.
ODATADetailLevel nodeDetail = new ODATADetailLevel(selectClause: "id, state");
IPagedEnumerable <ComputeNode> nodes = batchClient.PoolOperations.ListComputeNodes(pool.Id, nodeDetail);
await nodes.ForEachAsync(async (node) =>
{
// Check to ensure that the node is Idle before attempting to pull the text file.
// If the pool was just created, there is a chance that another node completed all
// of the tasks prior to the other node(s) completing their startup procedure.
if (node.State == ComputeNodeState.Idle)
{
NodeFile sharedTextFile = await node.GetNodeFileAsync("shared\\job_prep_and_release.txt");
Console.WriteLine("Contents of {0} on {1}:", sharedTextFile.Path, node.Id);
Console.WriteLine("-------------------------------------------");
Console.WriteLine(await sharedTextFile.ReadAsStringAsync());
}
});
// Terminate the job to mark it as Completed; this will initiate the Job Release Task on any node
// that executed job tasks. Note that the Job Release Task is also executed when a job is deleted,
// thus you need not call Terminate if you typically delete your jobs upon task completion.
await batchClient.JobOperations.TerminateJobAsync(job.Id);
// Wait for the job to reach state "Completed." Note that this wait is not typically necessary in
// production code, but is done here to enable the checking of the release tasks exit code below.
await ArticleHelpers.WaitForJobToReachStateAsync(batchClient, job.Id, JobState.Completed, TimeSpan.FromMinutes(2));
// Print the exit codes of the prep and release tasks by obtaining their execution info
List <JobPreparationAndReleaseTaskExecutionInformation> prepReleaseInfo = await batchClient.JobOperations.ListJobPreparationAndReleaseTaskStatus(job.Id).ToListAsync();
foreach (JobPreparationAndReleaseTaskExecutionInformation info in prepReleaseInfo)
{
Console.WriteLine();
Console.WriteLine("{0}: ", info.ComputeNodeId);
// If no tasks were scheduled to run on the node, the JobPreparationTaskExecutionInformation will be null
if (info.JobPreparationTaskExecutionInformation != null)
{
Console.WriteLine(" Prep task exit code: {0}", info.JobPreparationTaskExecutionInformation.ExitCode);
}
// If no tasks were scheduled to run on the node, the JobReleaseTaskExecutionInformation will be null
if (info.JobReleaseTaskExecutionInformation != null)
{
Console.WriteLine(" Release task exit code: {0}", info.JobReleaseTaskExecutionInformation.ExitCode);
}
}
// Clean up the resources we've created in the Batch account
Console.WriteLine();
Console.WriteLine("Delete job? [yes] no");
string response = Console.ReadLine().ToLower();
if (response != "n" && response != "no")
{
// Note that deleting the job will execute the job release task if the job was not previously terminated
await batchClient.JobOperations.DeleteJobAsync(job.Id);
}
Console.WriteLine("Delete pool? [yes] no");
response = Console.ReadLine();
if (response != "n" && response != "no")
{
await batchClient.PoolOperations.DeletePoolAsync(pool.Id);
}
}
}
public TaskModel(JobModel parentJob, CloudTask task)
{
this.attemptToLoadOutputs = true;
this.ParentJob = parentJob;
this.Task = task;
this.LastUpdatedTime = DateTime.UtcNow;
this.SubtasksInfo = null;
}
public async Task RunTaskAndUploadFiles_FilesAreSuccessfullyUploaded()
{
async Task test()
{
using BatchClient batchCli = TestUtilities.OpenBatchClient(TestUtilities.GetCredentialsFromEnvironment());
string jobId = "RunTaskAndUploadFiles-" + TestUtilities.GetMyName();
string containerName = "runtaskanduploadfiles";
StagingStorageAccount storageAccount = TestUtilities.GetStorageCredentialsFromEnvironment();
BlobServiceClient blobClient = BlobUtilities.GetBlobServiceClient(storageAccount);
BlobContainerClient containerClient = BlobUtilities.GetBlobContainerClient(containerName, blobClient, storageAccount);
try
{
// Create container and writeable SAS
containerClient.CreateIfNotExists();
string sasUri = BlobUtilities.GetWriteableSasUri(containerClient, storageAccount);
CloudJob createJob = batchCli.JobOperations.CreateJob(jobId, new PoolInformation {
PoolId = poolFixture.PoolId
});
createJob.Commit();
const string blobPrefix = "foo/bar";
const string taskId = "simpletask";
OutputFileBlobContainerDestination containerDestination = new OutputFileBlobContainerDestination(sasUri, blobPrefix);
containerDestination.UploadHeaders = new List <HttpHeader> {
new HttpHeader("x-ms-blob-content-type", "test-type")
};
OutputFileDestination destination = new OutputFileDestination(containerDestination);
OutputFileUploadOptions uploadOptions = new OutputFileUploadOptions(uploadCondition: OutputFileUploadCondition.TaskCompletion);
CloudTask unboundTask = new CloudTask(taskId, "echo test")
{
OutputFiles = new List <OutputFile>
{
new OutputFile(@"../*.txt", destination, uploadOptions)
}
};
batchCli.JobOperations.AddTask(jobId, unboundTask);
IPagedEnumerable <CloudTask> tasks = batchCli.JobOperations.ListTasks(jobId);
TaskStateMonitor monitor = batchCli.Utilities.CreateTaskStateMonitor();
monitor.WaitAll(tasks, TaskState.Completed, TimeSpan.FromMinutes(1));
// Ensure that the correct files got uploaded
List <BlobItem> blobs = containerClient.GetAllBlobs();
Assert.Equal(4, blobs.Count()); //There are 4 .txt files created, stdout, stderr, fileuploadout, and fileuploaderr
foreach (BlobItem blob in blobs)
{
Assert.StartsWith(blobPrefix, blob.Name);
Assert.Equal("test-type", blob.Properties.ContentType); // Ensure test Upload header was applied to blob.
}
}
finally
{
await TestUtilities.DeleteJobIfExistsAsync(batchCli, jobId).ConfigureAwait(false);
containerClient.DeleteIfExists();
}
}
await SynchronizationContextHelper.RunTestAsync(test, TestTimeout);
}
private async Task SubmitMapperTasksAsync(BatchClient batchClient, string containerSas)
{
Console.WriteLine("Submitting {0} mapper tasks.", this.configurationSettings.NumberOfMapperTasks);
//The collection of tasks to add to the Batch Service.
List<CloudTask> tasksToAdd = new List<CloudTask>();
for (int i = 0; i < this.configurationSettings.NumberOfMapperTasks; i++)
{
string taskId = Helpers.GetMapperTaskId(i);
string fileBlobName = Helpers.GetSplitFileName(i);
string fileBlobPath = SampleHelpers.ConstructBlobSource(containerSas, fileBlobName);
string commandLine = string.Format("{0} {1}", Constants.MapperTaskExecutable, fileBlobPath);
CloudTask unboundMapperTask = new CloudTask(taskId, commandLine);
//The set of files (exes, dlls and configuration files) required to run the mapper task.
IReadOnlyList<string> mapperTaskRequiredFiles = Constants.RequiredExecutableFiles;
List<ResourceFile> mapperTaskResourceFiles = SampleHelpers.GetResourceFiles(containerSas, mapperTaskRequiredFiles);
unboundMapperTask.ResourceFiles = mapperTaskResourceFiles;
tasksToAdd.Add(unboundMapperTask);
}
//Submit the unbound task collection to the Batch Service.
//Use the AddTask method which takes a collection of CloudTasks for the best performance.
await batchClient.JobOperations.AddTaskAsync(this.jobId, tasksToAdd);
}
public static void JobMain(string[] args)
{
Console.WriteLine("Setting up Batch Process - ImageBlur. \nPress Enter to begin.");
Console.WriteLine("-------------------------------------------------------------");
Console.ReadLine();
Settings imageBlurSettings = Settings.Default;
AccountSettings accountSettings = AccountSettings.Default;
/* Setting up credentials for Batch and Storage accounts
* =====================================================
*/
StorageCredentials storageCredentials = new StorageCredentials(
accountSettings.StorageAccountName,
accountSettings.StorageAccountKey);
CloudStorageAccount storageAccount = new CloudStorageAccount(storageCredentials, useHttps: true);
StagingStorageAccount stagingStorageAccount = new StagingStorageAccount(
accountSettings.StorageAccountName,
accountSettings.StorageAccountKey,
storageAccount.BlobEndpoint.ToString());
BatchSharedKeyCredentials batchCredentials = new BatchSharedKeyCredentials(
accountSettings.BatchServiceUrl,
accountSettings.BatchAccountName,
accountSettings.BatchAccountKey);
using (BatchClient client = BatchClient.Open(batchCredentials))
{
string stagingContainer = null;
/* Setting up pool to run job and tasks in
* =======================================
*/
CreatePool(client, imageBlurSettings, accountSettings);
try
{
/* Setting up Job ------------------------
* =======================================
*/
Console.WriteLine("Creating job {0}. \nPress Enter to continue.", imageBlurSettings.JobId);
Console.ReadLine();
CloudJob unboundJob = client.JobOperations.CreateJob();
unboundJob.Id = imageBlurSettings.JobId;
unboundJob.PoolInformation = new PoolInformation() { PoolId = imageBlurSettings.PoolId };
unboundJob.Commit();
/* Uploading Source Image(s) to run varying degrees of Blur on
* ===========================================================
* Here, the input data is uploaded separately to Storage and
* its URI is passed to the task as an argument.
*/
Console.WriteLine("Uploading source images. \nPress Enter to continue.");
Console.ReadLine();
string[] sourceImages = imageBlurSettings.SourceImageNames.Split(',');
List<String> sourceImageUris = new List<String>();
for( var i = 0; i < sourceImages.Length; i++)
{
Console.WriteLine(" Uploading {0}.", sourceImages[i]);
sourceImageUris.Add( UploadSourceImagesFileToCloudBlob(accountSettings, sourceImages[i]));
Console.WriteLine(" Source Image uploaded to: <{0}>.", sourceImageUris[i]);
}
Console.WriteLine();
Console.WriteLine("All Source Images uploaded. \nPress Enter to continue.");
Console.ReadLine();
/* Setting up tasks with dependencies ----------------
* ===================================================
*/
Console.WriteLine("Setting up files to stage for tasks. \nPress Enter to continue.");
Console.ReadLine();
// Setting up Files to Stage - Files to upload into each task (executables and dependent assemblies)
FileToStage imageBlurExe = new FileToStage(ImageBlurExeName, stagingStorageAccount);
FileToStage storageDll = new FileToStage(StorageClientDllName, stagingStorageAccount);
FileToStage imageProcessorDll = new FileToStage(ImageProcessorDllName, stagingStorageAccount);
// initialize collection to hold tasks that will be submitted in their entirety
List<CloudTask> tasksToRun = new List<CloudTask>(imageBlurSettings.NumberOfTasks);
for (int i = 0; i < imageBlurSettings.NumberOfTasks; i++)
{
// create individual tasks (cmd line passed in as argument)
CloudTask task = new CloudTask("task_" + i, String.Format("{0} --Task {1} {2} {3}",
ImageBlurExeName,
sourceImageUris[i],
accountSettings.StorageAccountName,
accountSettings.StorageAccountKey));
// list of files to stage to a container -- for each job, one container is created and
// files all resolve to Azure Blobs by their name
task.FilesToStage = new List<IFileStagingProvider> { imageBlurExe, storageDll, imageProcessorDll };
tasksToRun.Add(task);
Console.WriteLine("\t task {0} has been added", "task_" + i);
}
Console.WriteLine();
/* Commit tasks with dependencies ----------------
* ===============================================
*/
Console.WriteLine("Running Tasks. \nPress Enter to continue.");
Console.WriteLine("-------------------------------------------------------------");
Console.ReadLine();
ConcurrentBag<ConcurrentDictionary<Type, IFileStagingArtifact>> fsArtifactBag = new ConcurrentBag<ConcurrentDictionary<Type, IFileStagingArtifact>>();
client.JobOperations.AddTask(imageBlurSettings.JobId, tasksToRun, fileStagingArtifacts: fsArtifactBag);
foreach (var fsBagItem in fsArtifactBag)
{
IFileStagingArtifact fsValue;
if (fsBagItem.TryGetValue(typeof(FileToStage), out fsValue))
{
SequentialFileStagingArtifact stagingArtifact = fsValue as SequentialFileStagingArtifact;
if (stagingArtifact != null)
{
stagingContainer = stagingArtifact.BlobContainerCreated;
Console.WriteLine(
"Uploaded files to container: {0} -- \nyou will be charged for their storage unless you delete them.",
stagingArtifact.BlobContainerCreated);
}
}
}
//Get the job to monitor status.
CloudJob job = client.JobOperations.GetJob(imageBlurSettings.JobId);
Console.WriteLine();
Console.Write("Waiting for tasks to complete ... ");
IPagedEnumerable<CloudTask> ourTasks = job.ListTasks(new ODATADetailLevel(selectClause: "id"));
client.Utilities.CreateTaskStateMonitor().WaitAll(ourTasks, TaskState.Completed, TimeSpan.FromMinutes(20));
Console.WriteLine("tasks are done.");
Console.WriteLine();
Console.WriteLine("See below for Stdout / Stderr for each node.");
Console.WriteLine("============================================");
/* Display stdout/stderr for each task on completion
* =================================================
*/
foreach (CloudTask t in ourTasks)
{
Console.WriteLine("Task " + t.Id + ":");
Console.WriteLine(" stdout:" + Environment.NewLine + t.GetNodeFile("stdout.txt").ReadAsString());
Console.WriteLine();
Console.WriteLine(" stderr:" + Environment.NewLine + t.GetNodeFile("stderr.txt").ReadAsString());
}
Console.WriteLine();
Console.WriteLine("Please find the resulting images in storage. \nPress Enter to continue.");
Console.WriteLine("=======================================================================");
Console.ReadLine();
}
finally
{
/* If configured as such, Delete the resources that were used in this process
* ==========================================================================
*/
//Delete the pool that we created
if (imageBlurSettings.DeletePool)
{
Console.WriteLine("Deleting Pool. \nPress Enter to continue.");
Console.ReadLine();
Console.WriteLine("Deleting pool: {0}", imageBlurSettings.PoolId);
client.PoolOperations.DeletePool(imageBlurSettings.PoolId);
}
//Delete the job that we created
if (imageBlurSettings.DeleteJob)
{
Console.WriteLine("Deleting Job. \nPress Enter to continue.");
Console.ReadLine();
Console.WriteLine("Deleting job: {0}", imageBlurSettings.JobId);
client.JobOperations.DeleteJob(imageBlurSettings.JobId);
}
//Delete the containers we created
if (imageBlurSettings.DeleteContainer)
{
Console.WriteLine("Deleting Container. \nPress Enter to continue.");
Console.ReadLine();
DeleteContainers(accountSettings, stagingContainer);
}
Console.WriteLine();
Console.WriteLine("Please check the Azure portal to make sure that all resources you want deleted are in fact deleted");
Console.WriteLine("==================================================================================================");
Console.WriteLine();
Console.WriteLine("Press Enter to exit the program");
Console.WriteLine("Exiting program...");
}
}
}
