public void TaskDependenciesWithNullPropertiesFromProtocol()
{
var taskDependencies = new TaskDependencies(new Protocol.Models.TaskDependencies());
Assert.Null(taskDependencies.TaskIds);
Assert.Null(taskDependencies.TaskIdRanges);
}
private void CreateBatchJob(string jobName, int id, string profileUrl, string containerUrl, string nsUrl, int daysDuration, string timeZone, bool uamAsBasal)
{
// Connect to Azure Batch
var credentials = new BatchSharedKeyCredentials(
ConfigurationManager.AppSettings["BatchAccountUrl"],
ConfigurationManager.AppSettings["BatchAccountName"],
ConfigurationManager.AppSettings["BatchAccountKey"]);
using (var batchClient = BatchClient.Open(credentials))
{
var poolId = ConfigurationManager.AppSettings["BatchPoolId"];
// Create the job
var job = batchClient.JobOperations.CreateJob();
job.Id = jobName;
job.PoolInformation = new PoolInformation {
PoolId = poolId
};
job.UsesTaskDependencies = true;
job.OnAllTasksComplete = OnAllTasksComplete.TerminateJob;
job.Commit();
// Add a task to the job to run Autotune
var commandLine = "/bin/sh -c '" +
"cd \"$AZ_BATCH_TASK_WORKING_DIR\" && " +
"mkdir -p settings && " +
"mv profile.json settings && " +
"cp settings/profile.json settings/pumpprofile.json && " +
"cp settings/profile.json settings/autotune.json && " +
$"TZ='{timeZone}' && " +
"export TZ && " +
$"oref0-autotune --dir=$AZ_BATCH_TASK_WORKING_DIR --ns-host={nsUrl} --start-date={DateTime.Now.AddDays(-daysDuration):yyyy-MM-dd} --end-date={DateTime.Now.AddDays(-1):yyyy-MM-dd} --categorize-uam-as-basal={(uamAsBasal ? "true" : "false")}" +
"'";
var task = new CloudTask("Autotune", commandLine);
// The task needs to use the profile.json file previously added to Azure Storage
task.ResourceFiles = new List <ResourceFile>();
task.ResourceFiles.Add(ResourceFile.FromUrl(profileUrl, "profile.json"));
// Capture the recommendations generated by Autotune into Azure Storage
task.OutputFiles = new List <OutputFile>();
task.OutputFiles.Add(new OutputFile(
filePattern: "autotune/autotune_recommendations.log",
destination: new OutputFileDestination(new OutputFileBlobContainerDestination(containerUrl)),
uploadOptions: new OutputFileUploadOptions(OutputFileUploadCondition.TaskCompletion)));
task.OutputFiles.Add(new OutputFile(
filePattern: "autotune/autotune.*.log",
destination: new OutputFileDestination(new OutputFileBlobContainerDestination(containerUrl)),
uploadOptions: new OutputFileUploadOptions(OutputFileUploadCondition.TaskCompletion)));
batchClient.JobOperations.AddTask(jobName, task);
// Get the URL for the JobFinished action that the recommendations can be uploaded to to generate the email
var uploadUrl = new Uri(Request.Url, Url.Action("JobFinished", "Results", new { id, key = ConfigurationManager.AppSettings["ResultsCallbackKey"] }));
var uploadCommandLine = $"wget {uploadUrl}";
var uploadTask = new CloudTask("Upload", uploadCommandLine);
uploadTask.DependsOn = TaskDependencies.OnId(task.Id);
batchClient.JobOperations.AddTask(jobName, uploadTask);
}
}
private CloudTask CreateReducerTask(string inputContainerSas, string outputContainerSas, IEnumerable <CloudTask> mapperTasks)
{
CloudTask unboundReducerTask = new CloudTask(Constants.ReducerTaskId, Constants.ReducerTaskExecutable);
//The set of files (exes, dlls and configuration files) required to run the reducer task.
List <ResourceFile> reducerTaskResourceFiles = SampleHelpers.GetResourceFiles(inputContainerSas, Constants.RequiredExecutableFiles);
//The mapper outputs to reduce
var mapperOutputs = Enumerable.Range(0, this.textSearchSettings.NumberOfMapperTasks).Select(Helpers.GetMapperTaskId);
reducerTaskResourceFiles.AddRange(SampleHelpers.GetResourceFiles(outputContainerSas, mapperOutputs));
unboundReducerTask.ResourceFiles = reducerTaskResourceFiles;
// Upload the reducer task stdout as the result file for the entire job
unboundReducerTask.OutputFiles = new List <OutputFile>
{
new OutputFile(
filePattern: "..\\stdout.txt",
destination: new OutputFileDestination(
container: new OutputFileBlobContainerDestination(outputContainerSas, path: Constants.ReducerTaskResultBlobName)),
uploadOptions: new OutputFileUploadOptions(uploadCondition: OutputFileUploadCondition.TaskSuccess))
};
// Depend on the mapper tasks so that they are all complete before the reducer runs
unboundReducerTask.DependsOn = TaskDependencies.OnTasks(mapperTasks);
return(unboundReducerTask);
}
/// <summary>
/// Create a task to submit to the Batch instance
/// </summary>
/// <param name="jobId">Desired job id</param>
/// <param name="imageName">Desired docker image name</param>
/// <param name="batchClient">The batchclient communicating with the batch instance</param>
/// <param name="taskEnvironmentSettings">Task's settings</param>
/// <param name="taskName">The name of the task</param>
/// <param name="taskDependencies">Dependencies the task has to wait upon</param>
private static void CreateTask(string jobId, string imageName, BatchClient batchClient, List <EnvironmentSetting> taskEnvironmentSettings,
string taskName, List <string> taskDependencies)
{
// Assign the job preparation task to the job
TaskContainerSettings workerContainer = new TaskContainerSettings(
imageName
);
CloudTask workerTask = new CloudTask(taskName, "echo here")
{
ContainerSettings = workerContainer,
Id = jobId,
EnvironmentSettings = taskEnvironmentSettings,
UserIdentity = new UserIdentity(new AutoUserSpecification(elevationLevel: ElevationLevel.Admin, scope: AutoUserScope.Task)),
DependsOn = TaskDependencies.OnIds(taskDependencies)
};
// Commit Job to create it in the service
batchClient.JobOperations.AddTask(jobId, workerTask);
}
private void CreateBatchJob(string jobName, int id, string profileUrl, string containerUrl, string nsUrl, int daysDuration, string timeZone, bool uamAsBasal)
{
// Connect to Azure Batch
var credentials = new BatchSharedKeyCredentials(
Startup.Configuration["BatchAccountUrl"],
Startup.Configuration["BatchAccountName"],
Startup.Configuration["BatchAccountKey"]);
using (var batchClient = BatchClient.Open(credentials))
{
var poolId = Startup.Configuration["BatchPoolId"];
// Create the job
var job = batchClient.JobOperations.CreateJob();
job.Id = jobName;
job.PoolInformation = new PoolInformation {
PoolId = poolId
};
job.UsesTaskDependencies = true;
job.OnAllTasksComplete = OnAllTasksComplete.TerminateJob;
job.Commit();
// Add a task to the job to run Autotune
var commandLine = "/bin/sh -c '" +
"cd \"$AZ_BATCH_TASK_WORKING_DIR\" && " +
"mkdir -p settings && " +
"mv profile.json settings && " +
"cp settings/profile.json settings/pumpprofile.json && " +
"cp settings/profile.json settings/autotune.json && " +
$"TZ='{timeZone}' && " +
"export TZ && " +
$"oref0-autotune --dir=$AZ_BATCH_TASK_WORKING_DIR --ns-host={nsUrl} --start-date={DateTime.Now.AddDays(-daysDuration):yyyy-MM-dd} --end-date={DateTime.Now.AddDays(-1):yyyy-MM-dd} --categorize-uam-as-basal={(uamAsBasal ? "true" : "false")}" +
"'";
var task = new CloudTask("Autotune", commandLine);
// The task needs to use the profile.json file previously added to Azure Storage
task.ResourceFiles = new List <ResourceFile>();
task.ResourceFiles.Add(ResourceFile.FromUrl(profileUrl, "profile.json"));
// Capture the recommendations generated by Autotune into Azure Storage
task.OutputFiles = new List <OutputFile>();
task.OutputFiles.Add(new OutputFile(
filePattern: "autotune/autotune_recommendations.log",
destination: new OutputFileDestination(new OutputFileBlobContainerDestination(containerUrl)),
uploadOptions: new OutputFileUploadOptions(OutputFileUploadCondition.TaskCompletion)));
task.OutputFiles.Add(new OutputFile(
filePattern: "autotune/autotune.*.log",
destination: new OutputFileDestination(new OutputFileBlobContainerDestination(containerUrl)),
uploadOptions: new OutputFileUploadOptions(OutputFileUploadCondition.TaskCompletion)));
// Ensure the task still completes even if Autotune fails, as we still want to email the
// log file with the error details
task.ExitConditions = new ExitConditions
{
ExitCodes = new List <ExitCodeMapping>(new[]
{
new ExitCodeMapping(1, new ExitOptions {
DependencyAction = DependencyAction.Satisfy
})
})
};
batchClient.JobOperations.AddTask(jobName, task);
// Get the URL for the JobFinished action that the recommendations can be uploaded to to generate the email
var uploadUrl = Url.Action("JobFinished", "Results", new { id, key = Startup.Configuration["ResultsCallbackKey"] }, Request.Scheme);
var uploadCommandLine = $"/bin/sh -c '" +
"cd /usr/src/oref0 && " +
$"wget -O /dev/null -o /dev/null {uploadUrl}\\&commit=$(git rev-parse --short HEAD)" +
"'";
var uploadTask = new CloudTask("Upload", uploadCommandLine);
uploadTask.DependsOn = TaskDependencies.OnId(task.Id);
uploadTask.Constraints = new TaskConstraints(maxTaskRetryCount: 2);
batchClient.JobOperations.AddTask(jobName, uploadTask);
}
}
private static async Task MainAsync(string[] args)
{
// You may adjust these values to experiment with different compute resource scenarios.
const string nodeSize = "standard_d1_v2";
const string osFamily = "5";
const int nodeCount = 1;
const string poolId = "TaskDependenciesSamplePool";
const string jobId = "TaskDependenciesSampleJob";
// Amount of time to wait before timing out long-running tasks.
TimeSpan timeLimit = TimeSpan.FromMinutes(30);
// Set up access to your Batch account with a BatchClient. Configure your AccountSettings in the
// Microsoft.Azure.Batch.Samples.Common project within this solution.
AccountSettings accountSettings = SampleHelpers.LoadAccountSettings();
BatchSharedKeyCredentials cred = new BatchSharedKeyCredentials(
accountSettings.BatchServiceUrl,
accountSettings.BatchAccountName,
accountSettings.BatchAccountKey);
try
{
using (BatchClient batchClient = BatchClient.Open(cred))
{
// Create the pool.
Console.WriteLine("Creating pool [{0}]...", poolId);
CloudPool unboundPool =
batchClient.PoolOperations.CreatePool(
poolId: poolId,
cloudServiceConfiguration: new CloudServiceConfiguration(osFamily),
virtualMachineSize: nodeSize,
targetDedicatedComputeNodes: nodeCount);
await unboundPool.CommitAsync();
// Create the job and specify that it uses tasks dependencies.
Console.WriteLine("Creating job [{0}]...", jobId);
CloudJob unboundJob = batchClient.JobOperations.CreateJob(jobId,
new PoolInformation {
PoolId = poolId
});
// IMPORTANT: This is REQUIRED for using task dependencies.
unboundJob.UsesTaskDependencies = true;
await unboundJob.CommitAsync();
// Create the collection of tasks that will be added to the job.
List <CloudTask> tasks = new List <CloudTask>
{
// 'Rain' and 'Sun' don't depend on any other tasks
new CloudTask("Rain", "cmd.exe /c echo Rain"),
new CloudTask("Sun", "cmd.exe /c echo Sun"),
// Task 'Flowers' depends on completion of both 'Rain' and 'Sun'
// before it is run.
new CloudTask("Flowers", "cmd.exe /c echo Flowers")
{
DependsOn = TaskDependencies.OnIds("Rain", "Sun")
},
// Tasks 1, 2, and 3 don't depend on any other tasks. Because
// we will be using them for a task range dependency, we must
// specify string representations of integers as their ids.
new CloudTask("1", "cmd.exe /c echo 1"),
new CloudTask("2", "cmd.exe /c echo 2"),
new CloudTask("3", "cmd.exe /c echo 3"),
// Task A is the parent task.
new CloudTask("A", "cmd.exe /c echo A")
{
// Specify exit conditions for task A and their dependency actions.
ExitConditions = new ExitConditions
{
// If task A exits with a pre-processing error, block any downstream tasks (in this example, task B).
PreProcessingError = new ExitOptions
{
DependencyAction = DependencyAction.Block
},
// If task A exits with the specified error codes, block any downstream tasks (in this example, task B).
ExitCodes = new List <ExitCodeMapping>
{
new ExitCodeMapping(10, new ExitOptions()
{
DependencyAction = DependencyAction.Block
}),
new ExitCodeMapping(20, new ExitOptions()
{
DependencyAction = DependencyAction.Block
})
},
// If task A succeeds or fails with any other error, any downstream tasks become eligible to run
// (in this example, task B).
Default = new ExitOptions
{
DependencyAction = DependencyAction.Satisfy
}
}
},
// Task B depends on task A. Whether it becomes eligible to run depends on how task A exits.
new CloudTask("B", "cmd.exe /c echo B")
{
DependsOn = TaskDependencies.OnId("A")
},
};
// Add the tasks to the job.
await batchClient.JobOperations.AddTaskAsync(jobId, tasks);
// Pause execution while we wait for the tasks to complete, and notify
// whether the tasks completed successfully.
Console.WriteLine("Waiting for task completion...");
Console.WriteLine();
CloudJob job = await batchClient.JobOperations.GetJobAsync(jobId);
await batchClient.Utilities.CreateTaskStateMonitor().WhenAll(
job.ListTasks(),
TaskState.Completed,
timeLimit);
Console.WriteLine("All tasks completed successfully.");
Console.WriteLine();
}
}
catch (Exception e)
{
Console.WriteLine();
Console.WriteLine("An exception occurred.");
Console.WriteLine(e.Message);
Console.WriteLine(e.StackTrace);
}
finally
{
using (BatchClient batchClient = BatchClient.Open(cred))
{
CloudJob job = await batchClient.JobOperations.GetJobAsync(jobId);
// Clean up the resources we've created in the Batch account
Console.Write("Delete job? [yes] no: ");
string response = Console.ReadLine().ToLower();
if (response != "n" && response != "no")
{
await batchClient.JobOperations.DeleteJobAsync(job.Id);
}
Console.Write("Delete pool? [yes] no: ");
response = Console.ReadLine().ToLower();
if (response != "n" && response != "no")
{
await batchClient.PoolOperations.DeletePoolAsync(poolId);
}
}
}
}
private static async Task MainAsync(string[] args)
{
// You may adjust these values to experiment with different compute resource scenarios.
const string nodeSize = "small";
const string osFamily = "4";
const int nodeCount = 1;
const string poolId = "TaskDependenciesSamplePool";
const string jobId = "TaskDependenciesSampleJob";
// Amount of time to wait before timing out long-running tasks.
TimeSpan timeLimit = TimeSpan.FromMinutes(30);
// Set up access to your Batch account with a BatchClient. Configure your AccountSettings in the
// Microsoft.Azure.Batch.Samples.Common project within this solution.
BatchSharedKeyCredentials cred = new BatchSharedKeyCredentials(AccountSettings.Default.BatchServiceUrl,
AccountSettings.Default.BatchAccountName,
AccountSettings.Default.BatchAccountKey);
using (BatchClient batchClient = await BatchClient.OpenAsync(cred))
{
// Create the pool.
Console.WriteLine("Creating pool [{0}]...", poolId);
CloudPool unboundPool =
batchClient.PoolOperations.CreatePool(poolId: poolId,
cloudServiceConfiguration: new CloudServiceConfiguration(osFamily),
virtualMachineSize: nodeSize,
targetDedicated: nodeCount);
await unboundPool.CommitAsync();
// Create the job and specify that it uses tasks dependencies.
Console.WriteLine("Creating job [{0}]...", jobId);
CloudJob unboundJob = batchClient.JobOperations.CreateJob(jobId,
new PoolInformation {
PoolId = poolId
});
// IMPORTANT: This is REQUIRED for using task dependencies.
unboundJob.UsesTaskDependencies = true;
await unboundJob.CommitAsync();
// Create the collection of tasks that will be added to the job.
List <CloudTask> tasks = new List <CloudTask>
{
// 'Rain' and 'Sun' don't depend on any other tasks
new CloudTask("Rain", "cmd.exe /c echo Rain"),
new CloudTask("Sun", "cmd.exe /c echo Sun"),
// Task 'Flowers' depends on completion of both 'Rain' and 'Sun'
// before it is run.
new CloudTask("Flowers", "cmd.exe /c echo Flowers")
{
DependsOn = TaskDependencies.OnIds("Rain", "Sun")
},
// Tasks 1, 2, and 3 don't depend on any other tasks. Because
// we will be using them for a task range dependency, we must
// specify string representations of integers as their ids.
new CloudTask("1", "cmd.exe /c echo 1"),
new CloudTask("2", "cmd.exe /c echo 2"),
new CloudTask("3", "cmd.exe /c echo 3"),
// Task 4 depends on a range of tasks, 1 through 3
new CloudTask("4", "cmd.exe /c echo 4")
{
// To use a range of tasks, their ids must be integer values.
// Note that we pass integers as parameters to TaskIdRange,
// but their ids (above) are string representations of the ids.
DependsOn = TaskDependencies.OnIdRange(1, 3)
},
// Task 5 depends on a range of tasks, 1 through 3, and 'Flowers'
new CloudTask("5", "cmd.exe /c echo 5")
{
DependsOn = new TaskDependencies(
new[] { "Flowers" },
new[] { new TaskIdRange(1, 3) })
},
};
// Add the tasks to the job.
await batchClient.JobOperations.AddTaskAsync(jobId, tasks);
// Pause execution while we wait for the tasks to complete, and notify
// whether the tasks completed successfully.
Console.WriteLine("Waiting for task completion...");
Console.WriteLine();
CloudJob job = await batchClient.JobOperations.GetJobAsync(jobId);
try
{
await batchClient.Utilities.CreateTaskStateMonitor().WhenAll(
job.ListTasks(),
TaskState.Completed,
timeLimit);
Console.WriteLine("All tasks completed successfully.");
Console.WriteLine();
}
catch (TimeoutException e)
{
Console.WriteLine(e);
}
// Clean up the resources we've created in the Batch account
Console.Write("Delete job? [yes] no: ");
string response = Console.ReadLine().ToLower();
if (response != "n" && response != "no")
{
await batchClient.JobOperations.DeleteJobAsync(job.Id);
}
Console.Write("Delete pool? [yes] no: ");
response = Console.ReadLine().ToLower();
if (response != "n" && response != "no")
{
await batchClient.PoolOperations.DeletePoolAsync(poolId);
}
}
}
static void Main(string[] args)
{
if (String.IsNullOrEmpty(BatchAccountName) ||
String.IsNullOrEmpty(BatchAccountKey) ||
String.IsNullOrEmpty(BatchAccountUrl) ||
String.IsNullOrEmpty(StorageAccountName) ||
String.IsNullOrEmpty(StorageAccountKey))
{
throw new InvalidOperationException("One or more account credential strings have not been populated. Please ensure that your Batch and Storage account credentials have been specified.");
}
try
{
Console.WriteLine("Sample start: {0}", DateTime.Now);
Console.WriteLine();
Stopwatch timer = new Stopwatch();
timer.Start();
// Create the blob client, for use in obtaining references to blob storage containers
(CloudBlobClient, CloudStorageAccount)cc = CreateCloudBlobClient(StorageAccountName, StorageAccountKey);
CloudBlobClient blobClient = cc.Item1;
CloudStorageAccount linkedStorageAccount = cc.Item2;
// Use the blob client to create the input container in Azure Storage
const string inputContainerName = "input";
CloudBlobContainer container = blobClient.GetContainerReference(inputContainerName);
container.CreateIfNotExistsAsync().Wait();
// The collection of data files that are to be processed by the tasks
List <string> inputFilePaths = new List <string> {
"input.txt"
};
// Upload the data files to Azure Storage. This is the data that will be processed by each of the tasks that are
// executed on the compute nodes within the pool.
UploadFileToContainer(blobClient, inputContainerName, "input.txt");
// Get a Batch client using account creds
BatchSharedKeyCredentials cred = new BatchSharedKeyCredentials(BatchAccountUrl, BatchAccountName, BatchAccountKey);
using (BatchClient batchClient = BatchClient.Open(cred))
{
Console.WriteLine("Creating pool [{0}]...", PoolId);
// Create a Windows Server image, VM configuration, Batch pool
ImageReference imageReference = CreateImageReference();
VirtualMachineConfiguration vmConfiguration = CreateVirtualMachineConfiguration(imageReference);
CreateBatchPool(batchClient, vmConfiguration);
// Create a Batch job
Console.WriteLine("Creating job [{0}]...", JobId);
try
{
CloudJob job = batchClient.JobOperations.CreateJob();
job.Id = JobId;
job.PoolInformation = new PoolInformation {
PoolId = PoolId
};
job.UsesTaskDependencies = true;
// Create the blob storage container for the outputs.
Task t1 = job.PrepareOutputStorageAsync(linkedStorageAccount);
Task.WaitAll(t1);
string containerName = job.OutputStorageContainerName();
string containerUrl = job.GetOutputStorageContainerUrl(linkedStorageAccount);
job.CommonEnvironmentSettings = new[] { new EnvironmentSetting("JOB_CONTAINER_URL", containerUrl) };
job.Commit();
}
catch (BatchException be)
{
// Accept the specific error code JobExists as that is expected if the job already exists
if (be.RequestInformation?.BatchError?.Code == BatchErrorCodeStrings.JobExists)
{
Console.WriteLine("The job {0} already existed when we tried to create it", JobId);
}
else
{
throw; // Any other exception is unexpected
}
}
// Create a collection to hold the tasks that we'll be adding to the job
List <CloudTask> tasks = new List <CloudTask>();
// Create each of the tasks to process one of the input files.
string lastTaskId = null;
for (int i = 0; i < 3; i++)
{
string taskId = String.Format("Task{0}", i);
CloudTask task;
// first task
if (lastTaskId == null)
{
//task = new CloudTask(taskId, $"cmd /c %AZ_BATCH_APP_PACKAGE_TESTBATCHAPP%\\testbatchapp.exe {i}")
task = new CloudTask(taskId, $"cmd /c %AZ_BATCH_APP_PACKAGE_HPCAPP%\\testbatchapp.exe {i}")
{
ApplicationPackageReferences = new List <ApplicationPackageReference>
{
new ApplicationPackageReference
{
//ApplicationId = "testbatchapp",
ApplicationId = "hpcapp" //,
//Version = "1.1"
}
}
};
}
else
{
// task = new CloudTask(taskId, $"cmd /c %AZ_BATCH_APP_PACKAGE_TESTBATCHAPP%\\testbatchapp.exe {i}")
task = new CloudTask(taskId, $"cmd /c %AZ_BATCH_APP_PACKAGE_HPCAPP%\\testbatchapp.exe {i}")
{
ApplicationPackageReferences = new List <ApplicationPackageReference>
{
new ApplicationPackageReference
{
// ApplicationId = "testbatchapp",
ApplicationId = "hpcapp" //,
//Version = "1.1"
}
}
};
task.DependsOn = TaskDependencies.OnId(lastTaskId);
}
lastTaskId = taskId;
tasks.Add(task);
}
// Add all tasks to the job.
batchClient.JobOperations.AddTask(JobId, tasks);
// Monitor task success/failure, specifying a maximum amount of time to wait for the tasks to complete.
TimeSpan timeout = TimeSpan.FromMinutes(30);
Console.WriteLine("Monitoring all tasks for 'Completed' state, timeout in {0}...", timeout);
IEnumerable <CloudTask> addedTasks = batchClient.JobOperations.ListTasks(JobId);
batchClient.Utilities.CreateTaskStateMonitor().WaitAll(addedTasks, TaskState.Completed, timeout);
Console.WriteLine("All tasks reached state Completed.");
// Print task output
Console.WriteLine();
Console.WriteLine("Printing task output...");
IEnumerable <CloudTask> completedtasks = batchClient.JobOperations.ListTasks(JobId);
foreach (CloudTask task in completedtasks)
{
List <Task> readTasks = new List <Task>();
foreach (OutputFileReference output in task.OutputStorage(linkedStorageAccount).ListOutputs(TaskOutputKind.TaskOutput))
{
// Console.WriteLine($"output file: {output.FilePath}");
readTasks.Add(output.DownloadToFileAsync($"{JobId}-{output.FilePath}", System.IO.FileMode.Create));
}
Task.WaitAll(readTasks.ToArray());
}
// Print out some timing info
timer.Stop();
Console.WriteLine();
Console.WriteLine("Sample end: {0}", DateTime.Now);
Console.WriteLine("Elapsed time: {0}", timer.Elapsed);
// Clean up Storage resources
container.DeleteIfExistsAsync().Wait();
Console.WriteLine("Container [{0}] deleted.", inputContainerName);
// Clean up Batch resources (if the user so chooses)
Console.WriteLine();
Console.Write("Delete job? [yes] no: ");
string response = Console.ReadLine().ToLower();
if (response != "n" && response != "no")
{
batchClient.JobOperations.DeleteJob(JobId);
}
Console.Write("Delete pool? [yes] no: ");
response = Console.ReadLine().ToLower();
if (response != "n" && response != "no")
{
batchClient.PoolOperations.DeletePool(PoolId);
}
}
}
finally
{
Console.WriteLine();
Console.WriteLine("Sample complete, hit ENTER to exit...");
Console.ReadLine();
}
}
/// <summary>
/// Creates tasks to process each of the specified input files, and submits them to the
/// specified job for execution.
/// </summary>
/// <param name="batchClient">A <see cref="BatchClient"/>.</param>
/// <param name="jobId">The id of the job to which the tasks should be added.</param>
/// <param name="inputFiles">A collection of <see cref="ResourceFile"/> objects representing the input files to be
/// processed by the tasks executed on the compute nodes.</param>
/// <param name="outputContainerSasUrl">The shared access signature URL for the container within Azure Storage that
/// will receive the output files created by the tasks.</param>
/// <returns>A collection of the submitted tasks.</returns>
private static async Task <List <CloudTask> > AddTasksAsync(BatchClient batchClient, string jobId, List <ResourceFile> inputFiles, string outputContainerSasUrl, CloudStorageAccount linkedStorageAccount)
{
Console.WriteLine("Adding {0} tasks to job [{1}]...", inputFiles.Count, jobId);
// Create a collection to hold the tasks that we'll be adding to the job
List <CloudTask> tasks = new List <CloudTask>();
// Create each of the tasks. Because we copied the task application to the
// node's shared directory with the pool's StartTask, we can access it via
// the shared directory on whichever node each task will run.
foreach (ResourceFile inputFile in inputFiles)
{
string taskId = $"task-mapper-{inputFiles.IndexOf(inputFile)}";
string mapperTaskCommandLine = $"{BatchJobTaskMapperCommandline} {inputFile.FilePath}";
CloudTask mapper = new CloudTask(taskId, mapperTaskCommandLine)
{
OutputFiles = new List <OutputFile>
{
new OutputFile(filePattern: TaskMapOutputFilePattern,
destination: new OutputFileDestination(new OutputFileBlobContainerDestination(containerUrl: outputContainerSasUrl, path: $"{taskId}.txt")),
uploadOptions: new OutputFileUploadOptions(
uploadCondition: OutputFileUploadCondition.TaskCompletion))
},
ResourceFiles = new List <ResourceFile> {
inputFile
}
};
tasks.Add(mapper);
}
string reducerTaskId = "task-reducer";
string reducerTaskCommandLine = BatchJobTaskReducerCommandline;
CloudTask reducer = new CloudTask(reducerTaskId, reducerTaskCommandLine)
{
DependsOn = TaskDependencies.OnTasks(tasks),
OutputFiles = new List <OutputFile>
{
new OutputFile(filePattern: TaskReduceOutputFilePattern,
destination: new OutputFileDestination(new OutputFileBlobContainerDestination(containerUrl: outputContainerSasUrl, path: reducerTaskId)),
uploadOptions: new OutputFileUploadOptions(
uploadCondition: OutputFileUploadCondition.TaskCompletion))
}
};
var reducerInputFiles = new List <ResourceFile>();
// define the input files for the reducer task
foreach (var maptask in tasks)
{
foreach (var outFile in maptask.OutputFiles)
{
// ensure that the container SAS has the following rights: write|read|list - otherwise we can't download the files
var sasContainerUrl = outFile.Destination.Container.ContainerUrl;
var blobName = outFile.Destination.Container.Path;
var sasBlob = sasContainerUrl.Insert(sasContainerUrl.IndexOf('?'), $"/{blobName}");
reducerInputFiles.Add(new ResourceFile(sasBlob, blobName));
}
}
reducer.ResourceFiles = reducerInputFiles;
tasks.Add(reducer);
// Add the tasks as a collection opposed to a separate AddTask call for each. Bulk task submission
// helps to ensure efficient underlying API calls to the Batch service.
await batchClient.JobOperations.AddTaskAsync(jobId, tasks);
return(tasks);
}
public void ProvideDependencies(TaskDependencies deps)
{
_taskDependencies = deps;
}
private static async Task MainAsync()
{
const int nodeCount = 1;
string poolId = "TaskDependenciesSamplePool";
string jobId = "TaskDependenciesJob";
var settings = Config.LoadAccountSettings();
BatchSharedKeyCredentials cred = new BatchSharedKeyCredentials(settings.BatchServiceUrl, settings.BatchAccountName, settings.BatchAccountKey);
using (BatchClient batchClient = BatchClient.Open(cred))
{
var pool = await BatchUtils.CreatePoolIfNotExistAsync(batchClient, poolId, lowPriorityNodes : nodeCount);
var job = await BatchUtils.CreateJobIfNotExistAsync(batchClient, pool.Id, jobId, usesTaskDependencies : true);
string taskOutputFile = "$AZ_BATCH_NODE_SHARED_DIR/task_output.txt";
// Create the collection of tasks that will be added to the job.
List <CloudTask> tasks = new List <CloudTask>
{
// 'Rain' and 'Sun' don't depend on any other tasks
new CloudTask("Rain", $"/bin/bash -c \"echo Rain >> {taskOutputFile}\""),
new CloudTask("Sun", $"/bin/bash -c \"echo Sun >> {taskOutputFile}\""),
// Task 'Flowers' depends on completion of both 'Rain' and 'Sun'
// before it is run.
new CloudTask("Flowers", $"/bin/bash -c \"echo Flowers >> {taskOutputFile}\"")
{
DependsOn = TaskDependencies.OnIds("Rain", "Sun")
},
// Tasks 1, 2, and 3 don't depend on any other tasks. Because
// we will be using them for a task range dependency, we must
// specify string representations of integers as their ids.
new CloudTask("1", $"/bin/bash -c \"echo 1 >> {taskOutputFile}\""),
new CloudTask("2", $"/bin/bash -c \"echo 2 >> {taskOutputFile}\""),
new CloudTask("3", $"/bin/bash -c \"echo 3 >> {taskOutputFile}\""),
// Task dependency on ID range
new CloudTask("Final", $"/bin/bash -c \"echo Final >> {taskOutputFile}\"")
{
DependsOn = TaskDependencies.OnIdRange(1, 3)
},
// Task A is the parent task.
new CloudTask("A", $"/bin/bash -c \"echo A >> {taskOutputFile}\"")
{
// Specify exit conditions for task A and their dependency actions.
ExitConditions = new ExitConditions
{
// If task A exits with a pre-processing error, block any downstream tasks (in this example, task B).
PreProcessingError = new ExitOptions
{
DependencyAction = DependencyAction.Block
},
// If task A exits with the specified error codes, block any downstream tasks (in this example, task B).
ExitCodes = new List <ExitCodeMapping>
{
new ExitCodeMapping(10, new ExitOptions()
{
DependencyAction = DependencyAction.Block
}),
new ExitCodeMapping(20, new ExitOptions()
{
DependencyAction = DependencyAction.Block
})
},
// If task A succeeds or fails with any other error, any downstream tasks become eligible to run
// (in this example, task B).
Default = new ExitOptions
{
DependencyAction = DependencyAction.Satisfy
},
}
},
// Task B depends on task A. Whether it becomes eligible to run depends on how task A exits.
new CloudTask("B", $"/bin/bash -c \"echo B >> {taskOutputFile}\"")
{
DependsOn = TaskDependencies.OnId("A")
},
};
// Add the tasks in one API call as opposed to a separate AddTask call for each. Bulk task
// submission helps to ensure efficient underlying API calls to the Batch service.
Console.WriteLine("Submitting tasks and awaiting completion...");
await batchClient.JobOperations.AddTaskAsync(job.Id, tasks);
// Wait for the tasks to complete before proceeding. The long timeout here is to allow time
// for the nodes within the pool to be created and started if the pool had not yet been created.
await batchClient.Utilities.CreateTaskStateMonitor().WhenAll(
job.ListTasks(),
TaskState.Completed,
TimeSpan.FromMinutes(30));
Console.WriteLine("All tasks completed.");
Console.WriteLine();
// Print the contents of the shared text file modified by the job preparation and other tasks.
ODATADetailLevel nodeDetail = new ODATADetailLevel(selectClause: "id, state");
IPagedEnumerable <ComputeNode> nodes = batchClient.PoolOperations.ListComputeNodes(poolId, nodeDetail);
await nodes.ForEachAsync(async (node) =>
{
// Check to ensure that the node is Idle before attempting to pull the text file.
// If the pool was just created, there is a chance that another node completed all
// of the tasks prior to the other node(s) completing their startup procedure.
if (node.State == ComputeNodeState.Idle)
{
NodeFile sharedTextFile = await node.GetNodeFileAsync("shared/task_output.txt");
Console.WriteLine("Contents of {0} on {1}:", sharedTextFile.Path, node.Id);
Console.WriteLine("-------------------------------------------");
Console.WriteLine(await sharedTextFile.ReadAsStringAsync());
}
});
// Clean up the resources we've created in the Batch account
Console.WriteLine();
Console.WriteLine("Delete job? [yes] no");
string response = Console.ReadLine().ToLower();
if (response != "n" && response != "no")
{
// Note that deleting the job will execute the job release task if the job was not previously terminated
await batchClient.JobOperations.DeleteJobAsync(job.Id);
}
Console.WriteLine("Delete pool? [yes] no");
response = Console.ReadLine();
if (response != "n" && response != "no")
{
await batchClient.PoolOperations.DeletePoolAsync(poolId);
}
}
}