/// <summary>
/// Runs a series of tasks, using the OutputFiles feature in conjunction with the file conventions library to upload the tasks to a container.
/// Then downloads the files from the container to the local machine.
/// </summary>
public static async Task <CloudBlobContainer> RunWithConventions(
BatchClient batchClient,
CloudStorageAccount linkedStorageAccount,
string poolId,
int nodeCount,
string jobId)
{
await CreatePoolAsync(batchClient, poolId, nodeCount);
CloudJob job = batchClient.JobOperations.CreateJob(jobId, new PoolInformation {
PoolId = poolId
});
// Get the container URL to use
string containerName = job.OutputStorageContainerName();
CloudBlobContainer container = linkedStorageAccount.CreateCloudBlobClient().GetContainerReference(containerName);
await container.CreateIfNotExistsAsync();
string containerUrl = job.GetOutputStorageContainerUrl(linkedStorageAccount);
// Commit the job to the Batch service
await job.CommitAsync();
Console.WriteLine($"Created job {jobId}");
// Obtain the bound job from the Batch service
await job.RefreshAsync();
// Create a series of simple tasks which dump the task environment to a file and then write random values to a text file
IEnumerable <CloudTask> tasksToAdd = Enumerable.Range(1, 20).Select(i =>
{
var taskId = i.ToString().PadLeft(3, '0');
var task = new CloudTask(taskId, "cmd /v:ON /c \"echo off && set && (FOR /L %i IN (1,1,100000) DO (ECHO !RANDOM!)) > output.txt\"");
task.WithOutputFile(@"..\std*.txt", containerUrl, TaskOutputKind.TaskLog, OutputFileUploadCondition.TaskCompletion)
.WithOutputFile(@"output.txt", containerUrl, TaskOutputKind.TaskOutput, OutputFileUploadCondition.TaskSuccess);
return(task);
}
);
// Add the tasks to the job; the tasks are automatically
// scheduled for execution on the nodes by the Batch service.
await job.AddTaskAsync(tasksToAdd);
Console.WriteLine($"All tasks added to job {job.Id}");
Console.WriteLine();
Console.WriteLine($"Downloading outputs to {Directory.GetCurrentDirectory()}");
foreach (CloudTask task in job.CompletedTasks())
{
if (task.ExecutionInformation.Result != TaskExecutionResult.Success)
{
Console.WriteLine($"Task {task.Id} failed");
Console.WriteLine(SampleHelpers.GetFailureInfoDetails(task.ExecutionInformation.FailureInformation));
}
else
{
Console.WriteLine($"Task {task.Id} completed successfully");
}
foreach (OutputFileReference output in task.OutputStorage(linkedStorageAccount).ListOutputs(TaskOutputKind.TaskOutput))
{
Console.WriteLine($"output file: {output.FilePath}");
await output.DownloadToFileAsync($"{jobId}-{output.FilePath}", System.IO.FileMode.Create);
}
}
return(container);
}
/// <summary>
/// Runs a series of tasks, using the OutputFiles feature to upload the tasks files to a container.
/// Then downloads the files from the container to the local machine.
/// </summary>
public static async Task <CloudBlobContainer> Run(
BatchClient batchClient,
CloudStorageAccount storageAccount,
string poolId,
int nodeCount,
string jobId)
{
const string containerName = "outputfilescontainer";
await CreatePoolAsync(batchClient, poolId, nodeCount);
CloudJob job = batchClient.JobOperations.CreateJob(jobId, new PoolInformation {
PoolId = poolId
});
CloudBlobContainer container = storageAccount.CreateCloudBlobClient().GetContainerReference(containerName);
await container.CreateIfNotExistsAsync();
string containerSas = container.GetSharedAccessSignature(new SharedAccessBlobPolicy()
{
Permissions = SharedAccessBlobPermissions.Write,
SharedAccessExpiryTime = DateTimeOffset.UtcNow.AddDays(1)
});
string containerUrl = container.Uri.AbsoluteUri + containerSas;
// Commit the job to the Batch service
await job.CommitAsync();
Console.WriteLine($"Created job {jobId}");
// Obtain the bound job from the Batch service
await job.RefreshAsync();
// Create a series of simple tasks which dump the task environment to a file and then write random values to a text file
IEnumerable <CloudTask> tasksToAdd = Enumerable.Range(1, 20).Select(i =>
{
var taskId = i.ToString().PadLeft(3, '0');
return(new CloudTask(taskId, "cmd /v:ON /c \"echo off && set && (FOR /L %i IN (1,1,100000) DO (ECHO !RANDOM!)) > output.txt\"")
{
OutputFiles = new List <OutputFile>
{
new OutputFile(
filePattern: @"..\std*.txt",
destination: new OutputFileDestination(new OutputFileBlobContainerDestination(
containerUrl: containerUrl,
path: taskId)),
uploadOptions: new OutputFileUploadOptions(
uploadCondition: OutputFileUploadCondition.TaskCompletion)),
new OutputFile(
filePattern: @"output.txt",
destination: new OutputFileDestination(new OutputFileBlobContainerDestination(
containerUrl: containerUrl,
path: taskId + @"\output.txt")),
uploadOptions: new OutputFileUploadOptions(
uploadCondition: OutputFileUploadCondition.TaskCompletion)),
}
});
}
);
// Add the tasks to the job; the tasks are automatically
// scheduled for execution on the nodes by the Batch service.
await job.AddTaskAsync(tasksToAdd);
Console.WriteLine($"All tasks added to job {job.Id}");
Console.WriteLine();
Console.WriteLine($"Downloading outputs to {Directory.GetCurrentDirectory()}");
foreach (CloudTask task in job.CompletedTasks())
{
if (task.ExecutionInformation.Result != TaskExecutionResult.Success)
{
Console.WriteLine($"Task {task.Id} failed");
Console.WriteLine(SampleHelpers.GetFailureInfoDetails(task.ExecutionInformation.FailureInformation));
}
else
{
Console.WriteLine($"Task {task.Id} completed successfully");
}
CloudBlobDirectory directory = container.GetDirectoryReference(task.Id);
Directory.CreateDirectory(task.Id);
foreach (var blobInDirectory in directory.ListBlobs())
{
CloudBlockBlob blockBlob = blobInDirectory as CloudBlockBlob;
Console.WriteLine($" {blockBlob.Name}");
await blockBlob.DownloadToFileAsync(blockBlob.Name, FileMode.Create);
}
}
return(container);
}
public static async Task <CloudBlobContainer> Run(
BatchClient batchClient,
CloudStorageAccount linkedStorageAccount,
string poolId,
int nodeCount,
string jobId)
{
const string appPackageId = "PersistOutputsTask";
const string appPackageVersion = "1.0";
// Create and configure an unbound pool.
CloudPool pool = batchClient.PoolOperations.CreatePool(
poolId: poolId,
virtualMachineSize: "standard_d1_v2",
targetDedicatedComputeNodes: nodeCount,
cloudServiceConfiguration: new CloudServiceConfiguration(osFamily: "5"));
// Specify the application and version to deploy to the compute nodes. You must
// first build PersistOutputsTask, then upload it as an application package.
// See https://azure.microsoft.com/documentation/articles/batch-application-packages/
pool.ApplicationPackageReferences = new List <ApplicationPackageReference>
{
new ApplicationPackageReference
{
ApplicationId = appPackageId,
Version = appPackageVersion
}
};
// Commit the pool to the Batch service
await GettingStartedCommon.CreatePoolIfNotExistAsync(batchClient, pool);
CloudJob job = batchClient.JobOperations.CreateJob(jobId, new PoolInformation {
PoolId = poolId
});
// Create the blob storage container for the outputs.
await job.PrepareOutputStorageAsync(linkedStorageAccount);
// Create an environment variable on the compute nodes that the
// task application can reference when persisting its outputs.
string containerName = job.OutputStorageContainerName();
CloudBlobContainer container = linkedStorageAccount.CreateCloudBlobClient().GetContainerReference(containerName);
string containerUrl = job.GetOutputStorageContainerUrl(linkedStorageAccount);
job.CommonEnvironmentSettings = new[] { new EnvironmentSetting("JOB_CONTAINER_URL", containerUrl) };
// Commit the job to the Batch service
await job.CommitAsync();
Console.WriteLine($"Created job {jobId}");
// Obtain the bound job from the Batch service
await job.RefreshAsync();
IEnumerable <CloudTask> tasks = Enumerable.Range(1, 20).Select(i =>
new CloudTask(i.ToString().PadLeft(3, '0'), $"cmd /c %AZ_BATCH_APP_PACKAGE_{appPackageId.ToUpper()}#{appPackageVersion}%\\PersistOutputsTask.exe")
);
// Add the tasks to the job; the tasks are automatically
// scheduled for execution on the nodes by the Batch service.
await job.AddTaskAsync(tasks);
Console.WriteLine($"All tasks added to job {job.Id}");
Console.WriteLine();
Console.WriteLine($"Downloading outputs to {Directory.GetCurrentDirectory()}");
foreach (CloudTask task in job.CompletedTasks())
{
if (task.ExecutionInformation.Result != TaskExecutionResult.Success)
{
Console.WriteLine($"Task {task.Id} failed");
Console.WriteLine(SampleHelpers.GetFailureInfoDetails(task.ExecutionInformation.FailureInformation));
}
else
{
Console.WriteLine($"Task {task.Id} completed successfully");
}
foreach (OutputFileReference output in task.OutputStorage(linkedStorageAccount).ListOutputs(TaskOutputKind.TaskOutput))
{
Console.WriteLine($"output file: {output.FilePath}");
await output.DownloadToFileAsync($"{jobId}-{output.FilePath}", System.IO.FileMode.Create);
}
}
return(container);
}
