public void CannotGetOutputStorageUrlForNullJob()
{
CloudJob job = null;
CloudStorageAccount storageAccount = new CloudStorageAccount(new StorageCredentials("fake", "ZmFrZQ=="), true);
var ex = Assert.Throws <ArgumentNullException>(() => job.GetOutputStorageContainerUrl(storageAccount, TimeSpan.FromMinutes(5)));
Assert.Equal("job", ex.ParamName);
}
public void CannotGetOutputStorageUrlForNullStorageAccount()
{
using (var batchClient = BatchClient.Open(new FakeBatchServiceClient()))
{
CloudJob job = batchClient.JobOperations.CreateJob();
job.Id = "fakejob";
CloudStorageAccount storageAccount = null;
var ex = Assert.Throws <ArgumentNullException>(() => job.GetOutputStorageContainerUrl(storageAccount, TimeSpan.FromMinutes(5)));
Assert.Equal("storageAccount", ex.ParamName);
}
}
public void CannotGetOutputStorageUrlWithNegativeExpiryTime()
{
using (var batchClient = BatchClient.Open(new FakeBatchServiceClient()))
{
CloudJob job = batchClient.JobOperations.CreateJob();
job.Id = "fakejob";
CloudStorageAccount storageAccount = new CloudStorageAccount(new StorageCredentials("fake", "ZmFrZQ=="), true);
var ex = Assert.Throws <ArgumentException>(() => job.GetOutputStorageContainerUrl(storageAccount, TimeSpan.FromMinutes(-5)));
Assert.Equal("expiryTime", ex.ParamName);
}
}
public async Task CannotGetOutputStorageUrlWithZeroExpiryTime()
{
using (var batchClient = await BatchClient.OpenAsync(new FakeBatchServiceClient()))
{
CloudJob job = batchClient.JobOperations.CreateJob();
job.Id = "fakejob";
CloudStorageAccount storageAccount = new CloudStorageAccount(new StorageCredentials("fake", new byte[] { 65, 66, 67, 68 }), true);
var ex = Assert.Throws <ArgumentException>(() => job.GetOutputStorageContainerUrl(storageAccount, TimeSpan.FromMinutes(0)));
Assert.Equal("expiryTime", ex.ParamName);
}
}
/// <summary>
/// Runs a series of tasks, using the OutputFiles feature in conjunction with the file conventions library to upload the tasks to a container.
/// Then downloads the files from the container to the local machine.
/// </summary>
public static async Task <CloudBlobContainer> RunWithConventions(
BatchClient batchClient,
CloudStorageAccount linkedStorageAccount,
string poolId,
int nodeCount,
string jobId)
{
await CreatePoolAsync(batchClient, poolId, nodeCount);
CloudJob job = batchClient.JobOperations.CreateJob(jobId, new PoolInformation {
PoolId = poolId
});
// Get the container URL to use
string containerName = job.OutputStorageContainerName();
CloudBlobContainer container = linkedStorageAccount.CreateCloudBlobClient().GetContainerReference(containerName);
await container.CreateIfNotExistsAsync();
string containerUrl = job.GetOutputStorageContainerUrl(linkedStorageAccount);
// Commit the job to the Batch service
await job.CommitAsync();
Console.WriteLine($"Created job {jobId}");
// Obtain the bound job from the Batch service
await job.RefreshAsync();
// Create a series of simple tasks which dump the task environment to a file and then write random values to a text file
IEnumerable <CloudTask> tasksToAdd = Enumerable.Range(1, 20).Select(i =>
{
var taskId = i.ToString().PadLeft(3, '0');
var task = new CloudTask(taskId, "cmd /v:ON /c \"echo off && set && (FOR /L %i IN (1,1,100000) DO (ECHO !RANDOM!)) > output.txt\"");
task.WithOutputFile(@"..\std*.txt", containerUrl, TaskOutputKind.TaskLog, OutputFileUploadCondition.TaskCompletion)
.WithOutputFile(@"output.txt", containerUrl, TaskOutputKind.TaskOutput, OutputFileUploadCondition.TaskSuccess);
return(task);
}
);
// Add the tasks to the job; the tasks are automatically
// scheduled for execution on the nodes by the Batch service.
await job.AddTaskAsync(tasksToAdd);
Console.WriteLine($"All tasks added to job {job.Id}");
Console.WriteLine();
Console.WriteLine($"Downloading outputs to {Directory.GetCurrentDirectory()}");
foreach (CloudTask task in job.CompletedTasks())
{
if (task.ExecutionInformation.Result != TaskExecutionResult.Success)
{
Console.WriteLine($"Task {task.Id} failed");
Console.WriteLine(SampleHelpers.GetFailureInfoDetails(task.ExecutionInformation.FailureInformation));
}
else
{
Console.WriteLine($"Task {task.Id} completed successfully");
}
foreach (OutputFileReference output in task.OutputStorage(linkedStorageAccount).ListOutputs(TaskOutputKind.TaskOutput))
{
Console.WriteLine($"output file: {output.FilePath}");
await output.DownloadToFileAsync($"{jobId}-{output.FilePath}", System.IO.FileMode.Create);
}
}
return(container);
}
static void Main(string[] args)
{
if (String.IsNullOrEmpty(BatchAccountName) ||
String.IsNullOrEmpty(BatchAccountKey) ||
String.IsNullOrEmpty(BatchAccountUrl) ||
String.IsNullOrEmpty(StorageAccountName) ||
String.IsNullOrEmpty(StorageAccountKey))
{
throw new InvalidOperationException("One or more account credential strings have not been populated. Please ensure that your Batch and Storage account credentials have been specified.");
}
try
{
Console.WriteLine("Sample start: {0}", DateTime.Now);
Console.WriteLine();
Stopwatch timer = new Stopwatch();
timer.Start();
// Create the blob client, for use in obtaining references to blob storage containers
(CloudBlobClient, CloudStorageAccount)cc = CreateCloudBlobClient(StorageAccountName, StorageAccountKey);
CloudBlobClient blobClient = cc.Item1;
CloudStorageAccount linkedStorageAccount = cc.Item2;
// Use the blob client to create the input container in Azure Storage
const string inputContainerName = "input";
CloudBlobContainer container = blobClient.GetContainerReference(inputContainerName);
container.CreateIfNotExistsAsync().Wait();
// The collection of data files that are to be processed by the tasks
List <string> inputFilePaths = new List <string> {
"input.txt"
};
// Upload the data files to Azure Storage. This is the data that will be processed by each of the tasks that are
// executed on the compute nodes within the pool.
UploadFileToContainer(blobClient, inputContainerName, "input.txt");
// Get a Batch client using account creds
BatchSharedKeyCredentials cred = new BatchSharedKeyCredentials(BatchAccountUrl, BatchAccountName, BatchAccountKey);
using (BatchClient batchClient = BatchClient.Open(cred))
{
Console.WriteLine("Creating pool [{0}]...", PoolId);
// Create a Windows Server image, VM configuration, Batch pool
ImageReference imageReference = CreateImageReference();
VirtualMachineConfiguration vmConfiguration = CreateVirtualMachineConfiguration(imageReference);
CreateBatchPool(batchClient, vmConfiguration);
// Create a Batch job
Console.WriteLine("Creating job [{0}]...", JobId);
try
{
CloudJob job = batchClient.JobOperations.CreateJob();
job.Id = JobId;
job.PoolInformation = new PoolInformation {
PoolId = PoolId
};
job.UsesTaskDependencies = true;
// Create the blob storage container for the outputs.
Task t1 = job.PrepareOutputStorageAsync(linkedStorageAccount);
Task.WaitAll(t1);
string containerName = job.OutputStorageContainerName();
string containerUrl = job.GetOutputStorageContainerUrl(linkedStorageAccount);
job.CommonEnvironmentSettings = new[] { new EnvironmentSetting("JOB_CONTAINER_URL", containerUrl) };
job.Commit();
}
catch (BatchException be)
{
// Accept the specific error code JobExists as that is expected if the job already exists
if (be.RequestInformation?.BatchError?.Code == BatchErrorCodeStrings.JobExists)
{
Console.WriteLine("The job {0} already existed when we tried to create it", JobId);
}
else
{
throw; // Any other exception is unexpected
}
}
// Create a collection to hold the tasks that we'll be adding to the job
List <CloudTask> tasks = new List <CloudTask>();
// Create each of the tasks to process one of the input files.
string lastTaskId = null;
for (int i = 0; i < 3; i++)
{
string taskId = String.Format("Task{0}", i);
CloudTask task;
// first task
if (lastTaskId == null)
{
//task = new CloudTask(taskId, $"cmd /c %AZ_BATCH_APP_PACKAGE_TESTBATCHAPP%\\testbatchapp.exe {i}")
task = new CloudTask(taskId, $"cmd /c %AZ_BATCH_APP_PACKAGE_HPCAPP%\\testbatchapp.exe {i}")
{
ApplicationPackageReferences = new List <ApplicationPackageReference>
{
new ApplicationPackageReference
{
//ApplicationId = "testbatchapp",
ApplicationId = "hpcapp" //,
//Version = "1.1"
}
}
};
}
else
{
// task = new CloudTask(taskId, $"cmd /c %AZ_BATCH_APP_PACKAGE_TESTBATCHAPP%\\testbatchapp.exe {i}")
task = new CloudTask(taskId, $"cmd /c %AZ_BATCH_APP_PACKAGE_HPCAPP%\\testbatchapp.exe {i}")
{
ApplicationPackageReferences = new List <ApplicationPackageReference>
{
new ApplicationPackageReference
{
// ApplicationId = "testbatchapp",
ApplicationId = "hpcapp" //,
//Version = "1.1"
}
}
};
task.DependsOn = TaskDependencies.OnId(lastTaskId);
}
lastTaskId = taskId;
tasks.Add(task);
}
// Add all tasks to the job.
batchClient.JobOperations.AddTask(JobId, tasks);
// Monitor task success/failure, specifying a maximum amount of time to wait for the tasks to complete.
TimeSpan timeout = TimeSpan.FromMinutes(30);
Console.WriteLine("Monitoring all tasks for 'Completed' state, timeout in {0}...", timeout);
IEnumerable <CloudTask> addedTasks = batchClient.JobOperations.ListTasks(JobId);
batchClient.Utilities.CreateTaskStateMonitor().WaitAll(addedTasks, TaskState.Completed, timeout);
Console.WriteLine("All tasks reached state Completed.");
// Print task output
Console.WriteLine();
Console.WriteLine("Printing task output...");
IEnumerable <CloudTask> completedtasks = batchClient.JobOperations.ListTasks(JobId);
foreach (CloudTask task in completedtasks)
{
List <Task> readTasks = new List <Task>();
foreach (OutputFileReference output in task.OutputStorage(linkedStorageAccount).ListOutputs(TaskOutputKind.TaskOutput))
{
// Console.WriteLine($"output file: {output.FilePath}");
readTasks.Add(output.DownloadToFileAsync($"{JobId}-{output.FilePath}", System.IO.FileMode.Create));
}
Task.WaitAll(readTasks.ToArray());
}
// Print out some timing info
timer.Stop();
Console.WriteLine();
Console.WriteLine("Sample end: {0}", DateTime.Now);
Console.WriteLine("Elapsed time: {0}", timer.Elapsed);
// Clean up Storage resources
container.DeleteIfExistsAsync().Wait();
Console.WriteLine("Container [{0}] deleted.", inputContainerName);
// Clean up Batch resources (if the user so chooses)
Console.WriteLine();
Console.Write("Delete job? [yes] no: ");
string response = Console.ReadLine().ToLower();
if (response != "n" && response != "no")
{
batchClient.JobOperations.DeleteJob(JobId);
}
Console.Write("Delete pool? [yes] no: ");
response = Console.ReadLine().ToLower();
if (response != "n" && response != "no")
{
batchClient.PoolOperations.DeletePool(PoolId);
}
}
}
finally
{
Console.WriteLine();
Console.WriteLine("Sample complete, hit ENTER to exit...");
Console.ReadLine();
}
}
public static async Task <CloudBlobContainer> Run(
BatchClient batchClient,
CloudStorageAccount linkedStorageAccount,
string poolId,
int nodeCount,
string jobId)
{
const string appPackageId = "PersistOutputsTask";
const string appPackageVersion = "1.0";
// Create and configure an unbound pool.
CloudPool pool = batchClient.PoolOperations.CreatePool(
poolId: poolId,
virtualMachineSize: "standard_d1_v2",
targetDedicatedComputeNodes: nodeCount,
cloudServiceConfiguration: new CloudServiceConfiguration(osFamily: "5"));
// Specify the application and version to deploy to the compute nodes. You must
// first build PersistOutputsTask, then upload it as an application package.
// See https://azure.microsoft.com/documentation/articles/batch-application-packages/
pool.ApplicationPackageReferences = new List <ApplicationPackageReference>
{
new ApplicationPackageReference
{
ApplicationId = appPackageId,
Version = appPackageVersion
}
};
// Commit the pool to the Batch service
await GettingStartedCommon.CreatePoolIfNotExistAsync(batchClient, pool);
CloudJob job = batchClient.JobOperations.CreateJob(jobId, new PoolInformation {
PoolId = poolId
});
// Create the blob storage container for the outputs.
await job.PrepareOutputStorageAsync(linkedStorageAccount);
// Create an environment variable on the compute nodes that the
// task application can reference when persisting its outputs.
string containerName = job.OutputStorageContainerName();
CloudBlobContainer container = linkedStorageAccount.CreateCloudBlobClient().GetContainerReference(containerName);
string containerUrl = job.GetOutputStorageContainerUrl(linkedStorageAccount);
job.CommonEnvironmentSettings = new[] { new EnvironmentSetting("JOB_CONTAINER_URL", containerUrl) };
// Commit the job to the Batch service
await job.CommitAsync();
Console.WriteLine($"Created job {jobId}");
// Obtain the bound job from the Batch service
await job.RefreshAsync();
IEnumerable <CloudTask> tasks = Enumerable.Range(1, 20).Select(i =>
new CloudTask(i.ToString().PadLeft(3, '0'), $"cmd /c %AZ_BATCH_APP_PACKAGE_{appPackageId.ToUpper()}#{appPackageVersion}%\\PersistOutputsTask.exe")
);
// Add the tasks to the job; the tasks are automatically
// scheduled for execution on the nodes by the Batch service.
await job.AddTaskAsync(tasks);
Console.WriteLine($"All tasks added to job {job.Id}");
Console.WriteLine();
Console.WriteLine($"Downloading outputs to {Directory.GetCurrentDirectory()}");
foreach (CloudTask task in job.CompletedTasks())
{
if (task.ExecutionInformation.Result != TaskExecutionResult.Success)
{
Console.WriteLine($"Task {task.Id} failed");
Console.WriteLine(SampleHelpers.GetFailureInfoDetails(task.ExecutionInformation.FailureInformation));
}
else
{
Console.WriteLine($"Task {task.Id} completed successfully");
}
foreach (OutputFileReference output in task.OutputStorage(linkedStorageAccount).ListOutputs(TaskOutputKind.TaskOutput))
{
Console.WriteLine($"output file: {output.FilePath}");
await output.DownloadToFileAsync($"{jobId}-{output.FilePath}", System.IO.FileMode.Create);
}
}
return(container);
}