private static async Task MainAsync()
{
const string poolId = "FileHandlingPool";
const string jobId = "FileHandlingJobDemo";
var settings = Config.LoadAccountSettings();
SetupStorage(settings.StorageAccountName, settings.StorageAccountKey);
BatchSharedKeyCredentials cred = new BatchSharedKeyCredentials(settings.BatchServiceUrl, settings.BatchAccountName, settings.BatchAccountKey);
using (BatchClient batchClient = BatchClient.Open(cred))
{
var pool = await BatchUtils.CreatePoolIfNotExistAsync(batchClient, poolId);
var job = await BatchUtils.CreateJobIfNotExistAsync(batchClient, poolId, jobId);
//set up auto storage file
ResourceFile autoStorageFile = ResourceFile.FromAutoStorageContainer(AutoStorageContainerName, AutoStorageFileName);
Console.WriteLine("\n[INFO] Autostorage resource File reference: ");
Console.WriteLine("AutoStorageContainer: " + autoStorageFile.AutoStorageContainerName);
Console.WriteLine("FilePath: " + autoStorageFile.FilePath);
//upload file to external storage and add it as a resource file
string storageConnectionString =
$"DefaultEndpointsProtocol=https;AccountName={settings.StorageAccountName};AccountKey={settings.StorageAccountKey}";
CloudStorageAccount storageAccount = CloudStorageAccount.Parse(storageConnectionString);
CloudBlobClient blobClient = storageAccount.CreateCloudBlobClient();
CloudBlobContainer externalContainer = blobClient.GetContainerReference(ExternalStorageContainerName);
await externalContainer.CreateIfNotExistsAsync();
var externalFile = await UploadFileToContainer(blobClient, ExternalStorageContainerName, "resource_files/resource_file.txt", "resource_file.txt");
Console.WriteLine("\n[INFO] External storage resource File reference:");
Console.WriteLine("SAS Url: " + externalFile.HttpUrl);
Console.WriteLine("FilePath: " + externalFile.FilePath);
// using staging files API
var filesToStage = new List <IFileStagingProvider>();
StagingStorageAccount fileStagingStorageAccount = new StagingStorageAccount(
storageAccount: settings.StorageAccountName,
storageAccountKey: settings.StorageAccountKey,
blobEndpoint: storageAccount.BlobEndpoint.ToString());
FileToStage stagedFile = new FileToStage("resource_files/staged_file.txt", fileStagingStorageAccount);
Console.WriteLine("\n[INFO] Staged File added:");
Console.WriteLine("Local File: " + stagedFile.LocalFileToStage);
Console.WriteLine("Node File: " + stagedFile.NodeFileName);
filesToStage.Add(stagedFile);
// setup output files
// Generate SAS for outputcontainer
CloudBlobContainer outputContainer = blobClient.GetContainerReference(OutputContainerName);
await outputContainer.CreateIfNotExistsAsync();
string containerSas = outputContainer.GetSharedAccessSignature(new SharedAccessBlobPolicy()
{
Permissions = SharedAccessBlobPermissions.Write,
SharedAccessExpiryTime = DateTimeOffset.UtcNow.AddDays(1)
});
string containerUrl = outputContainer.Uri.AbsoluteUri + containerSas;
Console.WriteLine("\n[INFO] Output container: " + containerUrl);
Console.WriteLine("\nPress return to continue...");
Console.ReadLine();
// Create tasks
List <CloudTask> tasks = new List <CloudTask>();
for (var i = 1; i <= 10; i++)
{
var taskId = i.ToString().PadLeft(3, '0');
var commandLine = $@"/bin/bash -c ""echo 'Hello from {taskId}' && printf 'root dir:\n' > output.txt && ls -la >> output.txt && printf '\ninput dir:\n' >> output.txt && ls -la input >> output.txt""";
var task = new CloudTask(taskId, commandLine);
// add resource files to task (one autostorage, one in external storage)
task.ResourceFiles = new[] { autoStorageFile, externalFile };
// add staged files
task.FilesToStage = filesToStage;
// add output files
var outputFiles = new List <OutputFile>
{
new OutputFile(
filePattern: @"../std*.txt",
destination: new OutputFileDestination(new OutputFileBlobContainerDestination(
containerUrl: containerUrl,
path: taskId)),
uploadOptions: new OutputFileUploadOptions(
uploadCondition: OutputFileUploadCondition.TaskCompletion)),
new OutputFile(
filePattern: @"output.txt",
destination: new OutputFileDestination(new OutputFileBlobContainerDestination(
containerUrl: containerUrl,
path: taskId + @"\output.txt")),
uploadOptions: new OutputFileUploadOptions(
uploadCondition: OutputFileUploadCondition.TaskCompletion)),
};
task.OutputFiles = outputFiles;
tasks.Add(task);
}
Console.WriteLine("Submitting tasks and awaiting completion...");
// Add all tasks to the job.
batchClient.JobOperations.AddTask(job.Id, tasks);
await BatchUtils.WaitForTasksAndPrintOutputAsync(batchClient, job.ListTasks(), TimeSpan.FromMinutes(30));
// Clean up Batch resources (if the user so chooses)
Console.WriteLine();
Console.Write("Delete job? [yes] no: ");
string response = Console.ReadLine().ToLower();
if (response != "n" && response != "no")
{
batchClient.JobOperations.DeleteJob(jobId);
}
Console.Write("Delete pool? [yes] no: ");
response = Console.ReadLine().ToLower();
if (response != "n" && response != "no")
{
batchClient.PoolOperations.DeletePool(poolId);
}
}
}
