/// <summary>
/// Creates a pool if it doesn't already exist. If the pool already exists, this method resizes it to meet the expected
/// targets specified in settings.
/// </summary>
/// <param name="batchClient">The BatchClient to use when interacting with the Batch service.</param>
/// <param name="cloudStorageAccount">The CloudStorageAccount to upload start task required files to.</param>
/// <returns>An asynchronous <see cref="Task"/> representing the operation.</returns>
private async Task CreatePoolIfNotExistAsync(BatchClient batchClient, CloudStorageAccount cloudStorageAccount)
{
// You can learn more about os families and versions at:
// https://azure.microsoft.com/en-us/documentation/articles/cloud-services-guestos-update-matrix/
CloudPool pool = batchClient.PoolOperations.CreatePool(
poolId: this.jobManagerSettings.PoolId,
targetDedicated: this.jobManagerSettings.PoolTargetNodeCount,
virtualMachineSize: this.jobManagerSettings.PoolNodeVirtualMachineSize,
cloudServiceConfiguration: new CloudServiceConfiguration(this.jobManagerSettings.PoolOSFamily));
// Create a new start task to facilitate pool-wide file management or installation.
// In this case, we just add a single dummy data file to the StartTask.
string localSampleFilePath = GettingStartedCommon.GenerateTemporaryFile("StartTask.txt", "hello from Batch JobManager sample!");
List <string> files = new List <string> {
localSampleFilePath
};
List <ResourceFile> resourceFiles = await SampleHelpers.UploadResourcesAndCreateResourceFileReferencesAsync(
cloudStorageAccount,
this.jobManagerSettings.BlobContainer,
files);
pool.StartTask = new StartTask()
{
CommandLine = "cmd /c dir",
ResourceFiles = resourceFiles
};
await GettingStartedCommon.CreatePoolIfNotExistAsync(batchClient, pool);
}
/// <summary>
/// Creates a job and adds a task to it. The task is a
/// custom executable which has a resource file associated with it.
/// </summary>
/// <param name="batchClient">The BatchClient to use when interacting with the Batch service.</param>
/// <param name="cloudStorageAccount">The storage account to upload the files to.</param>
/// <param name="jobId">The ID of the job.</param>
/// <returns>The set of container names containing the jobs input files.</returns>
private async Task <HashSet <string> > SubmitJobAsync(BatchClient batchClient, CloudStorageAccount cloudStorageAccount, string jobId)
{
// create an empty unbound Job
CloudJob unboundJob = batchClient.JobOperations.CreateJob();
unboundJob.Id = jobId;
unboundJob.PoolInformation = new PoolInformation()
{
PoolId = this.poolsAndResourceFileSettings.PoolId
};
// Commit Job to create it in the service
await unboundJob.CommitAsync();
List <CloudTask> tasksToRun = new List <CloudTask>();
// Create a task which requires some resource files
CloudTask taskWithFiles = new CloudTask("task_with_file1", SimpleTaskExe);
// Set up a collection of files to be staged -- these files will be uploaded to Azure Storage
// when the tasks are submitted to the Azure Batch service.
taskWithFiles.FilesToStage = new List <IFileStagingProvider>();
// generate a local file in temp directory
string localSampleFile = Path.Combine(Environment.GetEnvironmentVariable("TEMP"), "HelloWorld.txt");
File.WriteAllText(localSampleFile, "hello from Batch PoolsAndResourceFiles sample!");
StagingStorageAccount fileStagingStorageAccount = new StagingStorageAccount(
storageAccount: this.accountSettings.StorageAccountName,
storageAccountKey: this.accountSettings.StorageAccountKey,
blobEndpoint: cloudStorageAccount.BlobEndpoint.ToString());
// add the files as a task dependency so they will be uploaded to storage before the task
// is submitted and downloaded to the node before the task starts execution.
FileToStage helloWorldFile = new FileToStage(localSampleFile, fileStagingStorageAccount);
FileToStage simpleTaskFile = new FileToStage(SimpleTaskExe, fileStagingStorageAccount);
// When this task is added via JobOperations.AddTaskAsync below, the FilesToStage are uploaded to storage once.
// The Batch service does not automatically delete content from your storage account, so files added in this
// way must be manually removed when they are no longer used.
taskWithFiles.FilesToStage.Add(helloWorldFile);
taskWithFiles.FilesToStage.Add(simpleTaskFile);
tasksToRun.Add(taskWithFiles);
var fileStagingArtifacts = new ConcurrentBag <ConcurrentDictionary <Type, IFileStagingArtifact> >();
// Use the AddTask method which takes an enumerable of tasks for best performance, as it submits up to 100
// tasks at once in a single request. If the list of tasks is N where N > 100, this will correctly parallelize
// the requests and return when all N tasks have been added.
await batchClient.JobOperations.AddTaskAsync(jobId, tasksToRun, fileStagingArtifacts : fileStagingArtifacts);
// Extract the names of the blob containers from the file staging artifacts
HashSet <string> blobContainerNames = GettingStartedCommon.ExtractBlobContainerNames(fileStagingArtifacts);
return(blobContainerNames);
}
/// <summary>
/// Creates a CloudPool with a single compute node associated with the Batch account.
/// </summary>
/// <param name="batchClient">A fully initialized <see cref="BatchClient"/>.</param>
/// <param name="poolId">The ID of the <see cref="CloudPool"/>.</param>
private async static Task CreatePool(BatchClient batchClient, string poolId)
{
// Create and configure an unbound pool with the specified ID
CloudPool pool = batchClient.PoolOperations.CreatePool(poolId: poolId,
osFamily: "3",
virtualMachineSize: "small",
targetDedicated: 1);
await GettingStartedCommon.CreatePoolIfNotExistAsync(batchClient, pool);
}
// Creates a pool so that the sample jobs have somewhere to run, so that they can
// make progress and you can see their progress being tracked by the MetricMonitor.
private async Task CreatePoolAsync()
{
var pool = this.batchClient.PoolOperations.CreatePool(
poolId: PoolId,
targetDedicated: PoolNodeCount,
virtualMachineSize: PoolNodeSize,
cloudServiceConfiguration: new CloudServiceConfiguration(PoolOSFamily));
pool.MaxTasksPerComputeNode = 2;
var createPoolResult = await GettingStartedCommon.CreatePoolIfNotExistAsync(this.batchClient, pool);
this.createdNewPool = (createPoolResult == CreatePoolResult.CreatedNew);
}
public async Task RunAsync()
{
Console.WriteLine("JobManager for account: {0}, job: {1} has started...",
this.accountName,
this.jobId);
Console.WriteLine();
Console.WriteLine("JobManager running with the following settings: ");
Console.WriteLine("----------------------------------------");
Console.WriteLine(this.configurationSettings.ToString());
BatchSharedKeyCredentials credentials = new BatchSharedKeyCredentials(
this.configurationSettings.BatchAccountUrl,
this.configurationSettings.BatchAccountName,
this.configurationSettings.BatchAccountKey);
CloudStorageAccount storageAccount = new CloudStorageAccount(
new StorageCredentials(
this.configurationSettings.StorageAccountName,
this.configurationSettings.StorageAccountKey),
new Uri(this.configurationSettings.StorageBlobEndpoint),
null,
null,
null);
using (BatchClient batchClient = await BatchClient.OpenAsync(credentials))
{
HashSet <string> blobContainerNames = new HashSet <string>();
try
{
// Submit some tasks
blobContainerNames = await this.SubmitTasks(batchClient);
// Wait for the tasks to finish
List <CloudTask> tasks = await batchClient.JobOperations.ListTasks(jobId).ToListAsync();
// don't wait for the job manager task since it won't finish until this method exists
tasks.RemoveAll(t => t.Id.Equals(this.taskId, StringComparison.CurrentCultureIgnoreCase));
await GettingStartedCommon.WaitForTasksAndPrintOutputAsync(batchClient, tasks, TimeSpan.FromMinutes(10));
}
finally
{
// Clean up the files for the tasks
SampleHelpers.DeleteContainersAsync(storageAccount, blobContainerNames).Wait();
}
}
}
/// <summary>
/// Creates a CloudPool associated with the Batch account. If an existing pool with the specified ID is found,
/// the pool is resized to match the specified node count.
/// </summary>
/// <param name="batchClient">A fully initialized <see cref="BatchClient"/>.</param>
/// <param name="poolId">The ID of the <see cref="CloudPool"/>.</param>
/// <param name="nodeSize">The size of the nodes within the pool.</param>
/// <param name="nodeCount">The number of nodes to create within the pool.</param>
/// <param name="maxTasksPerNode">The maximum number of tasks to run concurrently on each node.</param>
/// <returns>A <see cref="System.Threading.Tasks.Task"/> object that represents the asynchronous operation.</returns>
private async static Task CreatePoolAsync(BatchClient batchClient, string poolId, string nodeSize, int nodeCount, int maxTasksPerNode)
{
// Create and configure an unbound pool with the specified ID
CloudPool pool = batchClient.PoolOperations.CreatePool(poolId: poolId,
osFamily: "3",
virtualMachineSize: nodeSize,
targetDedicated: nodeCount);
pool.MaxTasksPerComputeNode = maxTasksPerNode;
// We want each node to be completely filled with tasks (i.e. up to maxTasksPerNode) before
// tasks are assigned to the next node in the pool
pool.TaskSchedulingPolicy = new TaskSchedulingPolicy(ComputeNodeFillType.Pack);
await GettingStartedCommon.CreatePoolIfNotExistAsync(batchClient, pool);
}
/// <summary>
/// Submits a set of tasks to the job
/// </summary>
/// <param name="batchClient">The batch client to use.</param>
/// <returns>The set of blob artifacts created by file staging.</returns>
private async Task <HashSet <string> > SubmitTasks(BatchClient batchClient)
{
List <CloudTask> tasksToRun = new List <CloudTask>();
// Create a task which requires some resource files
CloudTask taskWithFiles = new CloudTask("task_with_file1", SimpleTaskExe);
// Set up a collection of files to be staged -- these files will be uploaded to Azure Storage
// when the tasks are submitted to the Azure Batch service.
taskWithFiles.FilesToStage = new List <IFileStagingProvider>();
// generate a local file in temp directory
string localSampleFilePath = GettingStartedCommon.GenerateTemporaryFile("HelloWorld.txt", "hello from Batch JobManager sample!");
StagingStorageAccount fileStagingStorageAccount = new StagingStorageAccount(
storageAccount: this.configurationSettings.StorageAccountName,
storageAccountKey: this.configurationSettings.StorageAccountKey,
blobEndpoint: this.configurationSettings.StorageBlobEndpoint);
// add the files as a task dependency so they will be uploaded to storage before the task
// is submitted and downloaded to the node before the task starts execution.
FileToStage helloWorldFile = new FileToStage(localSampleFilePath, fileStagingStorageAccount);
FileToStage simpleTaskFile = new FileToStage(SimpleTaskExe, fileStagingStorageAccount);
// When this task is added via JobOperations.AddTaskAsync below, the FilesToStage are uploaded to storage once.
// The Batch service does not automatically delete content from your storage account, so files added in this
// way must be manually removed when they are no longer used.
taskWithFiles.FilesToStage.Add(helloWorldFile);
taskWithFiles.FilesToStage.Add(simpleTaskFile);
tasksToRun.Add(taskWithFiles);
var fileStagingArtifacts = new ConcurrentBag <ConcurrentDictionary <Type, IFileStagingArtifact> >();
// Use the AddTask method which takes an enumerable of tasks for best performance, as it submits up to 100
// tasks at once in a single request. If the list of tasks is N where N > 100, this will correctly parallelize
// the requests and return when all N tasks have been added.
await batchClient.JobOperations.AddTaskAsync(jobId, tasksToRun, fileStagingArtifacts : fileStagingArtifacts);
// Extract the names of the blob containers from the file staging artifacts
HashSet <string> blobContainerNames = GettingStartedCommon.ExtractBlobContainerNames(fileStagingArtifacts);
return(blobContainerNames);
}
private static async Task CreatePoolAsync(BatchClient batchClient, string poolId, int nodeCount)
{
Func <ImageReference, bool> imageScanner = imageRef =>
imageRef.Publisher == "MicrosoftWindowsServer" &&
imageRef.Offer == "WindowsServer" &&
imageRef.Sku.Contains("2012-R2-Datacenter");
SampleHelpers.SkuAndImage skuAndImage = await SampleHelpers.GetNodeAgentSkuReferenceAsync(batchClient, imageScanner);
// Create and configure an unbound pool.
CloudPool pool = batchClient.PoolOperations.CreatePool(poolId: poolId,
virtualMachineSize: "standard_d1_v2",
targetDedicatedComputeNodes: nodeCount,
virtualMachineConfiguration: new VirtualMachineConfiguration(
skuAndImage.Image,
skuAndImage.Sku.Id));
// Commit the pool to the Batch service
await GettingStartedCommon.CreatePoolIfNotExistAsync(batchClient, pool);
}
private static async Task CreatePoolAsync(BatchClient batchClient, string poolId, int nodeCount)
{
Func <ImageReference, bool> imageScanner = imageRef =>
imageRef.Publisher.Equals("MicrosoftWindowsServer", StringComparison.InvariantCultureIgnoreCase) &&
imageRef.Offer.Equals("WindowsServer", StringComparison.InvariantCultureIgnoreCase) &&
imageRef.Sku.IndexOf("2012-R2-Datacenter", StringComparison.InvariantCultureIgnoreCase) > -1;
ImageInformation imageInfo = await SampleHelpers.GetNodeAgentSkuReferenceAsync(batchClient, imageScanner);
// Create and configure an unbound pool.
CloudPool pool = batchClient.PoolOperations.CreatePool(poolId: poolId,
virtualMachineSize: "standard_d1_v2",
targetDedicatedComputeNodes: nodeCount,
virtualMachineConfiguration: new VirtualMachineConfiguration(
imageInfo.ImageReference,
imageInfo.NodeAgentSkuId));
// Commit the pool to the Batch service
await GettingStartedCommon.CreatePoolIfNotExistAsync(batchClient, pool);
}
/// <summary>
/// Submits a job to the Azure Batch service, and waits for it to complete
/// </summary>
private static async Task HelloWorldAsync(AccountSettings accountSettings, Settings helloWorldConfigurationSettings)
{
Console.WriteLine("Running with the following settings: ");
Console.WriteLine("-------------------------------------");
Console.WriteLine(helloWorldConfigurationSettings.ToString());
Console.WriteLine(accountSettings.ToString());
// Set up the Batch Service credentials used to authenticate with the Batch Service.
BatchSharedKeyCredentials credentials = new BatchSharedKeyCredentials(
accountSettings.BatchServiceUrl,
accountSettings.BatchAccountName,
accountSettings.BatchAccountKey);
// Get an instance of the BatchClient for a given Azure Batch account.
using (BatchClient batchClient = await BatchClient.OpenAsync(credentials))
{
// add a retry policy. The built-in policies are No Retry (default), Linear Retry, and Exponential Retry
batchClient.CustomBehaviors.Add(RetryPolicyProvider.LinearRetryProvider(TimeSpan.FromSeconds(10), 3));
string jobId = GettingStartedCommon.CreateJobId("HelloWorldJob");
try
{
// Submit the job
await SubmitJobAsync(batchClient, helloWorldConfigurationSettings, jobId);
// Wait for the job to complete
await WaitForJobAndPrintOutputAsync(batchClient, jobId);
}
finally
{
// Delete the job to ensure the tasks are cleaned up
if (!string.IsNullOrEmpty(jobId) && helloWorldConfigurationSettings.ShouldDeleteJob)
{
Console.WriteLine("Deleting job: {0}", jobId);
batchClient.JobOperations.DeleteJob(jobId);
}
}
}
}
public async static Task JobMain(string[] args)
{
//Load the configuration
Settings topNWordsConfiguration = Settings.Default;
AccountSettings accountSettings = AccountSettings.Default;
CloudStorageAccount cloudStorageAccount = new CloudStorageAccount(
new StorageCredentials(
accountSettings.StorageAccountName,
accountSettings.StorageAccountKey),
accountSettings.StorageServiceUrl,
useHttps: true);
StagingStorageAccount stagingStorageAccount = new StagingStorageAccount(
accountSettings.StorageAccountName,
accountSettings.StorageAccountKey,
cloudStorageAccount.BlobEndpoint.ToString());
using (BatchClient client = BatchClient.Open(new BatchSharedKeyCredentials(accountSettings.BatchServiceUrl, accountSettings.BatchAccountName, accountSettings.BatchAccountKey)))
{
string stagingContainer = null;
//OSFamily 4 == OS 2012 R2. You can learn more about os families and versions at:
//http://msdn.microsoft.com/en-us/library/azure/ee924680.aspx
CloudPool pool = client.PoolOperations.CreatePool(
topNWordsConfiguration.PoolId,
targetDedicatedComputeNodes: topNWordsConfiguration.PoolNodeCount,
virtualMachineSize: "small",
cloudServiceConfiguration: new CloudServiceConfiguration(osFamily: "4"));
List <string> files = new List <string>
{
Path.Combine(BatchStartTaskFolderName, BatchStartTaskTelemetryRunnerName),
};
files.AddRange(AIFilesToUpload);
var resourceHelperTask = SampleHelpers.UploadResourcesAndCreateResourceFileReferencesAsync(
cloudStorageAccount,
AIBlobConatinerName,
files);
List <ResourceFile> resourceFiles = resourceHelperTask.Result;
pool.StartTask = new StartTask()
{
CommandLine = string.Format("cmd /c {0}", BatchStartTaskTelemetryRunnerName),
ResourceFiles = resourceFiles
};
Console.WriteLine("Adding pool {0}", topNWordsConfiguration.PoolId);
try
{
await GettingStartedCommon.CreatePoolIfNotExistAsync(client, pool);
}
catch (AggregateException ae)
{
// Go through all exceptions and dump useful information
ae.Handle(x =>
{
Console.Error.WriteLine("Creating pool ID {0} failed", topNWordsConfiguration.PoolId);
if (x is BatchException)
{
BatchException be = x as BatchException;
Console.WriteLine(be.ToString());
Console.WriteLine();
}
else
{
Console.WriteLine(x);
}
// can't continue without a pool
return(false);
});
}
catch (BatchException be)
{
Console.Error.WriteLine("Creating pool ID {0} failed", topNWordsConfiguration.PoolId);
Console.WriteLine(be.ToString());
Console.WriteLine();
}
try
{
Console.WriteLine("Creating job: " + topNWordsConfiguration.JobId);
// get an empty unbound Job
CloudJob unboundJob = client.JobOperations.CreateJob();
unboundJob.Id = topNWordsConfiguration.JobId;
unboundJob.PoolInformation = new PoolInformation()
{
PoolId = topNWordsConfiguration.PoolId
};
// Commit Job to create it in the service
await unboundJob.CommitAsync();
// create file staging objects that represent the executable and its dependent assembly to run as the task.
// These files are copied to every node before the corresponding task is scheduled to run on that node.
FileToStage topNWordExe = new FileToStage(TopNWordsExeName, stagingStorageAccount);
FileToStage storageDll = new FileToStage(StorageClientDllName, stagingStorageAccount);
// Upload application insights assemblies
List <FileToStage> aiStagedFiles = new List <FileToStage>();
foreach (string aiFile in AIFilesToUpload)
{
aiStagedFiles.Add(new FileToStage(aiFile, stagingStorageAccount));
}
// In this sample, the input data is copied separately to Storage and its URI is passed to the task as an argument.
// This approach is appropriate when the amount of input data is large such that copying it to every node via FileStaging
// is not desired and the number of tasks is small since a large number of readers of the blob might get throttled
// by Storage which will lengthen the overall processing time.
//
// You'll need to observe the behavior and use published techniques for finding the right balance of performance versus
// complexity.
string[] documents = Directory.GetFiles(topNWordsConfiguration.DocumentsRootPath);
await SampleHelpers.UploadResourcesAsync(cloudStorageAccount, BooksContainerName, documents);
// initialize a collection to hold the tasks that will be submitted in their entirety
List <CloudTask> tasksToRun = new List <CloudTask>(documents.Length);
for (int i = 0; i < documents.Length; i++)
{
CloudTask task = new CloudTask("task_no_" + i, String.Format("{0} --Task {1} {2} {3} {4}",
TopNWordsExeName,
"https://onbehalfoutput.blob.core.windows.net/" + documents[i],
topNWordsConfiguration.TopWordCount,
accountSettings.StorageAccountName,
accountSettings.StorageAccountKey));
//This is the list of files to stage to a container -- for each job, one container is created and
//files all resolve to Azure Blobs by their name (so two tasks with the same named file will create just 1 blob in
//the container).
task.FilesToStage = new List <IFileStagingProvider>
{
topNWordExe,
storageDll,
};
foreach (FileToStage stagedFile in aiStagedFiles)
{
task.FilesToStage.Add(stagedFile);
}
tasksToRun.Add(task);
}
// Commit all the tasks to the Batch Service. Ask AddTask to return information about the files that were staged.
// The container information is used later on to remove these files from Storage.
ConcurrentBag <ConcurrentDictionary <Type, IFileStagingArtifact> > fsArtifactBag = new ConcurrentBag <ConcurrentDictionary <Type, IFileStagingArtifact> >();
client.JobOperations.AddTask(topNWordsConfiguration.JobId, tasksToRun, fileStagingArtifacts: fsArtifactBag);
// loop through the bag of artifacts, looking for the one that matches our staged files. Once there,
// capture the name of the container holding the files so they can be deleted later on if that option
// was configured in the settings.
foreach (var fsBagItem in fsArtifactBag)
{
IFileStagingArtifact fsValue;
if (fsBagItem.TryGetValue(typeof(FileToStage), out fsValue))
{
SequentialFileStagingArtifact stagingArtifact = fsValue as SequentialFileStagingArtifact;
if (stagingArtifact != null)
{
stagingContainer = stagingArtifact.BlobContainerCreated;
Console.WriteLine(
"Uploaded files to container: {0} -- you will be charged for their storage unless you delete them.",
stagingArtifact.BlobContainerCreated);
}
}
}
//Get the job to monitor status.
CloudJob job = client.JobOperations.GetJob(topNWordsConfiguration.JobId);
Console.Write("Waiting for tasks to complete ... ");
// Wait 20 minutes for all tasks to reach the completed state. The long timeout is necessary for the first
// time a pool is created in order to allow nodes to be added to the pool and initialized to run tasks.
IPagedEnumerable <CloudTask> ourTasks = job.ListTasks(new ODATADetailLevel(selectClause: "id"));
client.Utilities.CreateTaskStateMonitor().WaitAll(ourTasks, TaskState.Completed, TimeSpan.FromMinutes(20));
Console.WriteLine("tasks are done.");
foreach (CloudTask t in ourTasks)
{
Console.WriteLine("Task " + t.Id);
Console.WriteLine("stdout:" + Environment.NewLine + t.GetNodeFile(Batch.Constants.StandardOutFileName).ReadAsString());
Console.WriteLine();
Console.WriteLine("stderr:" + Environment.NewLine + t.GetNodeFile(Batch.Constants.StandardErrorFileName).ReadAsString());
}
}
finally
{
//Delete the pool that we created
if (topNWordsConfiguration.ShouldDeletePool)
{
Console.WriteLine("Deleting pool: {0}", topNWordsConfiguration.PoolId);
client.PoolOperations.DeletePool(topNWordsConfiguration.PoolId);
}
//Delete the job that we created
if (topNWordsConfiguration.ShouldDeleteJob)
{
Console.WriteLine("Deleting job: {0}", topNWordsConfiguration.JobId);
client.JobOperations.DeleteJob(topNWordsConfiguration.JobId);
}
//Delete the containers we created
if (topNWordsConfiguration.ShouldDeleteContainer)
{
DeleteContainers(accountSettings, stagingContainer);
}
}
}
}
/// <summary>
/// Populates Azure Storage with the required files, and
/// submits the job to the Azure Batch service.
/// </summary>
public async Task RunAsync()
{
Console.WriteLine("Running with the following settings: ");
Console.WriteLine("-------------------------------------");
Console.WriteLine(this.poolsAndResourceFileSettings.ToString());
Console.WriteLine(this.accountSettings.ToString());
// Set up the Batch Service credentials used to authenticate with the Batch Service.
BatchSharedKeyCredentials credentials = new BatchSharedKeyCredentials(
this.accountSettings.BatchServiceUrl,
this.accountSettings.BatchAccountName,
this.accountSettings.BatchAccountKey);
// Delete the blob containers which contain the task input files since we no longer need them
CloudStorageAccount cloudStorageAccount = new CloudStorageAccount(
new StorageCredentials(this.accountSettings.StorageAccountName,
this.accountSettings.StorageAccountKey),
this.accountSettings.StorageServiceUrl,
useHttps: true);
// Get an instance of the BatchClient for a given Azure Batch account.
using (BatchClient batchClient = await BatchClient.OpenAsync(credentials))
{
// add a retry policy. The built-in policies are No Retry (default), Linear Retry, and Exponential Retry
batchClient.CustomBehaviors.Add(RetryPolicyProvider.LinearRetryProvider(TimeSpan.FromSeconds(10), 3));
string jobId = null;
// Track the containers which are created as part of job submission so that we can clean them up later.
HashSet <string> blobContainerNames = new HashSet <string>();
try
{
// Allocate a pool
await this.CreatePoolIfNotExistAsync(batchClient, cloudStorageAccount);
// Submit the job
jobId = GettingStartedCommon.CreateJobId("SimpleJob");
blobContainerNames = await this.SubmitJobAsync(batchClient, cloudStorageAccount, jobId);
// Print out the status of the pools/jobs under this account
await GettingStartedCommon.PrintJobsAsync(batchClient);
await GettingStartedCommon.PrintPoolsAsync(batchClient);
// Wait for the job to complete
List <CloudTask> tasks = await batchClient.JobOperations.ListTasks(jobId).ToListAsync();
await GettingStartedCommon.WaitForTasksAndPrintOutputAsync(batchClient, tasks, TimeSpan.FromMinutes(10));
}
finally
{
// Delete the pool (if configured) and job
// TODO: In C# 6 we can await here instead of .Wait()
// Delete Azure Storage container data
SampleHelpers.DeleteContainersAsync(cloudStorageAccount, blobContainerNames).Wait();
// Delete Azure Batch resources
List <string> jobIdsToDelete = new List <string>();
List <string> poolIdsToDelete = new List <string>();
if (this.poolsAndResourceFileSettings.ShouldDeleteJob)
{
jobIdsToDelete.Add(jobId);
}
if (this.poolsAndResourceFileSettings.ShouldDeletePool)
{
poolIdsToDelete.Add(this.poolsAndResourceFileSettings.PoolId);
}
SampleHelpers.DeleteBatchResourcesAsync(batchClient, jobIdsToDelete, poolIdsToDelete).Wait();
}
}
}
public static async Task <CloudBlobContainer> Run(
BatchClient batchClient,
CloudStorageAccount linkedStorageAccount,
string poolId,
int nodeCount,
string jobId)
{
const string appPackageId = "PersistOutputsTask";
const string appPackageVersion = "1.0";
// Create and configure an unbound pool.
CloudPool pool = batchClient.PoolOperations.CreatePool(
poolId: poolId,
virtualMachineSize: "standard_d1_v2",
targetDedicatedComputeNodes: nodeCount,
cloudServiceConfiguration: new CloudServiceConfiguration(osFamily: "5"));
// Specify the application and version to deploy to the compute nodes. You must
// first build PersistOutputsTask, then upload it as an application package.
// See https://azure.microsoft.com/documentation/articles/batch-application-packages/
pool.ApplicationPackageReferences = new List <ApplicationPackageReference>
{
new ApplicationPackageReference
{
ApplicationId = appPackageId,
Version = appPackageVersion
}
};
// Commit the pool to the Batch service
await GettingStartedCommon.CreatePoolIfNotExistAsync(batchClient, pool);
CloudJob job = batchClient.JobOperations.CreateJob(jobId, new PoolInformation {
PoolId = poolId
});
// Create the blob storage container for the outputs.
await job.PrepareOutputStorageAsync(linkedStorageAccount);
// Create an environment variable on the compute nodes that the
// task application can reference when persisting its outputs.
string containerName = job.OutputStorageContainerName();
CloudBlobContainer container = linkedStorageAccount.CreateCloudBlobClient().GetContainerReference(containerName);
string containerUrl = job.GetOutputStorageContainerUrl(linkedStorageAccount);
job.CommonEnvironmentSettings = new[] { new EnvironmentSetting("JOB_CONTAINER_URL", containerUrl) };
// Commit the job to the Batch service
await job.CommitAsync();
Console.WriteLine($"Created job {jobId}");
// Obtain the bound job from the Batch service
await job.RefreshAsync();
IEnumerable <CloudTask> tasks = Enumerable.Range(1, 20).Select(i =>
new CloudTask(i.ToString().PadLeft(3, '0'), $"cmd /c %AZ_BATCH_APP_PACKAGE_{appPackageId.ToUpper()}#{appPackageVersion}%\\PersistOutputsTask.exe")
);
// Add the tasks to the job; the tasks are automatically
// scheduled for execution on the nodes by the Batch service.
await job.AddTaskAsync(tasks);
Console.WriteLine($"All tasks added to job {job.Id}");
Console.WriteLine();
Console.WriteLine($"Downloading outputs to {Directory.GetCurrentDirectory()}");
foreach (CloudTask task in job.CompletedTasks())
{
if (task.ExecutionInformation.Result != TaskExecutionResult.Success)
{
Console.WriteLine($"Task {task.Id} failed");
Console.WriteLine(SampleHelpers.GetFailureInfoDetails(task.ExecutionInformation.FailureInformation));
}
else
{
Console.WriteLine($"Task {task.Id} completed successfully");
}
foreach (OutputFileReference output in task.OutputStorage(linkedStorageAccount).ListOutputs(TaskOutputKind.TaskOutput))
{
Console.WriteLine($"output file: {output.FilePath}");
await output.DownloadToFileAsync($"{jobId}-{output.FilePath}", System.IO.FileMode.Create);
}
}
return(container);
}
public static void JobMain(string[] args)
{
//Load the configuration
Settings topNWordsConfiguration = new ConfigurationBuilder()
.SetBasePath(Directory.GetCurrentDirectory())
.AddJsonFile("settings.json")
.Build()
.Get <Settings>();
AccountSettings accountSettings = SampleHelpers.LoadAccountSettings();
CloudStorageAccount cloudStorageAccount = new CloudStorageAccount(
new StorageCredentials(
accountSettings.StorageAccountName,
accountSettings.StorageAccountKey),
accountSettings.StorageServiceUrl,
useHttps: true);
StagingStorageAccount stagingStorageAccount = new StagingStorageAccount(
accountSettings.StorageAccountName,
accountSettings.StorageAccountKey,
cloudStorageAccount.BlobEndpoint.ToString());
using (BatchClient client = BatchClient.Open(new BatchSharedKeyCredentials(accountSettings.BatchServiceUrl, accountSettings.BatchAccountName, accountSettings.BatchAccountKey)))
{
string stagingContainer = null;
//OSFamily 5 == Windows 2016. You can learn more about os families and versions at:
//http://msdn.microsoft.com/en-us/library/azure/ee924680.aspx
CloudPool pool = client.PoolOperations.CreatePool(
topNWordsConfiguration.PoolId,
targetDedicatedComputeNodes: topNWordsConfiguration.PoolNodeCount,
virtualMachineSize: "standard_d1_v2",
cloudServiceConfiguration: new CloudServiceConfiguration(osFamily: "6"));
Console.WriteLine("Adding pool {0}", topNWordsConfiguration.PoolId);
pool.TaskSchedulingPolicy = new TaskSchedulingPolicy(ComputeNodeFillType.Spread);
pool.MaxTasksPerComputeNode = 4;
GettingStartedCommon.CreatePoolIfNotExistAsync(client, pool).Wait();
var formula = @"startingNumberOfVMs = 2;
maxNumberofVMs = 4;
pendingTaskSamplePercent = $PendingTasks.GetSamplePercent(90 * TimeInterval_Second);
pendingTaskSamples = pendingTaskSamplePercent < 70 ? startingNumberOfVMs : avg($PendingTasks.GetSample(180 * TimeInterval_Second));
$TargetDedicatedNodes = min(maxNumberofVMs, pendingTaskSamples);
$NodeDeallocationOption = taskcompletion;";
var noOfSeconds = 150;
Thread.Sleep(noOfSeconds * 1000);
client.PoolOperations.EnableAutoScale(
poolId: topNWordsConfiguration.PoolId, autoscaleFormula: formula,
autoscaleEvaluationInterval: TimeSpan.FromMinutes(5));
try
{
Console.WriteLine("Creating job: " + topNWordsConfiguration.JobId);
// get an empty unbound Job
CloudJob unboundJob = client.JobOperations.CreateJob();
unboundJob.Id = topNWordsConfiguration.JobId;
unboundJob.PoolInformation = new PoolInformation()
{
PoolId = topNWordsConfiguration.PoolId
};
// Commit Job to create it in the service
unboundJob.Commit();
// create file staging objects that represent the executable and its dependent assembly to run as the task.
// These files are copied to every node before the corresponding task is scheduled to run on that node.
FileToStage topNWordExe = new FileToStage(TopNWordsExeName, stagingStorageAccount);
FileToStage storageDll = new FileToStage(StorageClientDllName, stagingStorageAccount);
FileToStage newtonJsoftDll = new FileToStage(NewtonJSoftDllName, stagingStorageAccount);
FileToStage microsoftEFDll = new FileToStage(MicrosoftEntityFrameworkDllName, stagingStorageAccount);
FileToStage microsoftEFCoreDll = new FileToStage(MicrosoftEntityFrameworkCoreDllName, stagingStorageAccount);
FileToStage microsoftBCLDll = new FileToStage(MicrosoftBCLDllName, stagingStorageAccount);
FileToStage systemTasksDll = new FileToStage(SystemTasksDllName, stagingStorageAccount);
FileToStage topNWordsConfigFile = new FileToStage(TopnWordsConfig, stagingStorageAccount);
FileToStage SystemValueTupleDll = new FileToStage(SystemValueTupleDllName, stagingStorageAccount);
FileToStage DependencyInjectionAbstractionsDll = new FileToStage(DependecyInjectionAbstractionsDllName, stagingStorageAccount);
FileToStage DependencyInjectionDll = new FileToStage(DependecyInjectionDllName, stagingStorageAccount);
FileToStage LoggingAbstractionsDll = new FileToStage(LoggingAbstractionsDllName, stagingStorageAccount);
FileToStage DiagnosticsDll = new FileToStage(DiagnosticssDllName, stagingStorageAccount);
FileToStage CachingAbstractionDll = new FileToStage(CachingAbstractionsDllName, stagingStorageAccount);
FileToStage MicrosoftSqlServerDll = new FileToStage(MicrosoftSqlServerDllName, stagingStorageAccount);
FileToStage SystemComponentDll = new FileToStage(SystemComponentDllName, stagingStorageAccount);
FileToStage SystemCollectionsDll = new FileToStage(SystemCollectionsDllName, stagingStorageAccount);
FileToStage pDll = new FileToStage(pdllName, stagingStorageAccount);
FileToStage oDll = new FileToStage(odllName, stagingStorageAccount);
FileToStage lDll = new FileToStage(ldllName, stagingStorageAccount);
FileToStage hashcodeDll = new FileToStage(hashcodeDllName, stagingStorageAccount);
FileToStage clientSqlDll = new FileToStage(clientSqlClientDllName, stagingStorageAccount);
FileToStage cachingMemoryDll = new FileToStage(CachingMemoryDllName, stagingStorageAccount);
FileToStage configAbstractionDll = new FileToStage(configAbstractionDllName, stagingStorageAccount);
FileToStage SNIDll = new FileToStage(SNIDllName, stagingStorageAccount);
FileToStage relationDll = new FileToStage(relationddllName, stagingStorageAccount);
var textFile = "E:\\WeatherAPIPOC\\cities_id.txt";
var text = File.ReadAllLines(textFile);
var cityList = new List <string>(text);
// In this sample, the input data is copied separately to Storage and its URI is passed to the task as an argument.
// This approach is appropriate when the amount of input data is large such that copying it to every node via FileStaging
// is not desired and the number of tasks is small since a large number of readers of the blob might get throttled
// by Storage which will lengthen the overall processing time.
//
// You'll need to observe the behavior and use published techniques for finding the right balance of performance versus
// complexity.
Console.WriteLine("{0} uploaded to cloud", topNWordsConfiguration.FileName);
// initialize a collection to hold the tasks that will be submitted in their entirety
List <CloudTask> tasksToRun = new List <CloudTask>(topNWordsConfiguration.NumberOfTasks);
for (int i = 0; i < cityList.Count; i++)
{
string programLaunchTime = DateTime.Now.ToString("h:mm:sstt");
CloudTask task = new CloudTask(
id: $"task_no_{i + 1}",
commandline: $"cmd /c mkdir x64 & move SNI.dll x64 & {TopNWordsExeName} --Task {cityList[i]} %AZ_BATCH_NODE_ID% {programLaunchTime}");
//This is the list of files to stage to a container -- for each job, one container is created and
//files all resolve to Azure Blobs by their name (so two tasks with the same named file will create just 1 blob in
//the container).
task.FilesToStage = new List <IFileStagingProvider>
{
topNWordExe,
storageDll,
newtonJsoftDll,
microsoftEFDll,
microsoftEFCoreDll,
microsoftBCLDll,
systemTasksDll,
topNWordsConfigFile,
SystemValueTupleDll,
DependencyInjectionAbstractionsDll,
DependencyInjectionDll,
LoggingAbstractionsDll,
DiagnosticsDll,
CachingAbstractionDll,
MicrosoftSqlServerDll,
SystemComponentDll,
SystemCollectionsDll,
oDll,
pDll,
lDll,
relationDll,
hashcodeDll,
clientSqlDll,
cachingMemoryDll,
configAbstractionDll,
SNIDll
};
tasksToRun.Add(task);
}
// Commit all the tasks to the Batch Service. Ask AddTask to return information about the files that were staged.
// The container information is used later on to remove these files from Storage.
ConcurrentBag <ConcurrentDictionary <Type, IFileStagingArtifact> > fsArtifactBag = new ConcurrentBag <ConcurrentDictionary <Type, IFileStagingArtifact> >();
client.JobOperations.AddTask(topNWordsConfiguration.JobId, tasksToRun, fileStagingArtifacts: fsArtifactBag);
// loop through the bag of artifacts, looking for the one that matches our staged files. Once there,
// capture the name of the container holding the files so they can be deleted later on if that option
// was configured in the settings.
foreach (var fsBagItem in fsArtifactBag)
{
IFileStagingArtifact fsValue;
if (fsBagItem.TryGetValue(typeof(FileToStage), out fsValue))
{
SequentialFileStagingArtifact stagingArtifact = fsValue as SequentialFileStagingArtifact;
if (stagingArtifact != null)
{
stagingContainer = stagingArtifact.BlobContainerCreated;
Console.WriteLine(
"Uploaded files to container: {0} -- you will be charged for their storage unless you delete them.",
stagingArtifact.BlobContainerCreated);
}
}
}
//Get the job to monitor status.
CloudJob job = client.JobOperations.GetJob(topNWordsConfiguration.JobId);
Console.Write("Waiting for tasks to complete ... ");
// Wait 20 minutes for all tasks to reach the completed state. The long timeout is necessary for the first
// time a pool is created in order to allow nodes to be added to the pool and initialized to run tasks.
IPagedEnumerable <CloudTask> ourTasks = job.ListTasks(new ODATADetailLevel(selectClause: "id"));
client.Utilities.CreateTaskStateMonitor().WaitAll(ourTasks, TaskState.Completed, TimeSpan.FromMinutes(20));
Console.WriteLine("tasks are done.");
foreach (CloudTask t in ourTasks)
{
Console.WriteLine("Task " + t.Id);
Console.WriteLine("stdout:" + Environment.NewLine + t.GetNodeFile(Batch.Constants.StandardOutFileName).ReadAsString());
Console.WriteLine();
Console.WriteLine("stderr:" + Environment.NewLine + t.GetNodeFile(Batch.Constants.StandardErrorFileName).ReadAsString());
}
}
finally
{
//Delete the pool that we created
if (topNWordsConfiguration.ShouldDeletePool)
{
Console.WriteLine("Deleting pool: {0}", topNWordsConfiguration.PoolId);
client.PoolOperations.DeletePool(topNWordsConfiguration.PoolId);
}
//Delete the job that we created
if (topNWordsConfiguration.ShouldDeleteJob)
{
Console.WriteLine("Deleting job: {0}", topNWordsConfiguration.JobId);
client.JobOperations.DeleteJob(topNWordsConfiguration.JobId);
}
//Delete the containers we created
if (topNWordsConfiguration.ShouldDeleteContainer)
{
DeleteContainers(accountSettings, stagingContainer);
}
}
}
}
/// <summary>
/// Populates Azure Storage with the required files, and
/// submits the job to the Azure Batch service.
/// </summary>
public async Task RunAsync()
{
Console.WriteLine("Running with the following settings: ");
Console.WriteLine("-------------------------------------");
Console.WriteLine(this.jobManagerSettings.ToString());
Console.WriteLine(this.accountSettings.ToString());
// Set up the Batch Service credentials used to authenticate with the Batch Service.
BatchSharedKeyCredentials credentials = new BatchSharedKeyCredentials(
this.accountSettings.BatchServiceUrl,
this.accountSettings.BatchAccountName,
this.accountSettings.BatchAccountKey);
CloudStorageAccount cloudStorageAccount = new CloudStorageAccount(
new StorageCredentials(this.accountSettings.StorageAccountName,
this.accountSettings.StorageAccountKey),
this.accountSettings.StorageServiceUrl,
useHttps: true);
// Get an instance of the BatchClient for a given Azure Batch account.
using (BatchClient batchClient = await BatchClient.OpenAsync(credentials))
{
// add a retry policy. The built-in policies are No Retry (default), Linear Retry, and Exponential Retry
batchClient.CustomBehaviors.Add(RetryPolicyProvider.ExponentialRetryProvider(TimeSpan.FromSeconds(5), 3));
string jobId = null;
try
{
// Allocate a pool
await this.CreatePoolIfNotExistAsync(batchClient, cloudStorageAccount);
// Submit the job
jobId = GettingStartedCommon.CreateJobId("SimpleJob");
await this.SubmitJobAsync(batchClient, cloudStorageAccount, jobId);
// Print out the status of the pools/jobs under this account
await GettingStartedCommon.PrintJobsAsync(batchClient);
await GettingStartedCommon.PrintPoolsAsync(batchClient);
// Wait for the job manager to complete
CloudTask jobManagerTask = await batchClient.JobOperations.GetTaskAsync(jobId, JobManagerTaskId);
await GettingStartedCommon.WaitForTasksAndPrintOutputAsync(batchClient, new List <CloudTask> {
jobManagerTask
}, TimeSpan.FromMinutes(10));
}
finally
{
// Delete Azure Batch resources
List <string> jobIdsToDelete = new List <string>();
List <string> poolIdsToDelete = new List <string>();
if (this.jobManagerSettings.ShouldDeleteJob)
{
jobIdsToDelete.Add(jobId);
}
if (this.jobManagerSettings.ShouldDeletePool)
{
poolIdsToDelete.Add(this.jobManagerSettings.PoolId);
}
await SampleHelpers.DeleteBatchResourcesAsync(batchClient, jobIdsToDelete, poolIdsToDelete);
}
}
}
private static async Task MainAsync(string[] args)
{
// You may adjust these values to experiment with different compute resource scenarios.
const string nodeSize = "small";
const int nodeCount = 4;
const int maxTasksPerNode = 4;
const int taskCount = 32;
// Ensure there are enough tasks to help avoid hitting some timeout conditions below
int minimumTaskCount = nodeCount * maxTasksPerNode * 2;
if (taskCount < minimumTaskCount)
{
Console.WriteLine("You must specify at least two tasks per node core for this sample ({0} tasks in this configuration).", minimumTaskCount);
Console.WriteLine();
// Not enough tasks, exit the application
return;
}
// In this sample, the tasks simply ping localhost on the compute nodes; adjust these
// values to simulate variable task duration
const int minPings = 30;
const int maxPings = 60;
const string poolId = "ParallelTasksSamplePool";
const string jobId = "ParallelTasksSampleJob";
// Amount of time to wait before timing out (potentially) long-running tasks
TimeSpan longTaskDurationLimit = TimeSpan.FromMinutes(30);
// Set up access to your Batch account with a BatchClient. Configure your AccountSettings in the
// Microsoft.Azure.Batch.Samples.Common project within this solution.
BatchSharedKeyCredentials cred = new BatchSharedKeyCredentials(AccountSettings.Default.BatchServiceUrl,
AccountSettings.Default.BatchAccountName,
AccountSettings.Default.BatchAccountKey);
using (BatchClient batchClient = await BatchClient.OpenAsync(cred))
{
// Create a CloudPool, or obtain an existing pool with the specified ID
CloudPool pool = await ArticleHelpers.CreatePoolIfNotExistAsync(batchClient,
poolId,
nodeSize,
nodeCount,
maxTasksPerNode);
// Create a CloudJob, or obtain an existing pool with the specified ID
CloudJob job = await ArticleHelpers.CreateJobIfNotExistAsync(batchClient, poolId, jobId);
// The job's tasks ping localhost a random number of times between minPings and maxPings.
// Adjust the minPings/maxPings values above to experiment with different task durations.
Random rand = new Random();
List <CloudTask> tasks = new List <CloudTask>();
for (int i = 1; i <= taskCount; i++)
{
string taskId = "task" + i.ToString().PadLeft(3, '0');
string taskCommandLine = "ping -n " + rand.Next(minPings, maxPings + 1).ToString() + " localhost";
CloudTask task = new CloudTask(taskId, taskCommandLine);
tasks.Add(task);
}
// Pause execution until the pool is steady and its compute nodes are ready to accept jobs.
// NOTE: Such a pause is not necessary within your own code. Tasks can be added to a job at any point and will be
// scheduled to execute on a compute node as soon any node has reached Idle state. Because the focus of this sample
// is the demonstration of running tasks in parallel on multiple compute nodes, we wait for all compute nodes to
// complete initialization and reach the Idle state in order to maximize the number of compute nodes available for
// parallelization.
await ArticleHelpers.WaitForPoolToReachStateAsync(batchClient, pool.Id, AllocationState.Steady, longTaskDurationLimit);
await ArticleHelpers.WaitForNodesToReachStateAsync(batchClient, pool.Id, ComputeNodeState.Idle, longTaskDurationLimit);
// Add the tasks in one API call as opposed to a separate AddTask call for each. Bulk task submission
// helps to ensure efficient underlying API calls to the Batch service.
await batchClient.JobOperations.AddTaskAsync(job.Id, tasks);
// Pause again to wait until *all* nodes are running tasks
await ArticleHelpers.WaitForNodesToReachStateAsync(batchClient, pool.Id, ComputeNodeState.Running, TimeSpan.FromMinutes(2));
Stopwatch stopwatch = Stopwatch.StartNew();
// Print out task assignment information.
Console.WriteLine();
await GettingStartedCommon.PrintNodeTasksAsync(batchClient, pool.Id);
Console.WriteLine();
// Pause execution while we wait for all of the tasks to complete
Console.WriteLine("Waiting for task completion...");
Console.WriteLine();
try
{
await batchClient.Utilities.CreateTaskStateMonitor().WhenAll(
job.ListTasks(),
TaskState.Completed,
longTaskDurationLimit);
}
catch (TimeoutException e)
{
Console.WriteLine(e.ToString());
}
stopwatch.Stop();
// Obtain the tasks, specifying a detail level to limit the number of properties returned for each task.
// If you have a large number of tasks, specifying a DetailLevel is extremely important in reducing the
// amount of data transferred, lowering your query response times in increasing performance.
ODATADetailLevel detail = new ODATADetailLevel(selectClause: "id,commandLine,nodeInfo,state");
IPagedEnumerable <CloudTask> allTasks = batchClient.JobOperations.ListTasks(job.Id, detail);
// Get a collection of the completed tasks sorted by the compute nodes on which they executed
List <CloudTask> completedTasks = allTasks
.Where(t => t.State == TaskState.Completed)
.OrderBy(t => t.ComputeNodeInformation.ComputeNodeId)
.ToList();
// Print the completed task information
Console.WriteLine();
Console.WriteLine("Completed tasks:");
string lastNodeId = string.Empty;
foreach (CloudTask task in completedTasks)
{
if (!string.Equals(lastNodeId, task.ComputeNodeInformation.ComputeNodeId))
{
Console.WriteLine();
Console.WriteLine(task.ComputeNodeInformation.ComputeNodeId);
}
lastNodeId = task.ComputeNodeInformation.ComputeNodeId;
Console.WriteLine("\t{0}: {1}", task.Id, task.CommandLine);
}
// Get a collection of the uncompleted tasks which may exist if the TaskMonitor timeout was hit
List <CloudTask> uncompletedTasks = allTasks
.Where(t => t.State != TaskState.Completed)
.OrderBy(t => t.Id)
.ToList();
// Print a list of uncompleted tasks, if any
Console.WriteLine();
Console.WriteLine("Uncompleted tasks:");
Console.WriteLine();
if (uncompletedTasks.Any())
{
foreach (CloudTask task in uncompletedTasks)
{
Console.WriteLine("\t{0}: {1}", task.Id, task.CommandLine);
}
}
else
{
Console.WriteLine("\t<none>");
}
// Print some summary information
Console.WriteLine();
Console.WriteLine(" Nodes: " + nodeCount);
Console.WriteLine(" Node size: " + nodeSize);
Console.WriteLine("Max tasks per node: " + pool.MaxTasksPerComputeNode);
Console.WriteLine(" Tasks: " + tasks.Count);
Console.WriteLine(" Duration: " + stopwatch.Elapsed);
Console.WriteLine();
Console.WriteLine("Done!");
Console.WriteLine();
// Clean up the resources we've created in the Batch account
Console.WriteLine("Delete job? [yes] no");
string response = Console.ReadLine().ToLower();
if (response != "n" && response != "no")
{
await batchClient.JobOperations.DeleteJobAsync(job.Id);
}
Console.WriteLine("Delete pool? [yes] no");
response = Console.ReadLine();
if (response != "n" && response != "no")
{
await batchClient.PoolOperations.DeletePoolAsync(pool.Id);
}
}
}
public static void JobMain(string[] args)
{
//Load the configuration
Settings topNWordsConfiguration = new ConfigurationBuilder()
.SetBasePath(Directory.GetCurrentDirectory())
.AddJsonFile("settings.json")
.Build()
.Get <Settings>();
AccountSettings accountSettings = SampleHelpers.LoadAccountSettings();
CloudStorageAccount cloudStorageAccount = new CloudStorageAccount(
new StorageCredentials(
accountSettings.StorageAccountName,
accountSettings.StorageAccountKey),
accountSettings.StorageServiceUrl,
useHttps: true);
StagingStorageAccount stagingStorageAccount = new StagingStorageAccount(
accountSettings.StorageAccountName,
accountSettings.StorageAccountKey,
cloudStorageAccount.BlobEndpoint.ToString());
using (BatchClient client = BatchClient.Open(new BatchSharedKeyCredentials(accountSettings.BatchServiceUrl, accountSettings.BatchAccountName, accountSettings.BatchAccountKey)))
{
string stagingContainer = null;
//OSFamily 5 == Windows 2016. You can learn more about os families and versions at:
//http://msdn.microsoft.com/en-us/library/azure/ee924680.aspx
CloudPool pool = client.PoolOperations.CreatePool(
topNWordsConfiguration.PoolId,
targetDedicatedComputeNodes: topNWordsConfiguration.PoolNodeCount,
virtualMachineSize: "standard_d1_v2",
cloudServiceConfiguration: new CloudServiceConfiguration(osFamily: "5"));
Console.WriteLine("Adding pool {0}", topNWordsConfiguration.PoolId);
GettingStartedCommon.CreatePoolIfNotExistAsync(client, pool).Wait();
try
{
Console.WriteLine("Creating job: " + topNWordsConfiguration.JobId);
// get an empty unbound Job
CloudJob unboundJob = client.JobOperations.CreateJob();
unboundJob.Id = topNWordsConfiguration.JobId;
unboundJob.PoolInformation = new PoolInformation()
{
PoolId = topNWordsConfiguration.PoolId
};
// Commit Job to create it in the service
unboundJob.Commit();
// create file staging objects that represent the executable and its dependent assembly to run as the task.
// These files are copied to every node before the corresponding task is scheduled to run on that node.
FileToStage topNWordExe = new FileToStage(TopNWordsExeName, stagingStorageAccount);
FileToStage storageDll = new FileToStage(StorageClientDllName, stagingStorageAccount);
// In this sample, the input data is copied separately to Storage and its URI is passed to the task as an argument.
// This approach is appropriate when the amount of input data is large such that copying it to every node via FileStaging
// is not desired and the number of tasks is small since a large number of readers of the blob might get throttled
// by Storage which will lengthen the overall processing time.
//
// You'll need to observe the behavior and use published techniques for finding the right balance of performance versus
// complexity.
string bookFileUri = UploadBookFileToCloudBlob(accountSettings, topNWordsConfiguration.FileName);
Console.WriteLine("{0} uploaded to cloud", topNWordsConfiguration.FileName);
// initialize a collection to hold the tasks that will be submitted in their entirety
List <CloudTask> tasksToRun = new List <CloudTask>(topNWordsConfiguration.NumberOfTasks);
for (int i = 1; i <= topNWordsConfiguration.NumberOfTasks; i++)
{
CloudTask task = new CloudTask("task_no_" + i, string.Format("{0} --Task {1} {2} {3} {4}",
TopNWordsExeName,
bookFileUri,
topNWordsConfiguration.TopWordCount,
accountSettings.StorageAccountName,
accountSettings.StorageAccountKey));
//This is the list of files to stage to a container -- for each job, one container is created and
//files all resolve to Azure Blobs by their name (so two tasks with the same named file will create just 1 blob in
//the container).
task.FilesToStage = new List <IFileStagingProvider>
{
topNWordExe,
storageDll
};
tasksToRun.Add(task);
}
// Commit all the tasks to the Batch Service. Ask AddTask to return information about the files that were staged.
// The container information is used later on to remove these files from Storage.
ConcurrentBag <ConcurrentDictionary <Type, IFileStagingArtifact> > fsArtifactBag = new ConcurrentBag <ConcurrentDictionary <Type, IFileStagingArtifact> >();
client.JobOperations.AddTask(topNWordsConfiguration.JobId, tasksToRun, fileStagingArtifacts: fsArtifactBag);
// loop through the bag of artifacts, looking for the one that matches our staged files. Once there,
// capture the name of the container holding the files so they can be deleted later on if that option
// was configured in the settings.
foreach (var fsBagItem in fsArtifactBag)
{
IFileStagingArtifact fsValue;
if (fsBagItem.TryGetValue(typeof(FileToStage), out fsValue))
{
SequentialFileStagingArtifact stagingArtifact = fsValue as SequentialFileStagingArtifact;
if (stagingArtifact != null)
{
stagingContainer = stagingArtifact.BlobContainerCreated;
Console.WriteLine(
"Uploaded files to container: {0} -- you will be charged for their storage unless you delete them.",
stagingArtifact.BlobContainerCreated);
}
}
}
//Get the job to monitor status.
CloudJob job = client.JobOperations.GetJob(topNWordsConfiguration.JobId);
Console.Write("Waiting for tasks to complete ... ");
// Wait 20 minutes for all tasks to reach the completed state. The long timeout is necessary for the first
// time a pool is created in order to allow nodes to be added to the pool and initialized to run tasks.
IPagedEnumerable <CloudTask> ourTasks = job.ListTasks(new ODATADetailLevel(selectClause: "id"));
client.Utilities.CreateTaskStateMonitor().WaitAll(ourTasks, TaskState.Completed, TimeSpan.FromMinutes(20));
Console.WriteLine("tasks are done.");
foreach (CloudTask t in ourTasks)
{
Console.WriteLine("Task " + t.Id);
Console.WriteLine("stdout:" + Environment.NewLine + t.GetNodeFile(Batch.Constants.StandardOutFileName).ReadAsString());
Console.WriteLine();
Console.WriteLine("stderr:" + Environment.NewLine + t.GetNodeFile(Batch.Constants.StandardErrorFileName).ReadAsString());
}
}
finally
{
//Delete the pool that we created
if (topNWordsConfiguration.ShouldDeletePool)
{
Console.WriteLine("Deleting pool: {0}", topNWordsConfiguration.PoolId);
client.PoolOperations.DeletePool(topNWordsConfiguration.PoolId);
}
//Delete the job that we created
if (topNWordsConfiguration.ShouldDeleteJob)
{
Console.WriteLine("Deleting job: {0}", topNWordsConfiguration.JobId);
client.JobOperations.DeleteJob(topNWordsConfiguration.JobId);
}
//Delete the containers we created
if (topNWordsConfiguration.ShouldDeleteContainer)
{
DeleteContainers(accountSettings, stagingContainer);
}
}
}
}
