/// <summary>
/// Creates a pool if it doesn't already exist. If the pool already exists, this method resizes it to meet the expected
/// targets specified in settings.
/// </summary>
/// <param name="batchClient">The BatchClient to use when interacting with the Batch service.</param>
/// <param name="cloudStorageAccount">The CloudStorageAccount to upload start task required files to.</param>
/// <returns>An asynchronous <see cref="Task"/> representing the operation.</returns>
private async Task CreatePoolIfNotExistAsync(BatchClient batchClient, CloudStorageAccount cloudStorageAccount)
{
// You can learn more about os families and versions at:
// https://azure.microsoft.com/en-us/documentation/articles/cloud-services-guestos-update-matrix/
CloudPool pool = batchClient.PoolOperations.CreatePool(
poolId: this.jobManagerSettings.PoolId,
targetDedicated: this.jobManagerSettings.PoolTargetNodeCount,
virtualMachineSize: this.jobManagerSettings.PoolNodeVirtualMachineSize,
cloudServiceConfiguration: new CloudServiceConfiguration(this.jobManagerSettings.PoolOSFamily));
// Create a new start task to facilitate pool-wide file management or installation.
// In this case, we just add a single dummy data file to the StartTask.
string localSampleFilePath = GettingStartedCommon.GenerateTemporaryFile("StartTask.txt", "hello from Batch JobManager sample!");
List <string> files = new List <string> {
localSampleFilePath
};
List <ResourceFile> resourceFiles = await SampleHelpers.UploadResourcesAndCreateResourceFileReferencesAsync(
cloudStorageAccount,
this.jobManagerSettings.BlobContainer,
files);
pool.StartTask = new StartTask()
{
CommandLine = "cmd /c dir",
ResourceFiles = resourceFiles
};
await GettingStartedCommon.CreatePoolIfNotExistAsync(batchClient, pool);
}
/// <summary>
/// Creates a CloudPool with a single compute node associated with the Batch account.
/// </summary>
/// <param name="batchClient">A fully initialized <see cref="BatchClient"/>.</param>
/// <param name="poolId">The ID of the <see cref="CloudPool"/>.</param>
private async static Task CreatePool(BatchClient batchClient, string poolId)
{
// Create and configure an unbound pool with the specified ID
CloudPool pool = batchClient.PoolOperations.CreatePool(poolId: poolId,
osFamily: "3",
virtualMachineSize: "small",
targetDedicated: 1);
await GettingStartedCommon.CreatePoolIfNotExistAsync(batchClient, pool);
}
// Creates a pool so that the sample jobs have somewhere to run, so that they can
// make progress and you can see their progress being tracked by the MetricMonitor.
private async Task CreatePoolAsync()
{
var pool = this.batchClient.PoolOperations.CreatePool(
poolId: PoolId,
targetDedicated: PoolNodeCount,
virtualMachineSize: PoolNodeSize,
cloudServiceConfiguration: new CloudServiceConfiguration(PoolOSFamily));
pool.MaxTasksPerComputeNode = 2;
var createPoolResult = await GettingStartedCommon.CreatePoolIfNotExistAsync(this.batchClient, pool);
this.createdNewPool = (createPoolResult == CreatePoolResult.CreatedNew);
}
/// <summary>
/// Creates a CloudPool associated with the Batch account. If an existing pool with the specified ID is found,
/// the pool is resized to match the specified node count.
/// </summary>
/// <param name="batchClient">A fully initialized <see cref="BatchClient"/>.</param>
/// <param name="poolId">The ID of the <see cref="CloudPool"/>.</param>
/// <param name="nodeSize">The size of the nodes within the pool.</param>
/// <param name="nodeCount">The number of nodes to create within the pool.</param>
/// <param name="maxTasksPerNode">The maximum number of tasks to run concurrently on each node.</param>
/// <returns>A <see cref="System.Threading.Tasks.Task"/> object that represents the asynchronous operation.</returns>
private async static Task CreatePoolAsync(BatchClient batchClient, string poolId, string nodeSize, int nodeCount, int maxTasksPerNode)
{
// Create and configure an unbound pool with the specified ID
CloudPool pool = batchClient.PoolOperations.CreatePool(poolId: poolId,
osFamily: "3",
virtualMachineSize: nodeSize,
targetDedicated: nodeCount);
pool.MaxTasksPerComputeNode = maxTasksPerNode;
// We want each node to be completely filled with tasks (i.e. up to maxTasksPerNode) before
// tasks are assigned to the next node in the pool
pool.TaskSchedulingPolicy = new TaskSchedulingPolicy(ComputeNodeFillType.Pack);
await GettingStartedCommon.CreatePoolIfNotExistAsync(batchClient, pool);
}
private static async Task CreatePoolAsync(BatchClient batchClient, string poolId, int nodeCount)
{
Func <ImageReference, bool> imageScanner = imageRef =>
imageRef.Publisher == "MicrosoftWindowsServer" &&
imageRef.Offer == "WindowsServer" &&
imageRef.Sku.Contains("2012-R2-Datacenter");
SampleHelpers.SkuAndImage skuAndImage = await SampleHelpers.GetNodeAgentSkuReferenceAsync(batchClient, imageScanner);
// Create and configure an unbound pool.
CloudPool pool = batchClient.PoolOperations.CreatePool(poolId: poolId,
virtualMachineSize: "standard_d1_v2",
targetDedicatedComputeNodes: nodeCount,
virtualMachineConfiguration: new VirtualMachineConfiguration(
skuAndImage.Image,
skuAndImage.Sku.Id));
// Commit the pool to the Batch service
await GettingStartedCommon.CreatePoolIfNotExistAsync(batchClient, pool);
}
private static async Task CreatePoolAsync(BatchClient batchClient, string poolId, int nodeCount)
{
Func <ImageReference, bool> imageScanner = imageRef =>
imageRef.Publisher.Equals("MicrosoftWindowsServer", StringComparison.InvariantCultureIgnoreCase) &&
imageRef.Offer.Equals("WindowsServer", StringComparison.InvariantCultureIgnoreCase) &&
imageRef.Sku.IndexOf("2012-R2-Datacenter", StringComparison.InvariantCultureIgnoreCase) > -1;
ImageInformation imageInfo = await SampleHelpers.GetNodeAgentSkuReferenceAsync(batchClient, imageScanner);
// Create and configure an unbound pool.
CloudPool pool = batchClient.PoolOperations.CreatePool(poolId: poolId,
virtualMachineSize: "standard_d1_v2",
targetDedicatedComputeNodes: nodeCount,
virtualMachineConfiguration: new VirtualMachineConfiguration(
imageInfo.ImageReference,
imageInfo.NodeAgentSkuId));
// Commit the pool to the Batch service
await GettingStartedCommon.CreatePoolIfNotExistAsync(batchClient, pool);
}
public async static Task JobMain(string[] args)
{
//Load the configuration
Settings topNWordsConfiguration = Settings.Default;
AccountSettings accountSettings = AccountSettings.Default;
CloudStorageAccount cloudStorageAccount = new CloudStorageAccount(
new StorageCredentials(
accountSettings.StorageAccountName,
accountSettings.StorageAccountKey),
accountSettings.StorageServiceUrl,
useHttps: true);
StagingStorageAccount stagingStorageAccount = new StagingStorageAccount(
accountSettings.StorageAccountName,
accountSettings.StorageAccountKey,
cloudStorageAccount.BlobEndpoint.ToString());
using (BatchClient client = BatchClient.Open(new BatchSharedKeyCredentials(accountSettings.BatchServiceUrl, accountSettings.BatchAccountName, accountSettings.BatchAccountKey)))
{
string stagingContainer = null;
//OSFamily 4 == OS 2012 R2. You can learn more about os families and versions at:
//http://msdn.microsoft.com/en-us/library/azure/ee924680.aspx
CloudPool pool = client.PoolOperations.CreatePool(
topNWordsConfiguration.PoolId,
targetDedicatedComputeNodes: topNWordsConfiguration.PoolNodeCount,
virtualMachineSize: "small",
cloudServiceConfiguration: new CloudServiceConfiguration(osFamily: "4"));
List <string> files = new List <string>
{
Path.Combine(BatchStartTaskFolderName, BatchStartTaskTelemetryRunnerName),
};
files.AddRange(AIFilesToUpload);
var resourceHelperTask = SampleHelpers.UploadResourcesAndCreateResourceFileReferencesAsync(
cloudStorageAccount,
AIBlobConatinerName,
files);
List <ResourceFile> resourceFiles = resourceHelperTask.Result;
pool.StartTask = new StartTask()
{
CommandLine = string.Format("cmd /c {0}", BatchStartTaskTelemetryRunnerName),
ResourceFiles = resourceFiles
};
Console.WriteLine("Adding pool {0}", topNWordsConfiguration.PoolId);
try
{
await GettingStartedCommon.CreatePoolIfNotExistAsync(client, pool);
}
catch (AggregateException ae)
{
// Go through all exceptions and dump useful information
ae.Handle(x =>
{
Console.Error.WriteLine("Creating pool ID {0} failed", topNWordsConfiguration.PoolId);
if (x is BatchException)
{
BatchException be = x as BatchException;
Console.WriteLine(be.ToString());
Console.WriteLine();
}
else
{
Console.WriteLine(x);
}
// can't continue without a pool
return(false);
});
}
catch (BatchException be)
{
Console.Error.WriteLine("Creating pool ID {0} failed", topNWordsConfiguration.PoolId);
Console.WriteLine(be.ToString());
Console.WriteLine();
}
try
{
Console.WriteLine("Creating job: " + topNWordsConfiguration.JobId);
// get an empty unbound Job
CloudJob unboundJob = client.JobOperations.CreateJob();
unboundJob.Id = topNWordsConfiguration.JobId;
unboundJob.PoolInformation = new PoolInformation()
{
PoolId = topNWordsConfiguration.PoolId
};
// Commit Job to create it in the service
await unboundJob.CommitAsync();
// create file staging objects that represent the executable and its dependent assembly to run as the task.
// These files are copied to every node before the corresponding task is scheduled to run on that node.
FileToStage topNWordExe = new FileToStage(TopNWordsExeName, stagingStorageAccount);
FileToStage storageDll = new FileToStage(StorageClientDllName, stagingStorageAccount);
// Upload application insights assemblies
List <FileToStage> aiStagedFiles = new List <FileToStage>();
foreach (string aiFile in AIFilesToUpload)
{
aiStagedFiles.Add(new FileToStage(aiFile, stagingStorageAccount));
}
// In this sample, the input data is copied separately to Storage and its URI is passed to the task as an argument.
// This approach is appropriate when the amount of input data is large such that copying it to every node via FileStaging
// is not desired and the number of tasks is small since a large number of readers of the blob might get throttled
// by Storage which will lengthen the overall processing time.
//
// You'll need to observe the behavior and use published techniques for finding the right balance of performance versus
// complexity.
string[] documents = Directory.GetFiles(topNWordsConfiguration.DocumentsRootPath);
await SampleHelpers.UploadResourcesAsync(cloudStorageAccount, BooksContainerName, documents);
// initialize a collection to hold the tasks that will be submitted in their entirety
List <CloudTask> tasksToRun = new List <CloudTask>(documents.Length);
for (int i = 0; i < documents.Length; i++)
{
CloudTask task = new CloudTask("task_no_" + i, String.Format("{0} --Task {1} {2} {3} {4}",
TopNWordsExeName,
"https://onbehalfoutput.blob.core.windows.net/" + documents[i],
topNWordsConfiguration.TopWordCount,
accountSettings.StorageAccountName,
accountSettings.StorageAccountKey));
//This is the list of files to stage to a container -- for each job, one container is created and
//files all resolve to Azure Blobs by their name (so two tasks with the same named file will create just 1 blob in
//the container).
task.FilesToStage = new List <IFileStagingProvider>
{
topNWordExe,
storageDll,
};
foreach (FileToStage stagedFile in aiStagedFiles)
{
task.FilesToStage.Add(stagedFile);
}
tasksToRun.Add(task);
}
// Commit all the tasks to the Batch Service. Ask AddTask to return information about the files that were staged.
// The container information is used later on to remove these files from Storage.
ConcurrentBag <ConcurrentDictionary <Type, IFileStagingArtifact> > fsArtifactBag = new ConcurrentBag <ConcurrentDictionary <Type, IFileStagingArtifact> >();
client.JobOperations.AddTask(topNWordsConfiguration.JobId, tasksToRun, fileStagingArtifacts: fsArtifactBag);
// loop through the bag of artifacts, looking for the one that matches our staged files. Once there,
// capture the name of the container holding the files so they can be deleted later on if that option
// was configured in the settings.
foreach (var fsBagItem in fsArtifactBag)
{
IFileStagingArtifact fsValue;
if (fsBagItem.TryGetValue(typeof(FileToStage), out fsValue))
{
SequentialFileStagingArtifact stagingArtifact = fsValue as SequentialFileStagingArtifact;
if (stagingArtifact != null)
{
stagingContainer = stagingArtifact.BlobContainerCreated;
Console.WriteLine(
"Uploaded files to container: {0} -- you will be charged for their storage unless you delete them.",
stagingArtifact.BlobContainerCreated);
}
}
}
//Get the job to monitor status.
CloudJob job = client.JobOperations.GetJob(topNWordsConfiguration.JobId);
Console.Write("Waiting for tasks to complete ... ");
// Wait 20 minutes for all tasks to reach the completed state. The long timeout is necessary for the first
// time a pool is created in order to allow nodes to be added to the pool and initialized to run tasks.
IPagedEnumerable <CloudTask> ourTasks = job.ListTasks(new ODATADetailLevel(selectClause: "id"));
client.Utilities.CreateTaskStateMonitor().WaitAll(ourTasks, TaskState.Completed, TimeSpan.FromMinutes(20));
Console.WriteLine("tasks are done.");
foreach (CloudTask t in ourTasks)
{
Console.WriteLine("Task " + t.Id);
Console.WriteLine("stdout:" + Environment.NewLine + t.GetNodeFile(Batch.Constants.StandardOutFileName).ReadAsString());
Console.WriteLine();
Console.WriteLine("stderr:" + Environment.NewLine + t.GetNodeFile(Batch.Constants.StandardErrorFileName).ReadAsString());
}
}
finally
{
//Delete the pool that we created
if (topNWordsConfiguration.ShouldDeletePool)
{
Console.WriteLine("Deleting pool: {0}", topNWordsConfiguration.PoolId);
client.PoolOperations.DeletePool(topNWordsConfiguration.PoolId);
}
//Delete the job that we created
if (topNWordsConfiguration.ShouldDeleteJob)
{
Console.WriteLine("Deleting job: {0}", topNWordsConfiguration.JobId);
client.JobOperations.DeleteJob(topNWordsConfiguration.JobId);
}
//Delete the containers we created
if (topNWordsConfiguration.ShouldDeleteContainer)
{
DeleteContainers(accountSettings, stagingContainer);
}
}
}
}
public static async Task <CloudBlobContainer> Run(
BatchClient batchClient,
CloudStorageAccount linkedStorageAccount,
string poolId,
int nodeCount,
string jobId)
{
const string appPackageId = "PersistOutputsTask";
const string appPackageVersion = "1.0";
// Create and configure an unbound pool.
CloudPool pool = batchClient.PoolOperations.CreatePool(
poolId: poolId,
virtualMachineSize: "standard_d1_v2",
targetDedicatedComputeNodes: nodeCount,
cloudServiceConfiguration: new CloudServiceConfiguration(osFamily: "5"));
// Specify the application and version to deploy to the compute nodes. You must
// first build PersistOutputsTask, then upload it as an application package.
// See https://azure.microsoft.com/documentation/articles/batch-application-packages/
pool.ApplicationPackageReferences = new List <ApplicationPackageReference>
{
new ApplicationPackageReference
{
ApplicationId = appPackageId,
Version = appPackageVersion
}
};
// Commit the pool to the Batch service
await GettingStartedCommon.CreatePoolIfNotExistAsync(batchClient, pool);
CloudJob job = batchClient.JobOperations.CreateJob(jobId, new PoolInformation {
PoolId = poolId
});
// Create the blob storage container for the outputs.
await job.PrepareOutputStorageAsync(linkedStorageAccount);
// Create an environment variable on the compute nodes that the
// task application can reference when persisting its outputs.
string containerName = job.OutputStorageContainerName();
CloudBlobContainer container = linkedStorageAccount.CreateCloudBlobClient().GetContainerReference(containerName);
string containerUrl = job.GetOutputStorageContainerUrl(linkedStorageAccount);
job.CommonEnvironmentSettings = new[] { new EnvironmentSetting("JOB_CONTAINER_URL", containerUrl) };
// Commit the job to the Batch service
await job.CommitAsync();
Console.WriteLine($"Created job {jobId}");
// Obtain the bound job from the Batch service
await job.RefreshAsync();
IEnumerable <CloudTask> tasks = Enumerable.Range(1, 20).Select(i =>
new CloudTask(i.ToString().PadLeft(3, '0'), $"cmd /c %AZ_BATCH_APP_PACKAGE_{appPackageId.ToUpper()}#{appPackageVersion}%\\PersistOutputsTask.exe")
);
// Add the tasks to the job; the tasks are automatically
// scheduled for execution on the nodes by the Batch service.
await job.AddTaskAsync(tasks);
Console.WriteLine($"All tasks added to job {job.Id}");
Console.WriteLine();
Console.WriteLine($"Downloading outputs to {Directory.GetCurrentDirectory()}");
foreach (CloudTask task in job.CompletedTasks())
{
if (task.ExecutionInformation.Result != TaskExecutionResult.Success)
{
Console.WriteLine($"Task {task.Id} failed");
Console.WriteLine(SampleHelpers.GetFailureInfoDetails(task.ExecutionInformation.FailureInformation));
}
else
{
Console.WriteLine($"Task {task.Id} completed successfully");
}
foreach (OutputFileReference output in task.OutputStorage(linkedStorageAccount).ListOutputs(TaskOutputKind.TaskOutput))
{
Console.WriteLine($"output file: {output.FilePath}");
await output.DownloadToFileAsync($"{jobId}-{output.FilePath}", System.IO.FileMode.Create);
}
}
return(container);
}
public static void JobMain(string[] args)
{
//Load the configuration
Settings topNWordsConfiguration = new ConfigurationBuilder()
.SetBasePath(Directory.GetCurrentDirectory())
.AddJsonFile("settings.json")
.Build()
.Get <Settings>();
AccountSettings accountSettings = SampleHelpers.LoadAccountSettings();
CloudStorageAccount cloudStorageAccount = new CloudStorageAccount(
new StorageCredentials(
accountSettings.StorageAccountName,
accountSettings.StorageAccountKey),
accountSettings.StorageServiceUrl,
useHttps: true);
StagingStorageAccount stagingStorageAccount = new StagingStorageAccount(
accountSettings.StorageAccountName,
accountSettings.StorageAccountKey,
cloudStorageAccount.BlobEndpoint.ToString());
using (BatchClient client = BatchClient.Open(new BatchSharedKeyCredentials(accountSettings.BatchServiceUrl, accountSettings.BatchAccountName, accountSettings.BatchAccountKey)))
{
string stagingContainer = null;
//OSFamily 5 == Windows 2016. You can learn more about os families and versions at:
//http://msdn.microsoft.com/en-us/library/azure/ee924680.aspx
CloudPool pool = client.PoolOperations.CreatePool(
topNWordsConfiguration.PoolId,
targetDedicatedComputeNodes: topNWordsConfiguration.PoolNodeCount,
virtualMachineSize: "standard_d1_v2",
cloudServiceConfiguration: new CloudServiceConfiguration(osFamily: "6"));
Console.WriteLine("Adding pool {0}", topNWordsConfiguration.PoolId);
pool.TaskSchedulingPolicy = new TaskSchedulingPolicy(ComputeNodeFillType.Spread);
pool.MaxTasksPerComputeNode = 4;
GettingStartedCommon.CreatePoolIfNotExistAsync(client, pool).Wait();
var formula = @"startingNumberOfVMs = 2;
maxNumberofVMs = 4;
pendingTaskSamplePercent = $PendingTasks.GetSamplePercent(90 * TimeInterval_Second);
pendingTaskSamples = pendingTaskSamplePercent < 70 ? startingNumberOfVMs : avg($PendingTasks.GetSample(180 * TimeInterval_Second));
$TargetDedicatedNodes = min(maxNumberofVMs, pendingTaskSamples);
$NodeDeallocationOption = taskcompletion;";
var noOfSeconds = 150;
Thread.Sleep(noOfSeconds * 1000);
client.PoolOperations.EnableAutoScale(
poolId: topNWordsConfiguration.PoolId, autoscaleFormula: formula,
autoscaleEvaluationInterval: TimeSpan.FromMinutes(5));
try
{
Console.WriteLine("Creating job: " + topNWordsConfiguration.JobId);
// get an empty unbound Job
CloudJob unboundJob = client.JobOperations.CreateJob();
unboundJob.Id = topNWordsConfiguration.JobId;
unboundJob.PoolInformation = new PoolInformation()
{
PoolId = topNWordsConfiguration.PoolId
};
// Commit Job to create it in the service
unboundJob.Commit();
// create file staging objects that represent the executable and its dependent assembly to run as the task.
// These files are copied to every node before the corresponding task is scheduled to run on that node.
FileToStage topNWordExe = new FileToStage(TopNWordsExeName, stagingStorageAccount);
FileToStage storageDll = new FileToStage(StorageClientDllName, stagingStorageAccount);
FileToStage newtonJsoftDll = new FileToStage(NewtonJSoftDllName, stagingStorageAccount);
FileToStage microsoftEFDll = new FileToStage(MicrosoftEntityFrameworkDllName, stagingStorageAccount);
FileToStage microsoftEFCoreDll = new FileToStage(MicrosoftEntityFrameworkCoreDllName, stagingStorageAccount);
FileToStage microsoftBCLDll = new FileToStage(MicrosoftBCLDllName, stagingStorageAccount);
FileToStage systemTasksDll = new FileToStage(SystemTasksDllName, stagingStorageAccount);
FileToStage topNWordsConfigFile = new FileToStage(TopnWordsConfig, stagingStorageAccount);
FileToStage SystemValueTupleDll = new FileToStage(SystemValueTupleDllName, stagingStorageAccount);
FileToStage DependencyInjectionAbstractionsDll = new FileToStage(DependecyInjectionAbstractionsDllName, stagingStorageAccount);
FileToStage DependencyInjectionDll = new FileToStage(DependecyInjectionDllName, stagingStorageAccount);
FileToStage LoggingAbstractionsDll = new FileToStage(LoggingAbstractionsDllName, stagingStorageAccount);
FileToStage DiagnosticsDll = new FileToStage(DiagnosticssDllName, stagingStorageAccount);
FileToStage CachingAbstractionDll = new FileToStage(CachingAbstractionsDllName, stagingStorageAccount);
FileToStage MicrosoftSqlServerDll = new FileToStage(MicrosoftSqlServerDllName, stagingStorageAccount);
FileToStage SystemComponentDll = new FileToStage(SystemComponentDllName, stagingStorageAccount);
FileToStage SystemCollectionsDll = new FileToStage(SystemCollectionsDllName, stagingStorageAccount);
FileToStage pDll = new FileToStage(pdllName, stagingStorageAccount);
FileToStage oDll = new FileToStage(odllName, stagingStorageAccount);
FileToStage lDll = new FileToStage(ldllName, stagingStorageAccount);
FileToStage hashcodeDll = new FileToStage(hashcodeDllName, stagingStorageAccount);
FileToStage clientSqlDll = new FileToStage(clientSqlClientDllName, stagingStorageAccount);
FileToStage cachingMemoryDll = new FileToStage(CachingMemoryDllName, stagingStorageAccount);
FileToStage configAbstractionDll = new FileToStage(configAbstractionDllName, stagingStorageAccount);
FileToStage SNIDll = new FileToStage(SNIDllName, stagingStorageAccount);
FileToStage relationDll = new FileToStage(relationddllName, stagingStorageAccount);
var textFile = "E:\\WeatherAPIPOC\\cities_id.txt";
var text = File.ReadAllLines(textFile);
var cityList = new List <string>(text);
// In this sample, the input data is copied separately to Storage and its URI is passed to the task as an argument.
// This approach is appropriate when the amount of input data is large such that copying it to every node via FileStaging
// is not desired and the number of tasks is small since a large number of readers of the blob might get throttled
// by Storage which will lengthen the overall processing time.
//
// You'll need to observe the behavior and use published techniques for finding the right balance of performance versus
// complexity.
Console.WriteLine("{0} uploaded to cloud", topNWordsConfiguration.FileName);
// initialize a collection to hold the tasks that will be submitted in their entirety
List <CloudTask> tasksToRun = new List <CloudTask>(topNWordsConfiguration.NumberOfTasks);
for (int i = 0; i < cityList.Count; i++)
{
string programLaunchTime = DateTime.Now.ToString("h:mm:sstt");
CloudTask task = new CloudTask(
id: $"task_no_{i + 1}",
commandline: $"cmd /c mkdir x64 & move SNI.dll x64 & {TopNWordsExeName} --Task {cityList[i]} %AZ_BATCH_NODE_ID% {programLaunchTime}");
//This is the list of files to stage to a container -- for each job, one container is created and
//files all resolve to Azure Blobs by their name (so two tasks with the same named file will create just 1 blob in
//the container).
task.FilesToStage = new List <IFileStagingProvider>
{
topNWordExe,
storageDll,
newtonJsoftDll,
microsoftEFDll,
microsoftEFCoreDll,
microsoftBCLDll,
systemTasksDll,
topNWordsConfigFile,
SystemValueTupleDll,
DependencyInjectionAbstractionsDll,
DependencyInjectionDll,
LoggingAbstractionsDll,
DiagnosticsDll,
CachingAbstractionDll,
MicrosoftSqlServerDll,
SystemComponentDll,
SystemCollectionsDll,
oDll,
pDll,
lDll,
relationDll,
hashcodeDll,
clientSqlDll,
cachingMemoryDll,
configAbstractionDll,
SNIDll
};
tasksToRun.Add(task);
}
// Commit all the tasks to the Batch Service. Ask AddTask to return information about the files that were staged.
// The container information is used later on to remove these files from Storage.
ConcurrentBag <ConcurrentDictionary <Type, IFileStagingArtifact> > fsArtifactBag = new ConcurrentBag <ConcurrentDictionary <Type, IFileStagingArtifact> >();
client.JobOperations.AddTask(topNWordsConfiguration.JobId, tasksToRun, fileStagingArtifacts: fsArtifactBag);
// loop through the bag of artifacts, looking for the one that matches our staged files. Once there,
// capture the name of the container holding the files so they can be deleted later on if that option
// was configured in the settings.
foreach (var fsBagItem in fsArtifactBag)
{
IFileStagingArtifact fsValue;
if (fsBagItem.TryGetValue(typeof(FileToStage), out fsValue))
{
SequentialFileStagingArtifact stagingArtifact = fsValue as SequentialFileStagingArtifact;
if (stagingArtifact != null)
{
stagingContainer = stagingArtifact.BlobContainerCreated;
Console.WriteLine(
"Uploaded files to container: {0} -- you will be charged for their storage unless you delete them.",
stagingArtifact.BlobContainerCreated);
}
}
}
//Get the job to monitor status.
CloudJob job = client.JobOperations.GetJob(topNWordsConfiguration.JobId);
Console.Write("Waiting for tasks to complete ... ");
// Wait 20 minutes for all tasks to reach the completed state. The long timeout is necessary for the first
// time a pool is created in order to allow nodes to be added to the pool and initialized to run tasks.
IPagedEnumerable <CloudTask> ourTasks = job.ListTasks(new ODATADetailLevel(selectClause: "id"));
client.Utilities.CreateTaskStateMonitor().WaitAll(ourTasks, TaskState.Completed, TimeSpan.FromMinutes(20));
Console.WriteLine("tasks are done.");
foreach (CloudTask t in ourTasks)
{
Console.WriteLine("Task " + t.Id);
Console.WriteLine("stdout:" + Environment.NewLine + t.GetNodeFile(Batch.Constants.StandardOutFileName).ReadAsString());
Console.WriteLine();
Console.WriteLine("stderr:" + Environment.NewLine + t.GetNodeFile(Batch.Constants.StandardErrorFileName).ReadAsString());
}
}
finally
{
//Delete the pool that we created
if (topNWordsConfiguration.ShouldDeletePool)
{
Console.WriteLine("Deleting pool: {0}", topNWordsConfiguration.PoolId);
client.PoolOperations.DeletePool(topNWordsConfiguration.PoolId);
}
//Delete the job that we created
if (topNWordsConfiguration.ShouldDeleteJob)
{
Console.WriteLine("Deleting job: {0}", topNWordsConfiguration.JobId);
client.JobOperations.DeleteJob(topNWordsConfiguration.JobId);
}
//Delete the containers we created
if (topNWordsConfiguration.ShouldDeleteContainer)
{
DeleteContainers(accountSettings, stagingContainer);
}
}
}
}
public static void JobMain(string[] args)
{
//Load the configuration
Settings topNWordsConfiguration = new ConfigurationBuilder()
.SetBasePath(Directory.GetCurrentDirectory())
.AddJsonFile("settings.json")
.Build()
.Get <Settings>();
AccountSettings accountSettings = SampleHelpers.LoadAccountSettings();
CloudStorageAccount cloudStorageAccount = new CloudStorageAccount(
new StorageCredentials(
accountSettings.StorageAccountName,
accountSettings.StorageAccountKey),
accountSettings.StorageServiceUrl,
useHttps: true);
StagingStorageAccount stagingStorageAccount = new StagingStorageAccount(
accountSettings.StorageAccountName,
accountSettings.StorageAccountKey,
cloudStorageAccount.BlobEndpoint.ToString());
using (BatchClient client = BatchClient.Open(new BatchSharedKeyCredentials(accountSettings.BatchServiceUrl, accountSettings.BatchAccountName, accountSettings.BatchAccountKey)))
{
string stagingContainer = null;
//OSFamily 5 == Windows 2016. You can learn more about os families and versions at:
//http://msdn.microsoft.com/en-us/library/azure/ee924680.aspx
CloudPool pool = client.PoolOperations.CreatePool(
topNWordsConfiguration.PoolId,
targetDedicatedComputeNodes: topNWordsConfiguration.PoolNodeCount,
virtualMachineSize: "standard_d1_v2",
cloudServiceConfiguration: new CloudServiceConfiguration(osFamily: "5"));
Console.WriteLine("Adding pool {0}", topNWordsConfiguration.PoolId);
GettingStartedCommon.CreatePoolIfNotExistAsync(client, pool).Wait();
try
{
Console.WriteLine("Creating job: " + topNWordsConfiguration.JobId);
// get an empty unbound Job
CloudJob unboundJob = client.JobOperations.CreateJob();
unboundJob.Id = topNWordsConfiguration.JobId;
unboundJob.PoolInformation = new PoolInformation()
{
PoolId = topNWordsConfiguration.PoolId
};
// Commit Job to create it in the service
unboundJob.Commit();
// create file staging objects that represent the executable and its dependent assembly to run as the task.
// These files are copied to every node before the corresponding task is scheduled to run on that node.
FileToStage topNWordExe = new FileToStage(TopNWordsExeName, stagingStorageAccount);
FileToStage storageDll = new FileToStage(StorageClientDllName, stagingStorageAccount);
// In this sample, the input data is copied separately to Storage and its URI is passed to the task as an argument.
// This approach is appropriate when the amount of input data is large such that copying it to every node via FileStaging
// is not desired and the number of tasks is small since a large number of readers of the blob might get throttled
// by Storage which will lengthen the overall processing time.
//
// You'll need to observe the behavior and use published techniques for finding the right balance of performance versus
// complexity.
string bookFileUri = UploadBookFileToCloudBlob(accountSettings, topNWordsConfiguration.FileName);
Console.WriteLine("{0} uploaded to cloud", topNWordsConfiguration.FileName);
// initialize a collection to hold the tasks that will be submitted in their entirety
List <CloudTask> tasksToRun = new List <CloudTask>(topNWordsConfiguration.NumberOfTasks);
for (int i = 1; i <= topNWordsConfiguration.NumberOfTasks; i++)
{
CloudTask task = new CloudTask("task_no_" + i, string.Format("{0} --Task {1} {2} {3} {4}",
TopNWordsExeName,
bookFileUri,
topNWordsConfiguration.TopWordCount,
accountSettings.StorageAccountName,
accountSettings.StorageAccountKey));
//This is the list of files to stage to a container -- for each job, one container is created and
//files all resolve to Azure Blobs by their name (so two tasks with the same named file will create just 1 blob in
//the container).
task.FilesToStage = new List <IFileStagingProvider>
{
topNWordExe,
storageDll
};
tasksToRun.Add(task);
}
// Commit all the tasks to the Batch Service. Ask AddTask to return information about the files that were staged.
// The container information is used later on to remove these files from Storage.
ConcurrentBag <ConcurrentDictionary <Type, IFileStagingArtifact> > fsArtifactBag = new ConcurrentBag <ConcurrentDictionary <Type, IFileStagingArtifact> >();
client.JobOperations.AddTask(topNWordsConfiguration.JobId, tasksToRun, fileStagingArtifacts: fsArtifactBag);
// loop through the bag of artifacts, looking for the one that matches our staged files. Once there,
// capture the name of the container holding the files so they can be deleted later on if that option
// was configured in the settings.
foreach (var fsBagItem in fsArtifactBag)
{
IFileStagingArtifact fsValue;
if (fsBagItem.TryGetValue(typeof(FileToStage), out fsValue))
{
SequentialFileStagingArtifact stagingArtifact = fsValue as SequentialFileStagingArtifact;
if (stagingArtifact != null)
{
stagingContainer = stagingArtifact.BlobContainerCreated;
Console.WriteLine(
"Uploaded files to container: {0} -- you will be charged for their storage unless you delete them.",
stagingArtifact.BlobContainerCreated);
}
}
}
//Get the job to monitor status.
CloudJob job = client.JobOperations.GetJob(topNWordsConfiguration.JobId);
Console.Write("Waiting for tasks to complete ... ");
// Wait 20 minutes for all tasks to reach the completed state. The long timeout is necessary for the first
// time a pool is created in order to allow nodes to be added to the pool and initialized to run tasks.
IPagedEnumerable <CloudTask> ourTasks = job.ListTasks(new ODATADetailLevel(selectClause: "id"));
client.Utilities.CreateTaskStateMonitor().WaitAll(ourTasks, TaskState.Completed, TimeSpan.FromMinutes(20));
Console.WriteLine("tasks are done.");
foreach (CloudTask t in ourTasks)
{
Console.WriteLine("Task " + t.Id);
Console.WriteLine("stdout:" + Environment.NewLine + t.GetNodeFile(Batch.Constants.StandardOutFileName).ReadAsString());
Console.WriteLine();
Console.WriteLine("stderr:" + Environment.NewLine + t.GetNodeFile(Batch.Constants.StandardErrorFileName).ReadAsString());
}
}
finally
{
//Delete the pool that we created
if (topNWordsConfiguration.ShouldDeletePool)
{
Console.WriteLine("Deleting pool: {0}", topNWordsConfiguration.PoolId);
client.PoolOperations.DeletePool(topNWordsConfiguration.PoolId);
}
//Delete the job that we created
if (topNWordsConfiguration.ShouldDeleteJob)
{
Console.WriteLine("Deleting job: {0}", topNWordsConfiguration.JobId);
client.JobOperations.DeleteJob(topNWordsConfiguration.JobId);
}
//Delete the containers we created
if (topNWordsConfiguration.ShouldDeleteContainer)
{
DeleteContainers(accountSettings, stagingContainer);
}
}
}
}