/// <summary> /// Creates a pool if it doesn't already exist. If the pool already exists, this method resizes it to meet the expected /// targets specified in settings. /// </summary> /// <param name="batchClient">The BatchClient to use when interacting with the Batch service.</param> /// <param name="cloudStorageAccount">The CloudStorageAccount to upload start task required files to.</param> /// <returns>An asynchronous <see cref="Task"/> representing the operation.</returns> private async Task CreatePoolIfNotExistAsync(BatchClient batchClient, CloudStorageAccount cloudStorageAccount) { // You can learn more about os families and versions at: // https://azure.microsoft.com/en-us/documentation/articles/cloud-services-guestos-update-matrix/ CloudPool pool = batchClient.PoolOperations.CreatePool( poolId: this.jobManagerSettings.PoolId, targetDedicated: this.jobManagerSettings.PoolTargetNodeCount, virtualMachineSize: this.jobManagerSettings.PoolNodeVirtualMachineSize, cloudServiceConfiguration: new CloudServiceConfiguration(this.jobManagerSettings.PoolOSFamily)); // Create a new start task to facilitate pool-wide file management or installation. // In this case, we just add a single dummy data file to the StartTask. string localSampleFilePath = GettingStartedCommon.GenerateTemporaryFile("StartTask.txt", "hello from Batch JobManager sample!"); List <string> files = new List <string> { localSampleFilePath }; List <ResourceFile> resourceFiles = await SampleHelpers.UploadResourcesAndCreateResourceFileReferencesAsync( cloudStorageAccount, this.jobManagerSettings.BlobContainer, files); pool.StartTask = new StartTask() { CommandLine = "cmd /c dir", ResourceFiles = resourceFiles }; await GettingStartedCommon.CreatePoolIfNotExistAsync(batchClient, pool); }
/// <summary> /// Creates a job and adds a task to it. The task is a /// custom executable which has a resource file associated with it. /// </summary> /// <param name="batchClient">The BatchClient to use when interacting with the Batch service.</param> /// <param name="storageAccount">The cloud storage account to upload files to.</param> /// <param name="jobId">The ID of the job.</param> /// <returns>An asynchronous <see cref="Task"/> representing the operation.</returns> private async Task SubmitJobAsync(BatchClient batchClient, CloudStorageAccount storageAccount, string jobId) { // create an empty unbound Job CloudJob unboundJob = batchClient.JobOperations.CreateJob(); unboundJob.Id = jobId; unboundJob.PoolInformation = new PoolInformation() { PoolId = this.jobManagerSettings.PoolId }; // Upload the required files for the job manager task await SampleHelpers.UploadResourcesAsync(storageAccount, this.jobManagerSettings.BlobContainer, JobManagerRequiredFiles); List <ResourceFile> jobManagerResourceFiles = await SampleHelpers.UploadResourcesAndCreateResourceFileReferencesAsync( storageAccount, this.jobManagerSettings.BlobContainer, JobManagerRequiredFiles); // Set up the JobManager environment settings List <EnvironmentSetting> jobManagerEnvironmentSettings = new List <EnvironmentSetting>() { // No need to pass the batch account name as an environment variable since the batch service provides // an environment variable for each task which contains the account name new EnvironmentSetting("SAMPLE_BATCH_KEY", this.accountSettings.BatchAccountKey), new EnvironmentSetting("SAMPLE_BATCH_URL", this.accountSettings.BatchServiceUrl), new EnvironmentSetting("SAMPLE_STORAGE_ACCOUNT", this.accountSettings.StorageAccountName), new EnvironmentSetting("SAMPLE_STORAGE_KEY", this.accountSettings.StorageAccountKey), new EnvironmentSetting("SAMPLE_STORAGE_URL", this.accountSettings.StorageServiceUrl), }; unboundJob.JobManagerTask = new JobManagerTask() { Id = JobManagerTaskId, CommandLine = JobManagerTaskExe, ResourceFiles = jobManagerResourceFiles, KillJobOnCompletion = true, EnvironmentSettings = jobManagerEnvironmentSettings }; // Commit Job to create it in the service await unboundJob.CommitAsync(); }
public async static Task JobMain(string[] args) { //Load the configuration Settings topNWordsConfiguration = Settings.Default; AccountSettings accountSettings = AccountSettings.Default; CloudStorageAccount cloudStorageAccount = new CloudStorageAccount( new StorageCredentials( accountSettings.StorageAccountName, accountSettings.StorageAccountKey), accountSettings.StorageServiceUrl, useHttps: true); StagingStorageAccount stagingStorageAccount = new StagingStorageAccount( accountSettings.StorageAccountName, accountSettings.StorageAccountKey, cloudStorageAccount.BlobEndpoint.ToString()); using (BatchClient client = BatchClient.Open(new BatchSharedKeyCredentials(accountSettings.BatchServiceUrl, accountSettings.BatchAccountName, accountSettings.BatchAccountKey))) { string stagingContainer = null; //OSFamily 4 == OS 2012 R2. You can learn more about os families and versions at: //http://msdn.microsoft.com/en-us/library/azure/ee924680.aspx CloudPool pool = client.PoolOperations.CreatePool( topNWordsConfiguration.PoolId, targetDedicatedComputeNodes: topNWordsConfiguration.PoolNodeCount, virtualMachineSize: "small", cloudServiceConfiguration: new CloudServiceConfiguration(osFamily: "4")); List <string> files = new List <string> { Path.Combine(BatchStartTaskFolderName, BatchStartTaskTelemetryRunnerName), }; files.AddRange(AIFilesToUpload); var resourceHelperTask = SampleHelpers.UploadResourcesAndCreateResourceFileReferencesAsync( cloudStorageAccount, AIBlobConatinerName, files); List <ResourceFile> resourceFiles = resourceHelperTask.Result; pool.StartTask = new StartTask() { CommandLine = string.Format("cmd /c {0}", BatchStartTaskTelemetryRunnerName), ResourceFiles = resourceFiles }; Console.WriteLine("Adding pool {0}", topNWordsConfiguration.PoolId); try { await GettingStartedCommon.CreatePoolIfNotExistAsync(client, pool); } catch (AggregateException ae) { // Go through all exceptions and dump useful information ae.Handle(x => { Console.Error.WriteLine("Creating pool ID {0} failed", topNWordsConfiguration.PoolId); if (x is BatchException) { BatchException be = x as BatchException; Console.WriteLine(be.ToString()); Console.WriteLine(); } else { Console.WriteLine(x); } // can't continue without a pool return(false); }); } catch (BatchException be) { Console.Error.WriteLine("Creating pool ID {0} failed", topNWordsConfiguration.PoolId); Console.WriteLine(be.ToString()); Console.WriteLine(); } try { Console.WriteLine("Creating job: " + topNWordsConfiguration.JobId); // get an empty unbound Job CloudJob unboundJob = client.JobOperations.CreateJob(); unboundJob.Id = topNWordsConfiguration.JobId; unboundJob.PoolInformation = new PoolInformation() { PoolId = topNWordsConfiguration.PoolId }; // Commit Job to create it in the service await unboundJob.CommitAsync(); // create file staging objects that represent the executable and its dependent assembly to run as the task. // These files are copied to every node before the corresponding task is scheduled to run on that node. FileToStage topNWordExe = new FileToStage(TopNWordsExeName, stagingStorageAccount); FileToStage storageDll = new FileToStage(StorageClientDllName, stagingStorageAccount); // Upload application insights assemblies List <FileToStage> aiStagedFiles = new List <FileToStage>(); foreach (string aiFile in AIFilesToUpload) { aiStagedFiles.Add(new FileToStage(aiFile, stagingStorageAccount)); } // In this sample, the input data is copied separately to Storage and its URI is passed to the task as an argument. // This approach is appropriate when the amount of input data is large such that copying it to every node via FileStaging // is not desired and the number of tasks is small since a large number of readers of the blob might get throttled // by Storage which will lengthen the overall processing time. // // You'll need to observe the behavior and use published techniques for finding the right balance of performance versus // complexity. string[] documents = Directory.GetFiles(topNWordsConfiguration.DocumentsRootPath); await SampleHelpers.UploadResourcesAsync(cloudStorageAccount, BooksContainerName, documents); // initialize a collection to hold the tasks that will be submitted in their entirety List <CloudTask> tasksToRun = new List <CloudTask>(documents.Length); for (int i = 0; i < documents.Length; i++) { CloudTask task = new CloudTask("task_no_" + i, String.Format("{0} --Task {1} {2} {3} {4}", TopNWordsExeName, "https://onbehalfoutput.blob.core.windows.net/" + documents[i], topNWordsConfiguration.TopWordCount, accountSettings.StorageAccountName, accountSettings.StorageAccountKey)); //This is the list of files to stage to a container -- for each job, one container is created and //files all resolve to Azure Blobs by their name (so two tasks with the same named file will create just 1 blob in //the container). task.FilesToStage = new List <IFileStagingProvider> { topNWordExe, storageDll, }; foreach (FileToStage stagedFile in aiStagedFiles) { task.FilesToStage.Add(stagedFile); } tasksToRun.Add(task); } // Commit all the tasks to the Batch Service. Ask AddTask to return information about the files that were staged. // The container information is used later on to remove these files from Storage. ConcurrentBag <ConcurrentDictionary <Type, IFileStagingArtifact> > fsArtifactBag = new ConcurrentBag <ConcurrentDictionary <Type, IFileStagingArtifact> >(); client.JobOperations.AddTask(topNWordsConfiguration.JobId, tasksToRun, fileStagingArtifacts: fsArtifactBag); // loop through the bag of artifacts, looking for the one that matches our staged files. Once there, // capture the name of the container holding the files so they can be deleted later on if that option // was configured in the settings. foreach (var fsBagItem in fsArtifactBag) { IFileStagingArtifact fsValue; if (fsBagItem.TryGetValue(typeof(FileToStage), out fsValue)) { SequentialFileStagingArtifact stagingArtifact = fsValue as SequentialFileStagingArtifact; if (stagingArtifact != null) { stagingContainer = stagingArtifact.BlobContainerCreated; Console.WriteLine( "Uploaded files to container: {0} -- you will be charged for their storage unless you delete them.", stagingArtifact.BlobContainerCreated); } } } //Get the job to monitor status. CloudJob job = client.JobOperations.GetJob(topNWordsConfiguration.JobId); Console.Write("Waiting for tasks to complete ... "); // Wait 20 minutes for all tasks to reach the completed state. The long timeout is necessary for the first // time a pool is created in order to allow nodes to be added to the pool and initialized to run tasks. IPagedEnumerable <CloudTask> ourTasks = job.ListTasks(new ODATADetailLevel(selectClause: "id")); client.Utilities.CreateTaskStateMonitor().WaitAll(ourTasks, TaskState.Completed, TimeSpan.FromMinutes(20)); Console.WriteLine("tasks are done."); foreach (CloudTask t in ourTasks) { Console.WriteLine("Task " + t.Id); Console.WriteLine("stdout:" + Environment.NewLine + t.GetNodeFile(Batch.Constants.StandardOutFileName).ReadAsString()); Console.WriteLine(); Console.WriteLine("stderr:" + Environment.NewLine + t.GetNodeFile(Batch.Constants.StandardErrorFileName).ReadAsString()); } } finally { //Delete the pool that we created if (topNWordsConfiguration.ShouldDeletePool) { Console.WriteLine("Deleting pool: {0}", topNWordsConfiguration.PoolId); client.PoolOperations.DeletePool(topNWordsConfiguration.PoolId); } //Delete the job that we created if (topNWordsConfiguration.ShouldDeleteJob) { Console.WriteLine("Deleting job: {0}", topNWordsConfiguration.JobId); client.JobOperations.DeleteJob(topNWordsConfiguration.JobId); } //Delete the containers we created if (topNWordsConfiguration.ShouldDeleteContainer) { DeleteContainers(accountSettings, stagingContainer); } } } }