public static void CreateBatchTasks(string action, string jobId, string containerUrl, string batchAccountUrl, string sqlServerName, string accessToken, dynamic databases, ILogger log) { // Get a Batch client using function identity log.LogInformation("CreateBatchTasks: entering"); var azureServiceTokenProvider = new AzureServiceTokenProvider(); BatchTokenCredentials batchCred = new BatchTokenCredentials(batchAccountUrl, azureServiceTokenProvider.GetAccessTokenAsync("https://batch.core.windows.net/").Result); using (BatchClient batchClient = BatchClient.Open(batchCred)) { // For each database, submit the Exporting job to Azure Batch Compute Pool. log.LogInformation("CreateBatchTasks: enumerating databases"); List <CloudTask> tasks = new List <CloudTask>(); foreach (var db in databases) { string serverDatabaseName = db.name.ToString(); string logicalDatabase = serverDatabaseName.Remove(0, sqlServerName.Length + 1); log.LogInformation("CreateBatchTasks: creating task for database {0}", logicalDatabase); string taskId = sqlServerName + "_" + logicalDatabase; string command = string.Format("cmd /c %AZ_BATCH_APP_PACKAGE_{0}#{1}%\\BatchWrapper {2}", AppPackageName.ToUpper(), AppPackageVersion, action); command += string.Format(" {0} {1} {2} {3} {4}", sqlServerName, logicalDatabase, accessToken, AppPackageName.ToUpper(), AppPackageVersion); string taskCommandLine = string.Format(command); CloudTask singleTask = new CloudTask(taskId, taskCommandLine); singleTask.EnvironmentSettings = new[] { new EnvironmentSetting("JOB_CONTAINER_URL", containerUrl) }; Console.WriteLine(string.Format("Adding task {0} to job ...", taskId)); tasks.Add(singleTask); } // Add all tasks to the job. batchClient.JobOperations.AddTask(jobId, tasks); } log.LogInformation("CreateBatchTasks: exiting"); }
private void InitEngine() { Console.WriteLine("Initializing batch engine: {0}", DateTime.Now); // Get a Batch client using account creds // BatchSharedKeyCredentials cred = // new BatchSharedKeyCredentials(BatchAccountUrl, BatchAccountName, BatchAccountKey); Func <Task <string> > tokenProvider = async() => await GetAuthenticationTokenAsync( AzureEnvConstants.AZURE_AD_TOKEN_EP + EnvVars[AzureEnvConstants.AZURE_AD_TENANT_ID], EnvVars[AzureEnvConstants.AZURE_AD_SP_CLIENT_ID], EnvVars[AzureEnvConstants.AZURE_AD_SP_CLIENT_SECRET]); Creds = new BatchTokenCredentials(EnvVars[AzureEnvConstants.AZURE_BATCH_ACCOUNT_URL], tokenProvider); }
public static async Task <string> CreateBatchPoolAndImportJob([ActivityTrigger] ImportRequest request, ILogger log) { var azureServiceTokenProvider = new AzureServiceTokenProvider(); // Get a Batch client using function identity BatchTokenCredentials batchCred = new BatchTokenCredentials(request.BatchAccountUrl, await azureServiceTokenProvider.GetAccessTokenAsync("https://batch.core.windows.net/")); string jobId = request.TargetSqlServerName + "-Import-" + DateTime.UtcNow.ToString("MMddHHmmss"); using (BatchClient batchClient = BatchClient.Open(batchCred)) { ImageReference imageReference = CreateImageReference(); VirtualMachineConfiguration vmConfiguration = CreateVirtualMachineConfiguration(imageReference); await CreateBatchPoolIfNotExist(batchClient, vmConfiguration, request.VNetSubnetId); await CreateBatchJob(batchClient, jobId, log); } return(jobId); }
public void Execute() { string reqfile = _config.ResourceFolder + _config.ResourceName; //> CREATE try { ResourceFile req = CreateResourceFile(reqfile); BatchTokenCredentials cred = new BatchTokenCredentials(_config.BatchAccountUrl, GetToken()); Client = BatchClient.Open(cred); ImageReference imgref = new ImageReference(_config.VmImage); VirtualMachineConfiguration vmc = new VirtualMachineConfiguration(imageReference: imgref, nodeAgentSkuId: "batch.node.windows amd64"); CreateBatchPool(vmc, _config.VmSKU); // Create Job CloudJob job = Client.JobOperations.CreateJob(); job.Id = _jobId; job.PoolInformation = new PoolInformation { PoolId = _poolId }; job.Commit(); // Create Tasks List <CloudTask> tasks = new List <CloudTask>(); string taskId = "task" + System.DateTime.Now.Ticks.ToString(); string cmd = String.Format("cmd /c type {0}", req.FilePath); CloudTask task = new CloudTask(taskId, cmd); task.ResourceFiles = new List <ResourceFile> { req }; tasks.Add(task); Client.JobOperations.AddTask(_jobId, tasks); } catch (Exception x) { //TODO: Log Somewhere throw; } }
static void Main(string[] args) { string storageConnectionString = $"DefaultEndpointsProtocol=https;AccountName={StorageAccountName};AccountKey={StorageAccountKey}"; // Retrieve the storage account CloudStorageAccount storageAccount = CloudStorageAccount.Parse(storageConnectionString); // Create the blob client CloudBlobClient blobClient = storageAccount.CreateCloudBlobClient(); const string inputContainerName = "batchinput"; List <string> inputFilePaths = new List <string> { "taskdata0.txt", "taskdata1.txt", "taskdata2.txt" }; // Upload the input files to blob storage List <ResourceFile> inputFiles = new List <ResourceFile>(); foreach (string filePath in inputFilePaths) { inputFiles.Add(UploadFileToContainer(blobClient, inputContainerName, filePath)); } // Get a SAS Url for the output container const string outputContainerName = "batchoutput"; string outputContainerSasUrl = GetOutputContainerSasUrl(blobClient, outputContainerName); // Retrieve an access token from Azure AD to authenticate with the Azure Batch API AuthenticationContext authContext = new AuthenticationContext(AuthorityUri); AuthenticationResult authResult = authContext.AcquireTokenAsync(BatchResourceUri, new ClientCredential(ClientId, ClientKey)).Result; BatchTokenCredentials batchTokenCredentials = new BatchTokenCredentials(BatchAccountUrl, authResult.AccessToken); using (BatchClient batchClient = BatchClient.Open(batchTokenCredentials)) { Console.WriteLine("Creating job [{0}]...", JobId); try { CloudJob job = batchClient.JobOperations.CreateJob(); job.Id = JobId; job.PoolInformation = new PoolInformation { PoolId = PoolId }; job.Commit(); } catch (BatchException be) { // Accept the specific error code JobExists as that is expected if the job already exists if (be.RequestInformation?.BatchError?.Code == BatchErrorCodeStrings.JobExists) { Console.WriteLine("The job {0} already existed when we tried to create it", JobId); } else { throw; // Any other exception is unexpected } } // Create a collection to hold the tasks that we'll be adding to the job Console.WriteLine("Adding {0} tasks to job [{1}]...", inputFiles.Count, JobId); List <CloudTask> tasks = new List <CloudTask>(); // Create each of the tasks to process one of the input files. for (int i = 0; i < inputFiles.Count; i++) { string taskId = String.Format("Task{0}", i); string inputFilename = inputFiles[i].FilePath; string outputFileName = string.Format("out{0}", inputFilename); string taskCommandLine = string.Format("cmd /c %AZ_BATCH_APP_PACKAGE_READWRITEFILE%\\ReadWriteFile.exe {0} {1}", inputFilename, outputFileName); CloudTask task = new CloudTask(taskId, taskCommandLine); // Set the resource files and output files for the task task.ResourceFiles = new List <ResourceFile> { inputFiles[i] }; task.OutputFiles = new List <OutputFile> { new OutputFile( filePattern: outputFileName, destination: new OutputFileDestination(new OutputFileBlobContainerDestination(containerUrl: outputContainerSasUrl, path: outputFileName)), uploadOptions: new OutputFileUploadOptions(OutputFileUploadCondition.TaskCompletion)) }; tasks.Add(task); } // Add all tasks to the job. batchClient.JobOperations.AddTask(JobId, tasks); // Monitor task success/failure, specifying a maximum amount of time to wait for the tasks to complete. TimeSpan timeout = TimeSpan.FromMinutes(30); Console.WriteLine("Monitoring all tasks for 'Completed' state, timeout in {0}...", timeout); IEnumerable <CloudTask> addedTasks = batchClient.JobOperations.ListTasks(JobId); batchClient.Utilities.CreateTaskStateMonitor().WaitAll(addedTasks, TaskState.Completed, timeout); Console.WriteLine("All tasks reached state Completed."); // Print task output Console.WriteLine(); Console.WriteLine("Printing task output..."); IEnumerable <CloudTask> completedtasks = batchClient.JobOperations.ListTasks(JobId); foreach (CloudTask task in completedtasks) { string nodeId = String.Format(task.ComputeNodeInformation.ComputeNodeId); Console.WriteLine("Task: {0}", task.Id); Console.WriteLine("Node: {0}", nodeId); Console.WriteLine("Standard out:"); Console.WriteLine(task.GetNodeFile(Constants.StandardOutFileName).ReadAsString()); } // Clean up Batch resources (if the user so chooses) Console.WriteLine(); Console.Write("Delete job? [yes] no: "); string response = Console.ReadLine().ToLower(); if (response != "n" && response != "no") { batchClient.JobOperations.DeleteJob(JobId); } } }
protected override void OnExpand() { int requirement = int.Parse(RuntimeSettings[gCEL.Core.Components.FABRIC.Constants.CAPACITY_SETTING_REQUIREMENT]); int workers = int.Parse(Settings[SETTING_WORKERS]); PoolSize = (requirement / workers) + ((requirement % workers) > 0 ? 1 : 0); if (PoolSize == 0) { PoolSize = 1; } string reqfile = gCEL.Core.Utils.FileSystem.GetTempPath() + "gcel_request_" + gCEL.Core.Utils.Environment.UniqueIdentifier + ".xml"; //> CREATE try { Stopwatch sw = new Stopwatch(); sw.Start(); ResourceFile req = CreateResourceFile(reqfile); //BatchSharedKeyCredentials cred = new BatchSharedKeyCredentials(Settings[SETTING_BATCH_ACCOUNT_URL], Settings[SETTING_BATCH_ACCOUNT_NAME], Settings[SETTING_BATCH_ACCOUNT_KEY]); BatchTokenCredentials cred = new BatchTokenCredentials(Settings[SETTING_BATCH_ACCOUNT_URL], GetToken()); Client = BatchClient.Open(cred); //> VM ImageReference imgref = new ImageReference(Settings[SETTING_BATCH_CUSTOM_IMAGE]); VirtualMachineConfiguration vmc = new VirtualMachineConfiguration(imageReference: imgref, nodeAgentSkuId: "batch.node.windows amd64"); //> POOL CreateBatchPool(vmc); //> JOB CloudJob job = Client.JobOperations.CreateJob(); job.Id = JobId; job.PoolInformation = new PoolInformation { PoolId = PoolId }; job.Commit(); //> TASKS List <CloudTask> tasks = new List <CloudTask>(); for (int i = 0; i < requirement; i++) { string taskId = "gCELTask" + i; string engine = Settings[SETTING_SYSTEM_FOLDER] + gCEL.Core.Constants.ENGINE_CMD; string cmd = String.Format("cmd /c {0} command=host component=engine request={1}", engine, req.FilePath); CloudTask task = new CloudTask(taskId, cmd); task.ResourceFiles = new List <ResourceFile> { req }; tasks.Add(task); } Client.JobOperations.AddTask(JobId, tasks); //> sw.Stop(); TimeSpan ts = sw.Elapsed; string elapsedTime = String.Format("{0:00}:{1:00}:{2:00}.{3:00}", ts.Hours, ts.Minutes, ts.Seconds, ts.Milliseconds / 10); Logger.Info(Tag + ":POOLCREATED:" + elapsedTime); } catch (Exception x) { Logger.Error(Tag + ":" + x.Message); throw; } finally { gCEL.Core.Utils.FileSystem.SafeDelete(reqfile); OnContract(); } Logger.Info(Tag + ":ALLOCATED:" + PoolSize); }