Exemplo n.º 1
0
        public static void CreateBatchTasks(string action, string jobId, string containerUrl, string batchAccountUrl, string sqlServerName, string accessToken, dynamic databases, ILogger log)
        {
            // Get a Batch client using function identity
            log.LogInformation("CreateBatchTasks: entering");
            var azureServiceTokenProvider   = new AzureServiceTokenProvider();
            BatchTokenCredentials batchCred = new BatchTokenCredentials(batchAccountUrl, azureServiceTokenProvider.GetAccessTokenAsync("https://batch.core.windows.net/").Result);

            using (BatchClient batchClient = BatchClient.Open(batchCred))
            {
                // For each database, submit the Exporting job to Azure Batch Compute Pool.
                log.LogInformation("CreateBatchTasks: enumerating databases");
                List <CloudTask> tasks = new List <CloudTask>();
                foreach (var db in databases)
                {
                    string serverDatabaseName = db.name.ToString();
                    string logicalDatabase    = serverDatabaseName.Remove(0, sqlServerName.Length + 1);

                    log.LogInformation("CreateBatchTasks: creating task for database {0}", logicalDatabase);
                    string taskId  = sqlServerName + "_" + logicalDatabase;
                    string command = string.Format("cmd /c %AZ_BATCH_APP_PACKAGE_{0}#{1}%\\BatchWrapper {2}", AppPackageName.ToUpper(), AppPackageVersion, action);
                    command += string.Format(" {0} {1} {2} {3} {4}", sqlServerName, logicalDatabase, accessToken, AppPackageName.ToUpper(), AppPackageVersion);
                    string taskCommandLine = string.Format(command);

                    CloudTask singleTask = new CloudTask(taskId, taskCommandLine);
                    singleTask.EnvironmentSettings = new[] { new EnvironmentSetting("JOB_CONTAINER_URL", containerUrl) };

                    Console.WriteLine(string.Format("Adding task {0} to job ...", taskId));
                    tasks.Add(singleTask);
                }

                // Add all tasks to the job.
                batchClient.JobOperations.AddTask(jobId, tasks);
            }
            log.LogInformation("CreateBatchTasks: exiting");
        }
        private void InitEngine()
        {
            Console.WriteLine("Initializing batch engine: {0}", DateTime.Now);

            // Get a Batch client using account creds
            // BatchSharedKeyCredentials cred =
            // new BatchSharedKeyCredentials(BatchAccountUrl, BatchAccountName, BatchAccountKey);
            Func <Task <string> > tokenProvider =
                async() => await GetAuthenticationTokenAsync(
                    AzureEnvConstants.AZURE_AD_TOKEN_EP + EnvVars[AzureEnvConstants.AZURE_AD_TENANT_ID],
                    EnvVars[AzureEnvConstants.AZURE_AD_SP_CLIENT_ID],
                    EnvVars[AzureEnvConstants.AZURE_AD_SP_CLIENT_SECRET]);

            Creds =
                new BatchTokenCredentials(EnvVars[AzureEnvConstants.AZURE_BATCH_ACCOUNT_URL], tokenProvider);
        }
Exemplo n.º 3
0
        public static async Task <string> CreateBatchPoolAndImportJob([ActivityTrigger] ImportRequest request, ILogger log)
        {
            var azureServiceTokenProvider = new AzureServiceTokenProvider();

            // Get a Batch client using function identity
            BatchTokenCredentials batchCred = new BatchTokenCredentials(request.BatchAccountUrl, await azureServiceTokenProvider.GetAccessTokenAsync("https://batch.core.windows.net/"));

            string jobId = request.TargetSqlServerName + "-Import-" + DateTime.UtcNow.ToString("MMddHHmmss");

            using (BatchClient batchClient = BatchClient.Open(batchCred))
            {
                ImageReference imageReference = CreateImageReference();
                VirtualMachineConfiguration vmConfiguration = CreateVirtualMachineConfiguration(imageReference);

                await CreateBatchPoolIfNotExist(batchClient, vmConfiguration, request.VNetSubnetId);
                await CreateBatchJob(batchClient, jobId, log);
            }

            return(jobId);
        }
Exemplo n.º 4
0
        public void Execute()
        {
            string reqfile = _config.ResourceFolder + _config.ResourceName;

            //> CREATE
            try
            {
                ResourceFile req = CreateResourceFile(reqfile);

                BatchTokenCredentials cred = new BatchTokenCredentials(_config.BatchAccountUrl, GetToken());
                Client = BatchClient.Open(cred);
                ImageReference imgref           = new ImageReference(_config.VmImage);
                VirtualMachineConfiguration vmc = new VirtualMachineConfiguration(imageReference: imgref, nodeAgentSkuId: "batch.node.windows amd64");
                CreateBatchPool(vmc, _config.VmSKU);
                // Create Job
                CloudJob job = Client.JobOperations.CreateJob();
                job.Id = _jobId;
                job.PoolInformation = new PoolInformation {
                    PoolId = _poolId
                };
                job.Commit();
                // Create Tasks
                List <CloudTask> tasks  = new List <CloudTask>();
                string           taskId = "task" + System.DateTime.Now.Ticks.ToString();
                string           cmd    = String.Format("cmd /c type {0}", req.FilePath);
                CloudTask        task   = new CloudTask(taskId, cmd);
                task.ResourceFiles = new List <ResourceFile> {
                    req
                };
                tasks.Add(task);
                Client.JobOperations.AddTask(_jobId, tasks);
            }

            catch (Exception x)
            {
                //TODO: Log Somewhere
                throw;
            }
        }
Exemplo n.º 5
0
        static void Main(string[] args)
        {
            string storageConnectionString =
                $"DefaultEndpointsProtocol=https;AccountName={StorageAccountName};AccountKey={StorageAccountKey}";

            // Retrieve the storage account
            CloudStorageAccount storageAccount = CloudStorageAccount.Parse(storageConnectionString);

            // Create the blob client
            CloudBlobClient blobClient         = storageAccount.CreateCloudBlobClient();
            const string    inputContainerName = "batchinput";
            List <string>   inputFilePaths     = new List <string>
            {
                "taskdata0.txt",
                "taskdata1.txt",
                "taskdata2.txt"
            };

            // Upload the input files to blob storage
            List <ResourceFile> inputFiles = new List <ResourceFile>();

            foreach (string filePath in inputFilePaths)
            {
                inputFiles.Add(UploadFileToContainer(blobClient, inputContainerName, filePath));
            }

            // Get a SAS Url for the output container
            const string outputContainerName   = "batchoutput";
            string       outputContainerSasUrl = GetOutputContainerSasUrl(blobClient, outputContainerName);

            // Retrieve an access token from Azure AD to authenticate with the Azure Batch API
            AuthenticationContext authContext           = new AuthenticationContext(AuthorityUri);
            AuthenticationResult  authResult            = authContext.AcquireTokenAsync(BatchResourceUri, new ClientCredential(ClientId, ClientKey)).Result;
            BatchTokenCredentials batchTokenCredentials = new BatchTokenCredentials(BatchAccountUrl, authResult.AccessToken);

            using (BatchClient batchClient = BatchClient.Open(batchTokenCredentials))
            {
                Console.WriteLine("Creating job [{0}]...", JobId);
                try
                {
                    CloudJob job = batchClient.JobOperations.CreateJob();
                    job.Id = JobId;
                    job.PoolInformation = new PoolInformation {
                        PoolId = PoolId
                    };

                    job.Commit();
                }
                catch (BatchException be)
                {
                    // Accept the specific error code JobExists as that is expected if the job already exists
                    if (be.RequestInformation?.BatchError?.Code == BatchErrorCodeStrings.JobExists)
                    {
                        Console.WriteLine("The job {0} already existed when we tried to create it", JobId);
                    }
                    else
                    {
                        throw; // Any other exception is unexpected
                    }
                }

                // Create a collection to hold the tasks that we'll be adding to the job
                Console.WriteLine("Adding {0} tasks to job [{1}]...", inputFiles.Count, JobId);

                List <CloudTask> tasks = new List <CloudTask>();

                // Create each of the tasks to process one of the input files.
                for (int i = 0; i < inputFiles.Count; i++)
                {
                    string taskId          = String.Format("Task{0}", i);
                    string inputFilename   = inputFiles[i].FilePath;
                    string outputFileName  = string.Format("out{0}", inputFilename);
                    string taskCommandLine = string.Format("cmd /c %AZ_BATCH_APP_PACKAGE_READWRITEFILE%\\ReadWriteFile.exe {0} {1}", inputFilename, outputFileName);

                    CloudTask task = new CloudTask(taskId, taskCommandLine);

                    // Set the resource files and output files for the task
                    task.ResourceFiles = new List <ResourceFile> {
                        inputFiles[i]
                    };
                    task.OutputFiles = new List <OutputFile>
                    {
                        new OutputFile(
                            filePattern: outputFileName,
                            destination: new OutputFileDestination(new OutputFileBlobContainerDestination(containerUrl: outputContainerSasUrl, path: outputFileName)),
                            uploadOptions: new OutputFileUploadOptions(OutputFileUploadCondition.TaskCompletion))
                    };
                    tasks.Add(task);
                }

                // Add all tasks to the job.
                batchClient.JobOperations.AddTask(JobId, tasks);

                // Monitor task success/failure, specifying a maximum amount of time to wait for the tasks to complete.
                TimeSpan timeout = TimeSpan.FromMinutes(30);
                Console.WriteLine("Monitoring all tasks for 'Completed' state, timeout in {0}...", timeout);

                IEnumerable <CloudTask> addedTasks = batchClient.JobOperations.ListTasks(JobId);

                batchClient.Utilities.CreateTaskStateMonitor().WaitAll(addedTasks, TaskState.Completed, timeout);

                Console.WriteLine("All tasks reached state Completed.");

                // Print task output
                Console.WriteLine();
                Console.WriteLine("Printing task output...");

                IEnumerable <CloudTask> completedtasks = batchClient.JobOperations.ListTasks(JobId);

                foreach (CloudTask task in completedtasks)
                {
                    string nodeId = String.Format(task.ComputeNodeInformation.ComputeNodeId);
                    Console.WriteLine("Task: {0}", task.Id);
                    Console.WriteLine("Node: {0}", nodeId);
                    Console.WriteLine("Standard out:");
                    Console.WriteLine(task.GetNodeFile(Constants.StandardOutFileName).ReadAsString());
                }

                // Clean up Batch resources (if the user so chooses)
                Console.WriteLine();
                Console.Write("Delete job? [yes] no: ");
                string response = Console.ReadLine().ToLower();
                if (response != "n" && response != "no")
                {
                    batchClient.JobOperations.DeleteJob(JobId);
                }
            }
        }
Exemplo n.º 6
0
            protected override void OnExpand()
            {
                int requirement = int.Parse(RuntimeSettings[gCEL.Core.Components.FABRIC.Constants.CAPACITY_SETTING_REQUIREMENT]);
                int workers     = int.Parse(Settings[SETTING_WORKERS]);

                PoolSize = (requirement / workers) + ((requirement % workers) > 0 ? 1 : 0);
                if (PoolSize == 0)
                {
                    PoolSize = 1;
                }

                string reqfile = gCEL.Core.Utils.FileSystem.GetTempPath() + "gcel_request_" + gCEL.Core.Utils.Environment.UniqueIdentifier + ".xml";

                //> CREATE
                try
                {
                    Stopwatch sw = new Stopwatch();
                    sw.Start();

                    ResourceFile req = CreateResourceFile(reqfile);

                    //BatchSharedKeyCredentials cred = new BatchSharedKeyCredentials(Settings[SETTING_BATCH_ACCOUNT_URL], Settings[SETTING_BATCH_ACCOUNT_NAME], Settings[SETTING_BATCH_ACCOUNT_KEY]);
                    BatchTokenCredentials cred = new BatchTokenCredentials(Settings[SETTING_BATCH_ACCOUNT_URL], GetToken());

                    Client = BatchClient.Open(cred);

                    //> VM
                    ImageReference imgref           = new ImageReference(Settings[SETTING_BATCH_CUSTOM_IMAGE]);
                    VirtualMachineConfiguration vmc = new VirtualMachineConfiguration(imageReference: imgref, nodeAgentSkuId: "batch.node.windows amd64");

                    //> POOL
                    CreateBatchPool(vmc);

                    //> JOB
                    CloudJob job = Client.JobOperations.CreateJob();
                    job.Id = JobId;
                    job.PoolInformation = new PoolInformation {
                        PoolId = PoolId
                    };

                    job.Commit();

                    //> TASKS
                    List <CloudTask> tasks = new List <CloudTask>();

                    for (int i = 0; i < requirement; i++)
                    {
                        string taskId = "gCELTask" + i;
                        string engine = Settings[SETTING_SYSTEM_FOLDER] + gCEL.Core.Constants.ENGINE_CMD;
                        string cmd    = String.Format("cmd /c {0} command=host component=engine request={1}", engine, req.FilePath);

                        CloudTask task = new CloudTask(taskId, cmd);
                        task.ResourceFiles = new List <ResourceFile> {
                            req
                        };
                        tasks.Add(task);
                    }

                    Client.JobOperations.AddTask(JobId, tasks);

                    //>
                    sw.Stop();

                    TimeSpan ts          = sw.Elapsed;
                    string   elapsedTime = String.Format("{0:00}:{1:00}:{2:00}.{3:00}", ts.Hours, ts.Minutes, ts.Seconds, ts.Milliseconds / 10);
                    Logger.Info(Tag + ":POOLCREATED:" + elapsedTime);
                }

                catch (Exception x)
                {
                    Logger.Error(Tag + ":" + x.Message);
                    throw;
                }

                finally
                {
                    gCEL.Core.Utils.FileSystem.SafeDelete(reqfile);
                    OnContract();
                }

                Logger.Info(Tag + ":ALLOCATED:" + PoolSize);
            }