public static void CreateBatchTasks(string action, string jobId, string containerUrl, string batchAccountUrl, string sqlServerName, string accessToken, dynamic databases, ILogger log)
{
// Get a Batch client using function identity
log.LogInformation("CreateBatchTasks: entering");
var azureServiceTokenProvider = new AzureServiceTokenProvider();
BatchTokenCredentials batchCred = new BatchTokenCredentials(batchAccountUrl, azureServiceTokenProvider.GetAccessTokenAsync("https://batch.core.windows.net/").Result);
using (BatchClient batchClient = BatchClient.Open(batchCred))
{
// For each database, submit the Exporting job to Azure Batch Compute Pool.
log.LogInformation("CreateBatchTasks: enumerating databases");
List <CloudTask> tasks = new List <CloudTask>();
foreach (var db in databases)
{
string serverDatabaseName = db.name.ToString();
string logicalDatabase = serverDatabaseName.Remove(0, sqlServerName.Length + 1);
log.LogInformation("CreateBatchTasks: creating task for database {0}", logicalDatabase);
string taskId = sqlServerName + "_" + logicalDatabase;
string command = string.Format("cmd /c %AZ_BATCH_APP_PACKAGE_{0}#{1}%\\BatchWrapper {2}", AppPackageName.ToUpper(), AppPackageVersion, action);
command += string.Format(" {0} {1} {2} {3} {4}", sqlServerName, logicalDatabase, accessToken, AppPackageName.ToUpper(), AppPackageVersion);
string taskCommandLine = string.Format(command);
CloudTask singleTask = new CloudTask(taskId, taskCommandLine);
singleTask.EnvironmentSettings = new[] { new EnvironmentSetting("JOB_CONTAINER_URL", containerUrl) };
Console.WriteLine(string.Format("Adding task {0} to job ...", taskId));
tasks.Add(singleTask);
}
// Add all tasks to the job.
batchClient.JobOperations.AddTask(jobId, tasks);
}
log.LogInformation("CreateBatchTasks: exiting");
}
private void InitEngine()
{
Console.WriteLine("Initializing batch engine: {0}", DateTime.Now);
// Get a Batch client using account creds
// BatchSharedKeyCredentials cred =
// new BatchSharedKeyCredentials(BatchAccountUrl, BatchAccountName, BatchAccountKey);
Func <Task <string> > tokenProvider =
async() => await GetAuthenticationTokenAsync(
AzureEnvConstants.AZURE_AD_TOKEN_EP + EnvVars[AzureEnvConstants.AZURE_AD_TENANT_ID],
EnvVars[AzureEnvConstants.AZURE_AD_SP_CLIENT_ID],
EnvVars[AzureEnvConstants.AZURE_AD_SP_CLIENT_SECRET]);
Creds =
new BatchTokenCredentials(EnvVars[AzureEnvConstants.AZURE_BATCH_ACCOUNT_URL], tokenProvider);
}
public static async Task <string> CreateBatchPoolAndImportJob([ActivityTrigger] ImportRequest request, ILogger log)
{
var azureServiceTokenProvider = new AzureServiceTokenProvider();
// Get a Batch client using function identity
BatchTokenCredentials batchCred = new BatchTokenCredentials(request.BatchAccountUrl, await azureServiceTokenProvider.GetAccessTokenAsync("https://batch.core.windows.net/"));
string jobId = request.TargetSqlServerName + "-Import-" + DateTime.UtcNow.ToString("MMddHHmmss");
using (BatchClient batchClient = BatchClient.Open(batchCred))
{
ImageReference imageReference = CreateImageReference();
VirtualMachineConfiguration vmConfiguration = CreateVirtualMachineConfiguration(imageReference);
await CreateBatchPoolIfNotExist(batchClient, vmConfiguration, request.VNetSubnetId);
await CreateBatchJob(batchClient, jobId, log);
}
return(jobId);
}
public void Execute()
{
string reqfile = _config.ResourceFolder + _config.ResourceName;
//> CREATE
try
{
ResourceFile req = CreateResourceFile(reqfile);
BatchTokenCredentials cred = new BatchTokenCredentials(_config.BatchAccountUrl, GetToken());
Client = BatchClient.Open(cred);
ImageReference imgref = new ImageReference(_config.VmImage);
VirtualMachineConfiguration vmc = new VirtualMachineConfiguration(imageReference: imgref, nodeAgentSkuId: "batch.node.windows amd64");
CreateBatchPool(vmc, _config.VmSKU);
// Create Job
CloudJob job = Client.JobOperations.CreateJob();
job.Id = _jobId;
job.PoolInformation = new PoolInformation {
PoolId = _poolId
};
job.Commit();
// Create Tasks
List <CloudTask> tasks = new List <CloudTask>();
string taskId = "task" + System.DateTime.Now.Ticks.ToString();
string cmd = String.Format("cmd /c type {0}", req.FilePath);
CloudTask task = new CloudTask(taskId, cmd);
task.ResourceFiles = new List <ResourceFile> {
req
};
tasks.Add(task);
Client.JobOperations.AddTask(_jobId, tasks);
}
catch (Exception x)
{
//TODO: Log Somewhere
throw;
}
}
static void Main(string[] args)
{
string storageConnectionString =
$"DefaultEndpointsProtocol=https;AccountName={StorageAccountName};AccountKey={StorageAccountKey}";
// Retrieve the storage account
CloudStorageAccount storageAccount = CloudStorageAccount.Parse(storageConnectionString);
// Create the blob client
CloudBlobClient blobClient = storageAccount.CreateCloudBlobClient();
const string inputContainerName = "batchinput";
List <string> inputFilePaths = new List <string>
{
"taskdata0.txt",
"taskdata1.txt",
"taskdata2.txt"
};
// Upload the input files to blob storage
List <ResourceFile> inputFiles = new List <ResourceFile>();
foreach (string filePath in inputFilePaths)
{
inputFiles.Add(UploadFileToContainer(blobClient, inputContainerName, filePath));
}
// Get a SAS Url for the output container
const string outputContainerName = "batchoutput";
string outputContainerSasUrl = GetOutputContainerSasUrl(blobClient, outputContainerName);
// Retrieve an access token from Azure AD to authenticate with the Azure Batch API
AuthenticationContext authContext = new AuthenticationContext(AuthorityUri);
AuthenticationResult authResult = authContext.AcquireTokenAsync(BatchResourceUri, new ClientCredential(ClientId, ClientKey)).Result;
BatchTokenCredentials batchTokenCredentials = new BatchTokenCredentials(BatchAccountUrl, authResult.AccessToken);
using (BatchClient batchClient = BatchClient.Open(batchTokenCredentials))
{
Console.WriteLine("Creating job [{0}]...", JobId);
try
{
CloudJob job = batchClient.JobOperations.CreateJob();
job.Id = JobId;
job.PoolInformation = new PoolInformation {
PoolId = PoolId
};
job.Commit();
}
catch (BatchException be)
{
// Accept the specific error code JobExists as that is expected if the job already exists
if (be.RequestInformation?.BatchError?.Code == BatchErrorCodeStrings.JobExists)
{
Console.WriteLine("The job {0} already existed when we tried to create it", JobId);
}
else
{
throw; // Any other exception is unexpected
}
}
// Create a collection to hold the tasks that we'll be adding to the job
Console.WriteLine("Adding {0} tasks to job [{1}]...", inputFiles.Count, JobId);
List <CloudTask> tasks = new List <CloudTask>();
// Create each of the tasks to process one of the input files.
for (int i = 0; i < inputFiles.Count; i++)
{
string taskId = String.Format("Task{0}", i);
string inputFilename = inputFiles[i].FilePath;
string outputFileName = string.Format("out{0}", inputFilename);
string taskCommandLine = string.Format("cmd /c %AZ_BATCH_APP_PACKAGE_READWRITEFILE%\\ReadWriteFile.exe {0} {1}", inputFilename, outputFileName);
CloudTask task = new CloudTask(taskId, taskCommandLine);
// Set the resource files and output files for the task
task.ResourceFiles = new List <ResourceFile> {
inputFiles[i]
};
task.OutputFiles = new List <OutputFile>
{
new OutputFile(
filePattern: outputFileName,
destination: new OutputFileDestination(new OutputFileBlobContainerDestination(containerUrl: outputContainerSasUrl, path: outputFileName)),
uploadOptions: new OutputFileUploadOptions(OutputFileUploadCondition.TaskCompletion))
};
tasks.Add(task);
}
// Add all tasks to the job.
batchClient.JobOperations.AddTask(JobId, tasks);
// Monitor task success/failure, specifying a maximum amount of time to wait for the tasks to complete.
TimeSpan timeout = TimeSpan.FromMinutes(30);
Console.WriteLine("Monitoring all tasks for 'Completed' state, timeout in {0}...", timeout);
IEnumerable <CloudTask> addedTasks = batchClient.JobOperations.ListTasks(JobId);
batchClient.Utilities.CreateTaskStateMonitor().WaitAll(addedTasks, TaskState.Completed, timeout);
Console.WriteLine("All tasks reached state Completed.");
// Print task output
Console.WriteLine();
Console.WriteLine("Printing task output...");
IEnumerable <CloudTask> completedtasks = batchClient.JobOperations.ListTasks(JobId);
foreach (CloudTask task in completedtasks)
{
string nodeId = String.Format(task.ComputeNodeInformation.ComputeNodeId);
Console.WriteLine("Task: {0}", task.Id);
Console.WriteLine("Node: {0}", nodeId);
Console.WriteLine("Standard out:");
Console.WriteLine(task.GetNodeFile(Constants.StandardOutFileName).ReadAsString());
}
// Clean up Batch resources (if the user so chooses)
Console.WriteLine();
Console.Write("Delete job? [yes] no: ");
string response = Console.ReadLine().ToLower();
if (response != "n" && response != "no")
{
batchClient.JobOperations.DeleteJob(JobId);
}
}
}
protected override void OnExpand()
{
int requirement = int.Parse(RuntimeSettings[gCEL.Core.Components.FABRIC.Constants.CAPACITY_SETTING_REQUIREMENT]);
int workers = int.Parse(Settings[SETTING_WORKERS]);
PoolSize = (requirement / workers) + ((requirement % workers) > 0 ? 1 : 0);
if (PoolSize == 0)
{
PoolSize = 1;
}
string reqfile = gCEL.Core.Utils.FileSystem.GetTempPath() + "gcel_request_" + gCEL.Core.Utils.Environment.UniqueIdentifier + ".xml";
//> CREATE
try
{
Stopwatch sw = new Stopwatch();
sw.Start();
ResourceFile req = CreateResourceFile(reqfile);
//BatchSharedKeyCredentials cred = new BatchSharedKeyCredentials(Settings[SETTING_BATCH_ACCOUNT_URL], Settings[SETTING_BATCH_ACCOUNT_NAME], Settings[SETTING_BATCH_ACCOUNT_KEY]);
BatchTokenCredentials cred = new BatchTokenCredentials(Settings[SETTING_BATCH_ACCOUNT_URL], GetToken());
Client = BatchClient.Open(cred);
//> VM
ImageReference imgref = new ImageReference(Settings[SETTING_BATCH_CUSTOM_IMAGE]);
VirtualMachineConfiguration vmc = new VirtualMachineConfiguration(imageReference: imgref, nodeAgentSkuId: "batch.node.windows amd64");
//> POOL
CreateBatchPool(vmc);
//> JOB
CloudJob job = Client.JobOperations.CreateJob();
job.Id = JobId;
job.PoolInformation = new PoolInformation {
PoolId = PoolId
};
job.Commit();
//> TASKS
List <CloudTask> tasks = new List <CloudTask>();
for (int i = 0; i < requirement; i++)
{
string taskId = "gCELTask" + i;
string engine = Settings[SETTING_SYSTEM_FOLDER] + gCEL.Core.Constants.ENGINE_CMD;
string cmd = String.Format("cmd /c {0} command=host component=engine request={1}", engine, req.FilePath);
CloudTask task = new CloudTask(taskId, cmd);
task.ResourceFiles = new List <ResourceFile> {
req
};
tasks.Add(task);
}
Client.JobOperations.AddTask(JobId, tasks);
//>
sw.Stop();
TimeSpan ts = sw.Elapsed;
string elapsedTime = String.Format("{0:00}:{1:00}:{2:00}.{3:00}", ts.Hours, ts.Minutes, ts.Seconds, ts.Milliseconds / 10);
Logger.Info(Tag + ":POOLCREATED:" + elapsedTime);
}
catch (Exception x)
{
Logger.Error(Tag + ":" + x.Message);
throw;
}
finally
{
gCEL.Core.Utils.FileSystem.SafeDelete(reqfile);
OnContract();
}
Logger.Info(Tag + ":ALLOCATED:" + PoolSize);
}