private async Task CreateTasksAsync(BatchClient batchClient, string jobId, StorageTask task)
{
TaskContainerSettings cmdContainerSettings = new TaskContainerSettings(
imageName: "ImageName",
containerRunOptions: ""
);
// create a simple task. Each task within a job must have a unique ID
for (var i = 0; i < task.Services.Count; i++)
{
var commandLine = $"cmd /c c:\\TestAppService.exe {task.Id} 1";
CloudTask cloudTask =
new CloudTask(
$"{task.Services[i]}{i}", commandLine);
cloudTask.ContainerSettings = cmdContainerSettings;
//La tâche doit être éxécutée en administrateur
cloudTask.UserIdentity = new UserIdentity(new AutoUserSpecification(elevationLevel: ElevationLevel.Admin, scope: AutoUserScope.Task));
cloudTask.Constraints = new TaskConstraints
{
MaxTaskRetryCount = 3
};
await batchClient.JobOperations.AddTaskAsync(jobId, cloudTask);
}
}
private static CloudTask CreateTask(string jobId, ResourceFile inputFile, string outputContainerSasUrl)
{
var containerSettings = new TaskContainerSettings("globomantics.azurecr.io/optionpricer");
var taskId = "simulationTask_" + Path.GetFileNameWithoutExtension(inputFile.FilePath);
var commandLine = $@"/app/OptionPricerEngine {inputFile.FilePath} ""{outputContainerSasUrl}"" {jobId}_{taskId}";
var task = new CloudTask(taskId, commandLine)
{
ContainerSettings = containerSettings
};
task.ResourceFiles = new[] { inputFile };
return(task);
}
/// <summary>
/// Create a task to submit to the Batch instance
/// </summary>
/// <param name="jobId">Desired job id</param>
/// <param name="imageName">Desired docker image name</param>
/// <param name="batchClient">The batchclient communicating with the batch instance</param>
/// <param name="taskEnvironmentSettings">Task's settings</param>
/// <param name="taskName">The name of the task</param>
/// <param name="taskDependencies">Dependencies the task has to wait upon</param>
private static void CreateTask(string jobId, string imageName, BatchClient batchClient, List <EnvironmentSetting> taskEnvironmentSettings,
string taskName, List <string> taskDependencies)
{
// Assign the job preparation task to the job
TaskContainerSettings workerContainer = new TaskContainerSettings(
imageName
);
CloudTask workerTask = new CloudTask(taskName, "echo here")
{
ContainerSettings = workerContainer,
Id = jobId,
EnvironmentSettings = taskEnvironmentSettings,
UserIdentity = new UserIdentity(new AutoUserSpecification(elevationLevel: ElevationLevel.Admin, scope: AutoUserScope.Task)),
DependsOn = TaskDependencies.OnIds(taskDependencies)
};
// Commit Job to create it in the service
batchClient.JobOperations.AddTask(jobId, workerTask);
}
static void Main(string[] args)
{
string storageConnectionString =
$"DefaultEndpointsProtocol=https;AccountName={StorageAccountName};AccountKey={StorageAccountKey}";
// Retrieve the storage account
CloudStorageAccount storageAccount = CloudStorageAccount.Parse(storageConnectionString);
// Create the blob client
CloudBlobClient blobClient = storageAccount.CreateCloudBlobClient();
// Upload the input files to blob storage
const string inputContainerName = "batchinput";
List <string> inputFilePaths = new List <string>
{
"taskdata0.txt",
"taskdata1.txt",
"taskdata2.txt"
};
List <ResourceFile> inputFiles = new List <ResourceFile>();
foreach (string filePath in inputFilePaths)
{
inputFiles.Add(UploadFileToContainer(blobClient, inputContainerName, filePath));
}
// Get a SAS Url for the output container
const string outputContainerName = "batchoutput";
string outputContainerSasUrl = GetOutputContainerSasUrl(blobClient, outputContainerName);
// Specify a container registry
ContainerRegistry containerRegistry = new ContainerRegistry(
registryServer: RegistryServer,
userName: RegistryUserName,
password: RegistryPassword);
// Create container configuration, prefetching Docker images from the container registry
ContainerConfiguration containerConfig = new ContainerConfiguration()
{
ContainerImageNames = ContainerImageNames,
ContainerRegistries = new List <ContainerRegistry> {
containerRegistry
}
};
// Create the virtual machine image reference - make sure to choose an image that supports containers
ImageReference imageReference = new ImageReference(
publisher: "MicrosoftWindowsServer",
offer: "WindowsServer",
sku: "2016-datacenter-with-containers",
version: "latest");
// Create the virtual machine configuration for the pool and set the container configuration
VirtualMachineConfiguration virtualMachineConfiguration = new VirtualMachineConfiguration(
imageReference: imageReference,
nodeAgentSkuId: "batch.node.windows amd64");
virtualMachineConfiguration.ContainerConfiguration = containerConfig;
BatchSharedKeyCredentials cred = new BatchSharedKeyCredentials(BatchAccountUrl, BatchAccountName, BatchAccountKey);
using (BatchClient batchClient = BatchClient.Open(cred))
{
Console.WriteLine("Creating pool [{0}]...", PoolId);
try
{
CloudPool pool = batchClient.PoolOperations.CreatePool(
poolId: PoolId,
targetDedicatedComputeNodes: PoolNodeCount,
virtualMachineSize: PoolVMSize,
virtualMachineConfiguration: virtualMachineConfiguration);
pool.Commit();
}
catch (BatchException be)
{
// Accept the specific error code PoolExists as that is expected if the pool already exists
if (be.RequestInformation?.BatchError?.Code == BatchErrorCodeStrings.PoolExists)
{
Console.WriteLine("The pool {0} already existed when we tried to create it", PoolId);
}
else
{
throw; // Any other exception is unexpected
}
}
Console.WriteLine("Creating job [{0}]...", JobId);
CloudJob job = null;
try
{
job = batchClient.JobOperations.CreateJob();
job.Id = JobId;
job.PoolInformation = new PoolInformation {
PoolId = PoolId
};
// Add job preparation task to remove existing Docker image
Console.WriteLine("Adding job preparation task to job [{0}]...", JobId);
string jobPreparationCmdLine = $"cmd /c docker rmi -f {ContainerImageNames[0]}:latest";
JobPreparationTask jobPreparationTask = new JobPreparationTask(jobPreparationCmdLine)
{
UserIdentity = new UserIdentity(new AutoUserSpecification(elevationLevel: ElevationLevel.Admin, scope: AutoUserScope.Task))
};
job.JobPreparationTask = jobPreparationTask;
job.Commit();
}
catch (BatchException be)
{
// Accept the specific error code JobExists as that is expected if the job already exists
if (be.RequestInformation?.BatchError?.Code == BatchErrorCodeStrings.JobExists)
{
Console.WriteLine("The job {0} already existed when we tried to create it", JobId);
}
else
{
throw; // Any other exception is unexpected
}
}
if (job != null)
{
// Create a collection to hold the tasks that we'll be adding to the job
Console.WriteLine("Adding {0} tasks to job [{1}]...", inputFiles.Count, JobId);
List <CloudTask> tasks = new List <CloudTask>();
// Create each of the tasks to process one of the input files.
for (int i = 0; i < inputFiles.Count; i++)
{
string taskId = String.Format("Task{0}", i);
string inputFilename = inputFiles[i].FilePath;
string outputFileName = string.Format("out{0}", inputFilename);
// Override the default entrypoint of the container
string taskCommandLine = string.Format("C:\\ReadWriteFile\\ReadWriteFile.exe {0} {1}", inputFilename, outputFileName);
// Specify the container the task will run
TaskContainerSettings cmdContainerSettings = new TaskContainerSettings(
imageName: "nimccollftacr.azurecr.io/batch/readwritefile"
);
CloudTask task = new CloudTask(taskId, taskCommandLine);
task.ContainerSettings = cmdContainerSettings;
// Set the resource files and output files for the task
task.ResourceFiles = new List <ResourceFile> {
inputFiles[i]
};
task.OutputFiles = new List <OutputFile>
{
new OutputFile(
filePattern: outputFileName,
destination: new OutputFileDestination(new OutputFileBlobContainerDestination(containerUrl: outputContainerSasUrl, path: outputFileName)),
uploadOptions: new OutputFileUploadOptions(OutputFileUploadCondition.TaskCompletion))
};
// You must elevate the identity of the task in order to run a container
task.UserIdentity = new UserIdentity(new AutoUserSpecification(elevationLevel: ElevationLevel.Admin, scope: AutoUserScope.Task));
tasks.Add(task);
}
// Add all tasks to the job.
batchClient.JobOperations.AddTask(JobId, tasks);
// Monitor task success/failure, specifying a maximum amount of time to wait for the tasks to complete.
TimeSpan timeout = TimeSpan.FromMinutes(30);
Console.WriteLine("Monitoring all tasks for 'Completed' state, timeout in {0}...", timeout);
IEnumerable <CloudTask> addedTasks = batchClient.JobOperations.ListTasks(JobId);
batchClient.Utilities.CreateTaskStateMonitor().WaitAll(addedTasks, TaskState.Completed, timeout);
Console.WriteLine("All tasks reached state Completed.");
// Print task output
Console.WriteLine();
Console.WriteLine("Printing task output...");
IEnumerable <CloudTask> completedtasks = batchClient.JobOperations.ListTasks(JobId);
foreach (CloudTask task in completedtasks)
{
string nodeId = String.Format(task.ComputeNodeInformation.ComputeNodeId);
Console.WriteLine("Task: {0}", task.Id);
Console.WriteLine("Node: {0}", nodeId);
Console.WriteLine("Standard out:");
Console.WriteLine(task.GetNodeFile(Constants.StandardOutFileName).ReadAsString());
}
}
// Clean up Batch resources (if the user so chooses)
Console.WriteLine();
Console.Write("Delete job? [yes] no: ");
string response = Console.ReadLine().ToLower();
if (response != "n" && response != "no")
{
batchClient.JobOperations.DeleteJob(JobId);
}
Console.Write("Delete pool? [yes] no: ");
response = Console.ReadLine().ToLower();
if (response != "n" && response != "no")
{
batchClient.PoolOperations.DeletePool(PoolId);
}
}
}
public async Task RunAsync()
{
string PoolId = "engie";
string JobId = "ingestion";
Console.WriteLine("Sample start: {0}", DateTime.Now);
BatchSharedKeyCredentials cred = new BatchSharedKeyCredentials(BatchAccountUrl, BatchAccountName, BatchAccountKey);
Func <Task <string> > tokenProvider = () => GetAuthenticationTokenAsync();
using (BatchClient client = await BatchClient.OpenAsync(new BatchTokenCredentials(BatchAccountUrl, tokenProvider)))
{
Console.WriteLine("Creating pool [{0}]...", PoolId);
ImageReference imageReference = new ImageReference("/subscriptions/e243327e-b18c-4766-8f44-d9a945082e57/resourceGroups/engie/providers/Microsoft.Compute/images/engieimg3");
ContainerConfiguration containerConfig = new ContainerConfiguration();
VirtualMachineConfiguration virtualMachineConfiguration =
new VirtualMachineConfiguration(imageReference: imageReference, nodeAgentSkuId: "batch.node.ubuntu 16.04");
virtualMachineConfiguration.ContainerConfiguration = containerConfig;
CloudPool pool = client.PoolOperations.CreatePool(
poolId: PoolId,
virtualMachineSize: "Standard_D11_v2",
virtualMachineConfiguration: virtualMachineConfiguration,
targetDedicatedComputeNodes: 1);
pool.MaxTasksPerComputeNode = 2;
pool.TaskSchedulingPolicy = new TaskSchedulingPolicy(ComputeNodeFillType.Spread);
// Commit pool creation
//pool.Commit();
// Simple task command
CloudJob job = client.JobOperations.CreateJob();
job.Id = JobId;
job.PoolInformation = new PoolInformation {
PoolId = PoolId
};
job.Commit();
TaskContainerSettings taskContainerSettings = new TaskContainerSettings(
imageName: "scoriani/ingestion", containerRunOptions: "--env DOTNET_SKIP_FIRST_TIME_EXPERIENCE=true --rm");
List <CloudTask> tasks = new List <CloudTask>();
CloudTask containerTask;
string cmdLine;
// Storage container root
string containerName = "work3";
// Starting from
DateTime startDate = DateTime.Parse("2017-01-01");
// For a given amount of days
for (int i = 1; i < 2; i++)
{
cmdLine = String.Format("{0} {1} {2}", startDate.ToString("yyyy-MM-dd"), startDate.ToString("yyyy-MM-dd"), containerName);
containerTask = new CloudTask("Task1-" + i.ToString(), cmdLine);
containerTask.ContainerSettings = taskContainerSettings;
tasks.Add(containerTask);
startDate = startDate.AddDays(1);
}
client.JobOperations.AddTask(JobId, tasks);
TimeSpan timeout = TimeSpan.FromMinutes(1800);
Console.WriteLine("Monitoring all tasks for 'Completed' state, timeout in {0}...", timeout);
IEnumerable <CloudTask> addedTasks = client.JobOperations.ListTasks(JobId);
client.Utilities.CreateTaskStateMonitor().WaitAll(addedTasks, TaskState.Completed, timeout);
Console.WriteLine("All tasks reached state Completed.");
Console.WriteLine();
Console.WriteLine("Printing task output...");
IEnumerable <CloudTask> completedtasks = client.JobOperations.ListTasks(JobId);
foreach (CloudTask task in completedtasks)
{
string nodeId = String.Format(task.ComputeNodeInformation.ComputeNodeId);
Console.WriteLine("Task: {0}", task.Id);
Console.WriteLine("Node: {0}", nodeId);
Console.WriteLine("Standard out:");
Console.WriteLine(task.GetNodeFile(Constants.StandardOutFileName).ReadAsString());
}
Console.WriteLine("Sample end: {0}", DateTime.Now);
client.JobOperations.DeleteJob(JobId);
//client.PoolOperations.DeletePool(PoolId);
}
}
