private CloudPool CreatePoolIfNotExists(String poolID, String vmSize, int computerNodes = 2, ImageReference imageReference = null)
{
messageContainer.AddInformationMessage("Configuring pool...");
CloudPool cPool = null;
try { cPool = batchClient.PoolOperations.GetPool(poolID); } catch { }
if (cPool == null)
{
//imageReference = (imageReference == null ? CreateImageReference("MicrosoftWindowsServer", "WindowsServer", "2016-Datacenter", "latest") : imageReference);
imageReference = (imageReference == null ? CreateImageReference() : imageReference);
VirtualMachineConfiguration vmConfig = new VirtualMachineConfiguration(imageReference, SKUReference(imageReference.Offer));
messageContainer.AddInformationMessage("Creating pool...");
cPool = batchClient.PoolOperations.CreatePool(poolID, vmSize, vmConfig, computerNodes);
cPool.ApplicationPackageReferences = new List <ApplicationPackageReference>
{
new ApplicationPackageReference
{
ApplicationId = "radiance",
Version = "1.0"
}
};
cPool.Commit();
messageContainer.AddInformationMessage("Pool created...");
}
messageContainer.AddInformationMessage("Pool configured...");
return(cPool);
}
private static void CreateBatchPool(BatchClient batchClient, VirtualMachineConfiguration vmConfiguration)
{
try
{
CloudPool pool = batchClient.PoolOperations.CreatePool(
poolId: PoolId,
targetDedicatedComputeNodes: PoolNodeCount,
virtualMachineSize: PoolVMSize,
virtualMachineConfiguration: vmConfiguration);
pool.Commit();
}
catch (BatchException be)
{
// Accept the specific error code PoolExists as that is expected if the pool already exists
if (be.RequestInformation?.BatchError?.Code == BatchErrorCodeStrings.PoolExists)
{
Console.WriteLine("The pool {0} already existed when we tried to create it", PoolId);
}
else
{
throw; // Any other exception is unexpected
}
}
}
// BATCH CLIENT OPERATIONS - FUNCTION IMPLEMENTATIONS
/// <summary>
/// Creates the Batch pool.
/// </summary>
/// <param name="batchClient">A BatchClient object</param>
/// <param name="poolId">ID of the CloudPool object to create.</param>
private static async Task CreatePoolIfNotExistAsync(BatchClient batchClient, string poolId)
{
CloudPool pool = null;
try
{
Console.WriteLine("Creating pool [{0}]...", poolId);
ImageReference imageReference = new ImageReference(
publisher: "MicrosoftWindowsServer",
offer: "WindowsServer",
sku: "2012-R2-Datacenter-smalldisk",
version: "latest");
VirtualMachineConfiguration virtualMachineConfiguration =
new VirtualMachineConfiguration(
imageReference: imageReference,
nodeAgentSkuId: "batch.node.windows amd64");
// Create an unbound pool. No pool is actually created in the Batch service until we call
// CloudPool.Commit(). This CloudPool instance is therefore considered "unbound," and we can
// modify its properties.
pool = batchClient.PoolOperations.CreatePool(
poolId: poolId,
targetDedicatedComputeNodes: DedicatedNodeCount,
targetLowPriorityComputeNodes: LowPriorityNodeCount,
virtualMachineSize: PoolVMSize,
virtualMachineConfiguration: virtualMachineConfiguration);
// Specify the application and version to install on the compute nodes
// This assumes that a Windows 64-bit zipfile of ffmpeg has been added to Batch account
// with Application Id of "ffmpeg" and Version of "4.3.1".
// Download the zipfile https://www.gyan.dev/ffmpeg/builds/packages/ffmpeg-4.3.1-2020-09-21-full_build.zip
// to upload as application package
pool.ApplicationPackageReferences = new List <ApplicationPackageReference>
{
new ApplicationPackageReference
{
ApplicationId = appPackageId,
Version = appPackageVersion
}
};
await pool.CommitAsync();
}
catch (BatchException be)
{
// Accept the specific error code PoolExists as that is expected if the pool already exists
if (be.RequestInformation?.BatchError?.Code == BatchErrorCodeStrings.PoolExists)
{
Console.WriteLine("The pool {0} already existed when we tried to create it", poolId);
}
else
{
throw; // Any other exception is unexpected
}
}
}
private static async Task CreateBatchPoolAsync(BatchClient batchClient, string poolId)
{
CloudPool pool = null;
Console.WriteLine("Creating pool [{0}]...", poolId);
// Create an image reference object to store the settings for the nodes to be added to the pool
ImageReference imageReference = new ImageReference(
publisher: "MicrosoftWindowsServer",
offer: "WindowsServer",
sku: "2012-R2-Datacenter-smalldisk",
version: "latest");
// Use the image reference to create a VirtualMachineConfiguration object
VirtualMachineConfiguration virtualMachineConfiguration =
new VirtualMachineConfiguration(
imageReference: imageReference,
nodeAgentSkuId: "batch.node.windows amd64");
try
{
// Create an unbound pool. No pool is actually created in the Batch service until we call
// CloudPool.CommitAsync(). This CloudPool instance is therefore considered "unbound," and we can
// modify its properties.
pool = batchClient.PoolOperations.CreatePool(
poolId: poolId,
targetDedicatedComputeNodes: DedicatedNodeCount,
targetLowPriorityComputeNodes: LowPriorityNodeCount,
virtualMachineSize: PoolVMSize,
virtualMachineConfiguration: virtualMachineConfiguration);
// Specify the application and version to install on the compute nodes
pool.ApplicationPackageReferences = new List <ApplicationPackageReference>
{
new ApplicationPackageReference
{
ApplicationId = appPackageId,
Version = appPackageVersion
}
};
// Create the pool
await pool.CommitAsync();
}
catch (BatchException be)
{
// Accept the specific error code PoolExists as that is expected if the pool already exists
if (be.RequestInformation?.BatchError?.Code == BatchErrorCodeStrings.PoolExists)
{
Console.WriteLine("The pool [{0}] already existed when we tried to create it", poolId);
}
else
{
throw; // Any other exception is unexpected
}
}
}
private static void CreatePool(BatchClient batchClient, VirtualMachineConfiguration vmConfiguration)
{
CloudPool pool = batchClient.PoolOperations.CreatePool(poolId: PoolId,
virtualMachineSize: PoolVMSize,
virtualMachineConfiguration: vmConfiguration,
targetDedicatedComputeNodes: PoolNodeCount);
pool.Commit();
}
// Create the Compute Pool of the Batch Account
public static async Task CreateBatchPoolIfNotExist(BatchClient batchClient, VirtualMachineConfiguration vmConfiguration, string vnetSubnetId)
{
Console.WriteLine("Creating pool [{0}]...", PoolId);
try
{
CloudPool pool = batchClient.PoolOperations.CreatePool(
poolId: PoolId,
targetDedicatedComputeNodes: PoolNodeCount,
virtualMachineSize: PoolVMSize,
virtualMachineConfiguration: vmConfiguration);
// Specify the application and version to install on the compute nodes
pool.ApplicationPackageReferences = new List <ApplicationPackageReference>
{
new ApplicationPackageReference {
ApplicationId = AppPackageName,
Version = AppPackageVersion
}
};
// Initial the first data disk for each VM in the pool
StartTask startTask = new StartTask("cmd /c Powershell -command \"Get-Disk | Where partitionstyle -eq 'raw' | sort number | Select-Object -first 1 |" +
" Initialize-Disk -PartitionStyle MBR -PassThru | New-Partition -UseMaximumSize -DriveLetter F |" +
" Format-Volume -FileSystem NTFS -NewFileSystemLabel data1 -Confirm:$false -Force\"");
startTask.MaxTaskRetryCount = 1;
startTask.UserIdentity = new UserIdentity(new AutoUserSpecification(AutoUserScope.Pool, ElevationLevel.Admin));
startTask.WaitForSuccess = true;
pool.StartTask = startTask;
// Create the Pool within the vnet subnet if it's specified.
if (vnetSubnetId != null)
{
pool.NetworkConfiguration = new NetworkConfiguration();
pool.NetworkConfiguration.SubnetId = vnetSubnetId;
}
await pool.CommitAsync();
await pool.RefreshAsync();
}
catch (BatchException be)
{
// Accept the specific error code PoolExists as that is expected if the pool already exists
if (be.RequestInformation?.BatchError?.Code == BatchErrorCodeStrings.PoolExists)
{
Console.WriteLine("The pool {0} already existed when we tried to create it", PoolId);
}
else
{
throw; // Any other exception is unexpected
}
}
}
// BATCH CLIENT SETUP - METHOD IMPLEMENTATIONS
// CreatePoolIfNoneExist(): Creates the Batch pool.
// batchClient: A BatchClient object.
// PoolId: ID of the CloudPool object to create.
// resourceFiles: A collection of ResourceFile objects representing blobs in a Storage account container.
private static void CreatePoolIfNoneExist(BatchClient batchClient, string poolId)
{
CloudPool pool = null;
try
{
Console.WriteLine("Creating pool [{0}]...", poolId);
ImageReference imageReference = new ImageReference(
publisher: "MicrosoftWindowsServer",
offer: "WindowsServer",
sku: "2012-R2-Datacenter",
version: "latest");
VirtualMachineConfiguration virtualMachineConfiguration = new VirtualMachineConfiguration(
imageReference: imageReference,
nodeAgentSkuId: "batch.node.windows amd64"
);
// Create an unbound pool. No pool is actually created in the Batch service until we call
// CloudPool.Commit(). This CloudPool instance is therefore considered "unbound," and we can
// modify its properties.
pool = batchClient.PoolOperations.CreatePool(
poolId: poolId,
targetDedicatedComputeNodes: 5, // 5 compute nodes
virtualMachineSize: "STANDARD_A1_v2", // Single-core, 1.75 GB memory, 225 GB disk
virtualMachineConfiguration: virtualMachineConfiguration); // Windows Server 2012 R2
// Specify the application and version to install on the compute nodes
// This assumes that a Windows 64-bit zipfile of ffmpeg has been added to Batch account
// with Application Id of "ffmpeg" and Version of "3.4".
// Download the zipfile https://ffmpeg.zeranoe.com/builds/win64/static/ffmpeg-3.4-win64-static.zip
// to upload as application package
pool.ApplicationPackageReferences = new List <ApplicationPackageReference>
{
new ApplicationPackageReference {
ApplicationId = appPackageId,
Version = appPackageVersion
}
};
pool.Commit();
}
catch (BatchException be)
{
// Accept the specific error code PoolExists as that is expected if the pool already exists
if (be.RequestInformation?.BatchError != null && be.RequestInformation.BatchError.Code == BatchErrorCodeStrings.PoolExists)
{
Console.WriteLine("The pool {0} already existed when we tried to create it", poolId);
}
else
{
throw; // Any other exception is unexpected
}
}
}
public static VirtualMachineConfiguration CreateVirtualMachineConfiguration(ImageReference imageReference)
{
VirtualMachineConfiguration config = new VirtualMachineConfiguration(
imageReference: imageReference,
nodeAgentSkuId: "batch.node.windows amd64");
config.DataDisks = new List <DataDisk>();
config.DataDisks.Add(new DataDisk(0, 2048, CachingType.ReadOnly, StorageAccountType.PremiumLrs));
return(config);
}
private static async Task CreateBatchPoolAsync(BatchClient batchClient, string poolId)
{
CloudPool pool = null;
Console.WriteLine("Creating Pool [{0}]", poolId);
// create image reference object for nodes
var imageReference = new ImageReference(
publisher: "MicrosoftWindowsServer",
offer: "WindowsServer",
sku: "2012-R2-Datacenter-smalldisk",
version: "latest");
// create VirtualMachineConfiguration object from image reference
var virtualMachineConfiguration = new VirtualMachineConfiguration(
imageReference,
"batch.node.windows amd64");
try
{
// create unbound pool, only created when we call CommitAsync()
pool = batchClient.PoolOperations.CreatePool(
poolId,
targetDedicatedComputeNodes: DedicatedNodeCount,
targetLowPriorityComputeNodes: LowPriorityNodeCount,
virtualMachineSize: PoolVMSize,
virtualMachineConfiguration: virtualMachineConfiguration
);
pool.ApplicationPackageReferences = new List <ApplicationPackageReference>
{
new ApplicationPackageReference
{
ApplicationId = appPackageId,
Version = appPackageVersion
}
};
await pool.CommitAsync();
}
catch (BatchException be)
{
// accept if poolexists
if (be.RequestInformation?.BatchError?.Code == BatchErrorCodeStrings.PoolExists)
{
Console.WriteLine("The pool [{0}] already existed", poolId);
}
else
{
throw;
}
}
}
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
{
var accountUrl = "***";
var accountName = "***";
var accountSecret = "**";
var jobtoDelete = "as_1bf795a45fda4ad8ab0f79bc24a942eb";
var sharedKeyCredentials = new BatchSharedKeyCredentials(accountUrl, accountName, accountSecret);
var imageReference = new ImageReference(
publisher: "microsoft-azure-batch",
offer: "ubuntu-server-container",
sku: "16-04-lts",
version: "latest");
var virtualMachineConfiguration = new VirtualMachineConfiguration(
imageReference,
"batch.node.ubuntu 16.04");
using (var bc = await BatchClient.OpenAsync(sharedKeyCredentials))
{
try
{
var newJob = bc.JobOperations.CreateJob("test", new PoolInformation()
{
AutoPoolSpecification = new AutoPoolSpecification()
{
AutoPoolIdPrefix = "test",
PoolLifetimeOption = PoolLifetimeOption.Job,
KeepAlive = false,
PoolSpecification = new PoolSpecification()
{
TargetDedicatedComputeNodes = 1,
VirtualMachineSize = "Standard_A1_v2",
VirtualMachineConfiguration = virtualMachineConfiguration
},
}
});
newJob.Commit();
// never works fails with forbidden
await bc.JobOperations.DeleteJobAsync(newJob.Id, cancellationToken : stoppingToken);
}
catch (Exception e)
{
Console.WriteLine($"Could not delete the job because {e}");
throw;
}
Console.WriteLine("Could delete job!");
}
}
private async Task CreatePoolIfNotExistAsync(BatchClient batchClient)
{
try
{
Console.WriteLine("Creating pool [{0}]...", PoolKeys.PoolId);
var imageReference = new ImageReference(
publisher: "MicrosoftWindowsServer",
offer: "WindowsServer",
sku: "2012-R2-Datacenter-smalldisk",
version: "latest");
var virtualMachineConfiguration =
new VirtualMachineConfiguration(
imageReference,
"batch.node.windows amd64");
// Create an unbound pool. No pool is actually created in the Batch service until we call
// CloudPool.Commit(). This CloudPool instance is therefore considered "unbound," and we can
// modify its properties.
var pool = batchClient.PoolOperations.CreatePool(
PoolKeys.PoolId,
targetDedicatedComputeNodes: PoolKeys.DedicatedNodeCount,
targetLowPriorityComputeNodes: PoolKeys.LowPriorityNodeCount,
virtualMachineSize: PoolKeys.PoolVMSize,
virtualMachineConfiguration: virtualMachineConfiguration);
pool.ApplicationPackageReferences = new List <ApplicationPackageReference>
{
new ApplicationPackageReference
{
ApplicationId = AppInformation.PackageId,
Version = AppInformation.Version
}
};
await pool.CommitAsync();
}
catch (BatchException be)
{
// Accept the specific error code PoolExists as that is expected if the pool already exists
if (be.RequestInformation?.BatchError?.Code == BatchErrorCodeStrings.PoolExists)
{
Console.WriteLine("The pool {0} already existed when we tried to create it", PoolKeys.PoolId);
}
else
{
throw; // Any other exception is unexpected
}
}
}
private static CloudPool CreatePool(BatchClient batchCli, string poolId)
{
var ubuntuImageDetails = IaasLinuxPoolFixture.GetUbuntuImageDetails(batchCli);
VirtualMachineConfiguration virtualMachineConfiguration = new VirtualMachineConfiguration(
ubuntuImageDetails.ImageReference,
nodeAgentSkuId: ubuntuImageDetails.NodeAgentSku.Id);
return(batchCli.PoolOperations.CreatePool(
poolId: poolId,
virtualMachineSize: PoolFixture.VMSize,
virtualMachineConfiguration: virtualMachineConfiguration,
targetDedicatedComputeNodes: 1));
}
/// <summary>
/// Constructs an Azure Batch PoolInformation instance
/// </summary>
/// <param name="image">The image name for the current <see cref="TesTask"/></param>
/// <param name="vmSize">The Azure VM sku</param>
/// <param name="preemptible">True if preemptible machine should be used</param>
/// <returns></returns>
private async Task <PoolInformation> CreatePoolInformation(string image, string vmSize, bool preemptible)
{
var vmConfig = new VirtualMachineConfiguration(
imageReference: new ImageReference("ubuntu-server-container", "microsoft-azure-batch", "16-04-lts", "latest"),
nodeAgentSkuId: "batch.node.ubuntu 16.04");
var containerRegistryInfo = await azureProxy.GetContainerRegistryInfoAsync(image);
if (containerRegistryInfo != null)
{
var containerRegistry = new ContainerRegistry(
userName: containerRegistryInfo.Username,
registryServer: containerRegistryInfo.RegistryServer,
password: containerRegistryInfo.Password);
// Download private images at node startup, since those cannot be downloaded in the main task that runs multiple containers.
// Doing this also requires that the main task runs inside a container, hence downloading the "docker" image (contains docker client) as well.
vmConfig.ContainerConfiguration = new ContainerConfiguration
{
ContainerImageNames = new List <string> {
image, DockerInDockerImageName
},
ContainerRegistries = new List <ContainerRegistry> {
containerRegistry
}
};
}
var poolSpecification = new PoolSpecification
{
VirtualMachineConfiguration = vmConfig,
VirtualMachineSize = vmSize,
ResizeTimeout = TimeSpan.FromMinutes(30),
TargetLowPriorityComputeNodes = preemptible ? 1 : 0,
TargetDedicatedComputeNodes = preemptible ? 0 : 1
};
var poolInformation = new PoolInformation
{
AutoPoolSpecification = new AutoPoolSpecification
{
AutoPoolIdPrefix = "TES",
PoolLifetimeOption = PoolLifetimeOption.Job,
PoolSpecification = poolSpecification,
KeepAlive = false
}
};
return(poolInformation);
}
/// <summary>
/// This method can create a pool, it is not used in our example.
/// The code remains for documentation purposes.
/// </summary>
public static void PreparePool()
{
BatchClient batchClient = PrepareConnection();
if (Environment.GetEnvironmentVariable("REGISTRYNAME") != null)
{
ContainerRegistry containerRegistry = new ContainerRegistry(
registryServer: Environment.GetEnvironmentVariable("REGISTRYNAME"),
userName: Environment.GetEnvironmentVariable("REGISTRYUSERNAME"),
password: Environment.GetEnvironmentVariable("REGISTRYPASSWORD")
);
// Create container configuration, prefetching Docker images from the container registry
ContainerConfiguration containerConfig = new ContainerConfiguration()
{
ContainerImageNames = new List <string>()
{
Environment.GetEnvironmentVariable("WORKERIMAGENAME")
},
ContainerRegistries = new List <ContainerRegistry>
{
containerRegistry
}
};
ImageReference imageReference = new ImageReference(
publisher: "microsoft-azure-batch",
offer: "ubuntu-server-container",
sku: "16-04-lts",
version: "latest");
// VM configuration
VirtualMachineConfiguration virtualMachineConfiguration = new VirtualMachineConfiguration(
imageReference: imageReference,
nodeAgentSkuId: "batch.node.ubuntu 16.04")
{
ContainerConfiguration = containerConfig,
};
//Create pool
CloudPool pool = batchClient.PoolOperations.CreatePool(
poolId: "docker",
targetDedicatedComputeNodes: 1,
virtualMachineSize: "Standard_A2_v2",
virtualMachineConfiguration: virtualMachineConfiguration);
pool.Commit();
}
}
/// <summary>
/// Creates a <see cref="CloudPool"/> associated with the specified Batch account. If an existing pool with the
/// specified ID is found, the pool is resized to match the specified node count.
/// </summary>
/// <param name="batchClient">A fully initialized <see cref="BatchClient"/>.</param>
/// <param name="poolId">The ID of the <see cref="CloudPool"/>.</param>
/// <param name="nodeSize">The size of the nodes within the pool.</param>
/// <param name="nodeCount">The number of nodes to create within the pool.</param>
/// <param name="maxTasksPerNode">The maximum number of tasks to run concurrently on each node.</param>
/// <returns>A bound <see cref="CloudPool"/> with the specified properties.</returns>
public async Task <CloudPool> CreatePoolIfNotExistAsync(BatchClient batchClient, string poolId, string nodeSize = null, int nodeCount = 0, int maxTasksPerNode = 0)
{
try
{
// Provide a reference to an Azure Marketplace image for
// "Windows Server 2016 Datacenter with Containers"
ImageReference imageReference = new ImageReference(
publisher: "MicrosoftWindowsServer",
offer: "WindowsServer",
sku: "2016-Datacenter-with-Containers",
version: "latest");
ContainerConfiguration containerConfig = new ContainerConfiguration
{
ContainerImageNames = new List <string> {
"ContainerImage"
}
};
// VM configuration
VirtualMachineConfiguration virtualMachineConfiguration = new VirtualMachineConfiguration(
imageReference: imageReference,
nodeAgentSkuId: "batch.node.windows amd64");
virtualMachineConfiguration.ContainerConfiguration = containerConfig;
// You can learn more about os families and versions at:
// https://azure.microsoft.com/en-us/documentation/articles/cloud-services-guestos-update-matrix/
CloudPool pool = batchClient.PoolOperations.CreatePool();
pool.Id = poolId;
pool.TargetDedicatedComputeNodes = 1;
pool.VirtualMachineSize = "Standard_A1";
pool.VirtualMachineConfiguration = virtualMachineConfiguration;
pool.StartTask = new StartTask
{
CommandLine = "cmd /C echo start task"
};
await this.CreatePoolIfNotExistAsync(batchClient, pool).ConfigureAwait(continueOnCapturedContext: false);
}
catch (Exception ex)
{
}
return(await batchClient.PoolOperations.GetPoolAsync(poolId).ConfigureAwait(continueOnCapturedContext: false));
}
private void CreateBatchPool(VirtualMachineConfiguration vmc, string vmsize = "Standard_E2_v3")
{
try
{
CloudPool pool = Client.PoolOperations.CreatePool(PoolId, vmsize, vmc, PoolSize);
pool.Commit();
}
catch (BatchException be)
{
if (be.RequestInformation?.BatchError?.Code == BatchErrorCodeStrings.PoolExists)
{
Logger.Warning("The pool " + PoolId + " already exist");
}
else
{
throw;
}
}
}
/// <summary>
/// Creates a <see cref="CloudPool"/> with the specified id and configures its StartTask with the
/// specified <see cref="ResourceFile"/> collection.
/// </summary>
/// <param name="batchClient">A <see cref="BatchClient"/>.</param>
/// <param name="poolId">The id of the <see cref="CloudPool"/> to create.</param>
/// <param name="resourceFiles">A collection of <see cref="ResourceFile"/> objects representing blobs within
/// a Storage account container. The StartTask will download these files from Storage prior to execution.</param>
/// <returns>A <see cref="System.Threading.Tasks.Task"/> object that represents the asynchronous operation.</returns>
private static async Task CreatePoolAsync(BatchClient batchClient, string poolId, IList <ResourceFile> resourceFiles)
{
Console.WriteLine("Creating pool [{0}]...", poolId);
// Set the OS and version of the VM image
ImageReference imageReference = new ImageReference(
publisher: "MicrosoftWindowsServer",
offer: "WindowsServer",
sku: "2016-Datacenter",
version: "latest");
VirtualMachineConfiguration virtualMachineConfiguration =
new VirtualMachineConfiguration(
imageReference: imageReference,
nodeAgentSkuId: "batch.node.windows amd64");
// Create the unbound pool. Until we call CloudPool.Commit() or CommitAsync(), no pool is actually created in the
// Batch service. This CloudPool instance is therefore considered "unbound," and we can modify its properties.
CloudPool pool = batchClient.PoolOperations.CreatePool(
poolId: poolId,
targetDedicatedComputeNodes: nodeNumberDedicated, // Number of dedicated compute nodes
targetLowPriorityComputeNodes: nodeNumberLowPriority, // Number of low priority compute nodes
virtualMachineSize: NodeSize, // Node instance size
virtualMachineConfiguration: virtualMachineConfiguration); // Windows Server 2012 R2
// Create and assign the StartTask that will be executed when compute nodes join the pool.
// In this case, we copy the StartTask's resource files (that will be automatically downloaded
// to the node by the StartTask) into the shared directory that all tasks will have access to.
pool.StartTask = new StartTask
{
// Specify a command line for the StartTask that copies the task application files to the
// node's shared directory. Every compute node in a Batch pool is configured with a number
// of pre-defined environment variables that can be referenced by commands or applications
// run by tasks.
CommandLine = "cmd /c starttask.cmd",
ResourceFiles = resourceFiles,
WaitForSuccess = true
};
await pool.CommitAsync();
}
public static async Task <string> CreateBatchPoolAndImportJob([ActivityTrigger] ImportRequest request, ILogger log)
{
var azureServiceTokenProvider = new AzureServiceTokenProvider();
// Get a Batch client using function identity
BatchTokenCredentials batchCred = new BatchTokenCredentials(request.BatchAccountUrl, await azureServiceTokenProvider.GetAccessTokenAsync("https://batch.core.windows.net/"));
string jobId = request.TargetSqlServerName + "-Import-" + DateTime.UtcNow.ToString("MMddHHmmss");
using (BatchClient batchClient = BatchClient.Open(batchCred))
{
ImageReference imageReference = CreateImageReference();
VirtualMachineConfiguration vmConfiguration = CreateVirtualMachineConfiguration(imageReference);
await CreateBatchPoolIfNotExist(batchClient, vmConfiguration, request.VNetSubnetId);
await CreateBatchJob(batchClient, jobId, log);
}
return(jobId);
}
private static async Task PoolCreation(BatchClient p_batchClient, string p_poolId)
{
Console.WriteLine("Creating the pool of virtual machines");
try
{
ImageReference demo_image = new ImageReference(
publisher: "MicrosoftWindowsServer",
offer: "WindowsServer",
sku: "2016-Datacenter",
version: "latest");
VirtualMachineConfiguration demo_configuration =
new VirtualMachineConfiguration(
imageReference: demo_image,
nodeAgentSkuId: "batch.node.windows amd64");
CloudPool demo_pool = null;
demo_pool = p_batchClient.PoolOperations.CreatePool(
poolId: p_poolId,
targetDedicatedComputeNodes: 1,
targetLowPriorityComputeNodes: 0,
virtualMachineSize: "STANDARD_A1_v2",
virtualMachineConfiguration: demo_configuration);
demo_pool.ApplicationPackageReferences = new List <ApplicationPackageReference>
{
new ApplicationPackageReference
{
ApplicationId = demo_packageid,
Version = demo_packageversion
}
};
await demo_pool.CommitAsync();
}
catch (BatchException pool_error)
{
Console.WriteLine(pool_error.Message);
}
}
/// <summary>
/// Constructs an Azure Batch PoolInformation instance
/// </summary>
/// <param name="image">The image name for the current <see cref="TesTask"/></param>
/// <param name="vmSize">The Azure VM sku</param>
/// <param name="preemptible">True if preemptible machine should be used</param>
/// <returns></returns>
private async Task <PoolInformation> CreatePoolInformation(string image, string vmSize, bool preemptible)
{
var vmConfig = new VirtualMachineConfiguration(
imageReference: new ImageReference("ubuntu-server-container", "microsoft-azure-batch", "16-04-lts", "latest"),
nodeAgentSkuId: "batch.node.ubuntu 16.04");
var containerRegistry = await GetContainerRegistry(image);
vmConfig.ContainerConfiguration = new ContainerConfiguration
{
ContainerImageNames = new List <string> {
image
},
ContainerRegistries = containerRegistry != null ? new List <ContainerRegistry> {
containerRegistry
} : null
};
var poolSpecification = new PoolSpecification
{
VirtualMachineConfiguration = vmConfig,
VirtualMachineSize = vmSize,
ResizeTimeout = TimeSpan.FromMinutes(30),
TargetLowPriorityComputeNodes = preemptible ? 1 : 0,
TargetDedicatedComputeNodes = preemptible ? 0 : 1
};
var poolInformation = new PoolInformation
{
AutoPoolSpecification = new AutoPoolSpecification
{
AutoPoolIdPrefix = "TES",
PoolLifetimeOption = PoolLifetimeOption.Job,
PoolSpecification = poolSpecification,
KeepAlive = false
}
};
return(poolInformation);
}
public void Execute()
{
string reqfile = _config.ResourceFolder + _config.ResourceName;
//> CREATE
try
{
ResourceFile req = CreateResourceFile(reqfile);
BatchTokenCredentials cred = new BatchTokenCredentials(_config.BatchAccountUrl, GetToken());
Client = BatchClient.Open(cred);
ImageReference imgref = new ImageReference(_config.VmImage);
VirtualMachineConfiguration vmc = new VirtualMachineConfiguration(imageReference: imgref, nodeAgentSkuId: "batch.node.windows amd64");
CreateBatchPool(vmc, _config.VmSKU);
// Create Job
CloudJob job = Client.JobOperations.CreateJob();
job.Id = _jobId;
job.PoolInformation = new PoolInformation {
PoolId = _poolId
};
job.Commit();
// Create Tasks
List <CloudTask> tasks = new List <CloudTask>();
string taskId = "task" + System.DateTime.Now.Ticks.ToString();
string cmd = String.Format("cmd /c type {0}", req.FilePath);
CloudTask task = new CloudTask(taskId, cmd);
task.ResourceFiles = new List <ResourceFile> {
req
};
tasks.Add(task);
Client.JobOperations.AddTask(_jobId, tasks);
}
catch (Exception x)
{
//TODO: Log Somewhere
throw;
}
}
protected CloudPool CreatePool()
{
CloudPool currentPool = this.FindPoolIfExists();
// gotta create a new pool
if (currentPool == null)
{
var ubuntuImageDetails = GetUbuntuImageDetails(this.client);
VirtualMachineConfiguration virtualMachineConfiguration = new VirtualMachineConfiguration(
ubuntuImageDetails.ImageReference,
nodeAgentSkuId: ubuntuImageDetails.NodeAgentSku.Id);
currentPool = this.client.PoolOperations.CreatePool(
poolId: this.PoolId,
virtualMachineSize: VMSize,
virtualMachineConfiguration: virtualMachineConfiguration,
targetDedicatedComputeNodes: 1);
currentPool.Commit();
}
return(WaitForPoolAllocation(this.client, this.PoolId));
}
static void Main(string[] args)
{
var config = new VirtualMachineConfiguration(
true,
true,
true,
true,
true);
using (var container = new Win64Container(config))
{
var intType = container.VirtualMachine.TypeProvider.FindPrimitiveType(PrimitiveTypes.Int);
var voidType = container.VirtualMachine.TypeProvider.FindPrimitiveType(PrimitiveTypes.Void);
var pointMetadata = new ClassMetadata("Point");
pointMetadata.DefineField(new FieldDefinition("x", intType, AccessModifier.Public));
pointMetadata.DefineField(new FieldDefinition("y", intType, AccessModifier.Public));
pointMetadata.CreateFields();
container.VirtualMachine.ClassMetadataProvider.Add(pointMetadata);
var pointType = container.VirtualMachine.TypeProvider.FindClassType("Point");
var pointArrayType = container.VirtualMachine.TypeProvider.FindArrayType(pointType);
void Println(long objectReference)
{
Console.WriteLine($"0x{objectReference.ToString("x8")}");
}
container.VirtualMachine.Binder.Define(FunctionDefinition.NewExternal <PrintlnPoint>(
"std.println",
new List <BaseType>()
{
pointType
},
voidType,
Println));
var constructorFunction = new ManagedFunction(
new FunctionDefinition(".constructor", new List <BaseType>(), voidType, pointType, true),
new List <BaseType>(),
new List <Instruction>()
{
//new Instruction(OpCodes.LoadArgument, 0),
//new Instruction(OpCodes.LoadInt, 1337),
//new Instruction(OpCodes.StoreField, "Point::x"),
new Instruction(OpCodes.Return)
});
var mainFunction = new ManagedFunction(
new FunctionDefinition("main", new List <BaseType>(), intType),
new List <BaseType>()
{
pointArrayType
},
//new List<BaseType>() { pointType },
new List <Instruction>
{
//new Instruction(OpCodes.LoadInt, 10),
//new Instruction(OpCodes.NewArray, pointType.Name),
//new Instruction(OpCodes.StoreLocal, 0),
//new Instruction(OpCodes.LoadLocal, 0),
//new Instruction(OpCodes.LoadInt, 0),
//new Instruction(OpCodes.NewObject, ".constructor", pointType, new List<BaseType>()),
//new Instruction(OpCodes.StoreElement, pointType.Name),
//new Instruction(OpCodes.Call, "std.gc.collect", new List<BaseType>()),
//new Instruction(OpCodes.LoadInt, 0),
//new Instruction(OpCodes.Return),
new Instruction(OpCodes.LoadInt, 10),
new Instruction(OpCodes.NewArray, pointType.Name),
new Instruction(OpCodes.StoreLocal, 0),
new Instruction(OpCodes.NewObject, ".constructor", pointType, new List <BaseType>()),
new Instruction(OpCodes.Pop),
new Instruction(OpCodes.Call, "std.gc.collect", new List <BaseType>()),
new Instruction(OpCodes.Call, "std.gc.collect", new List <BaseType>()),
new Instruction(OpCodes.LoadInt, 0),
new Instruction(OpCodes.Return),
});
var functions = new List <ManagedFunction>()
{
mainFunction,
constructorFunction
};
container.VirtualMachine.LoadFunctionsAsAssembly(functions);
container.VirtualMachine.Compile();
foreach (var function in functions)
{
var disassembler = new Disassembler(
container.VirtualMachine.Compiler.GetCompilationData(function),
x => new Compiler.Win64.Disassembler(x));
Console.WriteLine(disassembler.Disassemble());
}
int returnValue = container.VirtualMachine.GetEntryPoint()();
Console.WriteLine(returnValue);
}
Console.ReadLine();
}
static void Main(string[] args)
{
string storageConnectionString =
$"DefaultEndpointsProtocol=https;AccountName={StorageAccountName};AccountKey={StorageAccountKey}";
// Retrieve the storage account
CloudStorageAccount storageAccount = CloudStorageAccount.Parse(storageConnectionString);
// Create the blob client
CloudBlobClient blobClient = storageAccount.CreateCloudBlobClient();
// Upload the input files to blob storage
const string inputContainerName = "batchinput";
List <string> inputFilePaths = new List <string>
{
"taskdata0.txt",
"taskdata1.txt",
"taskdata2.txt"
};
List <ResourceFile> inputFiles = new List <ResourceFile>();
foreach (string filePath in inputFilePaths)
{
inputFiles.Add(UploadFileToContainer(blobClient, inputContainerName, filePath));
}
// Get a SAS Url for the output container
const string outputContainerName = "batchoutput";
string outputContainerSasUrl = GetOutputContainerSasUrl(blobClient, outputContainerName);
// Specify a container registry
ContainerRegistry containerRegistry = new ContainerRegistry(
registryServer: RegistryServer,
userName: RegistryUserName,
password: RegistryPassword);
// Create container configuration, prefetching Docker images from the container registry
ContainerConfiguration containerConfig = new ContainerConfiguration()
{
ContainerImageNames = ContainerImageNames,
ContainerRegistries = new List <ContainerRegistry> {
containerRegistry
}
};
// Create the virtual machine image reference - make sure to choose an image that supports containers
ImageReference imageReference = new ImageReference(
publisher: "MicrosoftWindowsServer",
offer: "WindowsServer",
sku: "2016-datacenter-with-containers",
version: "latest");
// Create the virtual machine configuration for the pool and set the container configuration
VirtualMachineConfiguration virtualMachineConfiguration = new VirtualMachineConfiguration(
imageReference: imageReference,
nodeAgentSkuId: "batch.node.windows amd64");
virtualMachineConfiguration.ContainerConfiguration = containerConfig;
BatchSharedKeyCredentials cred = new BatchSharedKeyCredentials(BatchAccountUrl, BatchAccountName, BatchAccountKey);
using (BatchClient batchClient = BatchClient.Open(cred))
{
Console.WriteLine("Creating pool [{0}]...", PoolId);
try
{
CloudPool pool = batchClient.PoolOperations.CreatePool(
poolId: PoolId,
targetDedicatedComputeNodes: PoolNodeCount,
virtualMachineSize: PoolVMSize,
virtualMachineConfiguration: virtualMachineConfiguration);
pool.Commit();
}
catch (BatchException be)
{
// Accept the specific error code PoolExists as that is expected if the pool already exists
if (be.RequestInformation?.BatchError?.Code == BatchErrorCodeStrings.PoolExists)
{
Console.WriteLine("The pool {0} already existed when we tried to create it", PoolId);
}
else
{
throw; // Any other exception is unexpected
}
}
Console.WriteLine("Creating job [{0}]...", JobId);
CloudJob job = null;
try
{
job = batchClient.JobOperations.CreateJob();
job.Id = JobId;
job.PoolInformation = new PoolInformation {
PoolId = PoolId
};
// Add job preparation task to remove existing Docker image
Console.WriteLine("Adding job preparation task to job [{0}]...", JobId);
string jobPreparationCmdLine = $"cmd /c docker rmi -f {ContainerImageNames[0]}:latest";
JobPreparationTask jobPreparationTask = new JobPreparationTask(jobPreparationCmdLine)
{
UserIdentity = new UserIdentity(new AutoUserSpecification(elevationLevel: ElevationLevel.Admin, scope: AutoUserScope.Task))
};
job.JobPreparationTask = jobPreparationTask;
job.Commit();
}
catch (BatchException be)
{
// Accept the specific error code JobExists as that is expected if the job already exists
if (be.RequestInformation?.BatchError?.Code == BatchErrorCodeStrings.JobExists)
{
Console.WriteLine("The job {0} already existed when we tried to create it", JobId);
}
else
{
throw; // Any other exception is unexpected
}
}
if (job != null)
{
// Create a collection to hold the tasks that we'll be adding to the job
Console.WriteLine("Adding {0} tasks to job [{1}]...", inputFiles.Count, JobId);
List <CloudTask> tasks = new List <CloudTask>();
// Create each of the tasks to process one of the input files.
for (int i = 0; i < inputFiles.Count; i++)
{
string taskId = String.Format("Task{0}", i);
string inputFilename = inputFiles[i].FilePath;
string outputFileName = string.Format("out{0}", inputFilename);
// Override the default entrypoint of the container
string taskCommandLine = string.Format("C:\\ReadWriteFile\\ReadWriteFile.exe {0} {1}", inputFilename, outputFileName);
// Specify the container the task will run
TaskContainerSettings cmdContainerSettings = new TaskContainerSettings(
imageName: "nimccollftacr.azurecr.io/batch/readwritefile"
);
CloudTask task = new CloudTask(taskId, taskCommandLine);
task.ContainerSettings = cmdContainerSettings;
// Set the resource files and output files for the task
task.ResourceFiles = new List <ResourceFile> {
inputFiles[i]
};
task.OutputFiles = new List <OutputFile>
{
new OutputFile(
filePattern: outputFileName,
destination: new OutputFileDestination(new OutputFileBlobContainerDestination(containerUrl: outputContainerSasUrl, path: outputFileName)),
uploadOptions: new OutputFileUploadOptions(OutputFileUploadCondition.TaskCompletion))
};
// You must elevate the identity of the task in order to run a container
task.UserIdentity = new UserIdentity(new AutoUserSpecification(elevationLevel: ElevationLevel.Admin, scope: AutoUserScope.Task));
tasks.Add(task);
}
// Add all tasks to the job.
batchClient.JobOperations.AddTask(JobId, tasks);
// Monitor task success/failure, specifying a maximum amount of time to wait for the tasks to complete.
TimeSpan timeout = TimeSpan.FromMinutes(30);
Console.WriteLine("Monitoring all tasks for 'Completed' state, timeout in {0}...", timeout);
IEnumerable <CloudTask> addedTasks = batchClient.JobOperations.ListTasks(JobId);
batchClient.Utilities.CreateTaskStateMonitor().WaitAll(addedTasks, TaskState.Completed, timeout);
Console.WriteLine("All tasks reached state Completed.");
// Print task output
Console.WriteLine();
Console.WriteLine("Printing task output...");
IEnumerable <CloudTask> completedtasks = batchClient.JobOperations.ListTasks(JobId);
foreach (CloudTask task in completedtasks)
{
string nodeId = String.Format(task.ComputeNodeInformation.ComputeNodeId);
Console.WriteLine("Task: {0}", task.Id);
Console.WriteLine("Node: {0}", nodeId);
Console.WriteLine("Standard out:");
Console.WriteLine(task.GetNodeFile(Constants.StandardOutFileName).ReadAsString());
}
}
// Clean up Batch resources (if the user so chooses)
Console.WriteLine();
Console.Write("Delete job? [yes] no: ");
string response = Console.ReadLine().ToLower();
if (response != "n" && response != "no")
{
batchClient.JobOperations.DeleteJob(JobId);
}
Console.Write("Delete pool? [yes] no: ");
response = Console.ReadLine().ToLower();
if (response != "n" && response != "no")
{
batchClient.PoolOperations.DeletePool(PoolId);
}
}
}
private static async Task MainAsync()
{
if (String.IsNullOrEmpty(BatchAccountName) ||
String.IsNullOrEmpty(BatchAccountKey) ||
String.IsNullOrEmpty(BatchAccountUrl) ||
String.IsNullOrEmpty(StorageAccountName) ||
String.IsNullOrEmpty(StorageAccountKey))
{
throw new InvalidOperationException("One or more account credential strings have not been populated. Please ensure that your Batch and Storage account credentials have been specified.");
}
try
{
Console.WriteLine("Sample start: {0}", DateTime.Now);
Console.WriteLine();
Stopwatch timer = new Stopwatch();
timer.Start();
// Construct the Storage account connection string
string storageConnectionString = String.Format("DefaultEndpointsProtocol=https;AccountName={0};AccountKey={1}", StorageAccountName, StorageAccountKey);
// Retrieve the storage account
CloudStorageAccount storageAccount = CloudStorageAccount.Parse(storageConnectionString);
// Create the blob client, for use in obtaining references to blob storage containers
CloudBlobClient blobClient = storageAccount.CreateCloudBlobClient();
// Use the blob client to create the input container in Azure Storage
const string appContainerName = "application";
const string inputContainerName = "input";
const string outputContainerName = "output";
await CreateContainerIfNotExistAsync(blobClient, appContainerName);
await CreateContainerIfNotExistAsync(blobClient, inputContainerName);
await CreateContainerIfNotExistAsync(blobClient, outputContainerName);
// The collection of data files that are to be processed by the tasks
// The collection of data files that are to be processed by the tasks
List <string> inputFilePaths = new List <string>
{
@"..\..\taskdata1.txt",
@"..\..\taskdata2.txt",
@"..\..\taskdata3.txt"
};
// Paths to the executable and its dependencies that will be executed by the tasks
List <string> applicationFilePaths = new List <string>
{
// The DotNetTutorial project includes a project reference to TaskApplication, allowing us to
// determine the path of the task application binary dynamically
typeof(WordCounter.Program).Assembly.Location,
"Microsoft.WindowsAzure.Storage.dll"
};
// Upload the application and its dependencies to Azure Storage. This is the application that will
// process the data files, and will be executed by each of the tasks on the compute nodes.
List <ResourceFile> applicationFiles = await UploadFilesToContainerAsync(blobClient, appContainerName, applicationFilePaths);
// Upload the data files. This is the data that will be processed by each of the tasks that are
// executed on the compute nodes within the pool.
List <ResourceFile> inputFiles = await UploadFilesToContainerAsync(blobClient, inputContainerName, inputFilePaths);
// Obtain a shared access signature that provides write access to the output container to which
// the tasks will upload their output.
string outputContainerSasUrl = GetContainerSasUrl(blobClient, outputContainerName, SharedAccessBlobPermissions.Write);
BatchSharedKeyCredentials cred = new BatchSharedKeyCredentials(BatchAccountUrl, BatchAccountName, BatchAccountKey);
using (BatchClient batchClient = BatchClient.Open(cred))
{
// Create a Batch pool, VM configuration, Windows Server image
Console.WriteLine("Creating pool [{0}]...", PoolId);
ImageReference imageReference = new ImageReference(
publisher: "MicrosoftWindowsServer",
offer: "WindowsServer",
sku: "2012-R2-datacenter-smalldisk",
version: "latest");
VirtualMachineConfiguration virtualMachineConfiguration =
new VirtualMachineConfiguration(
imageReference: imageReference,
nodeAgentSkuId: "batch.node.windows amd64");
try
{
CloudPool pool = batchClient.PoolOperations.CreatePool(
poolId: PoolId,
targetDedicatedComputeNodes: PoolNodeCount,
virtualMachineSize: PoolVMSize,
virtualMachineConfiguration: virtualMachineConfiguration);
// Create and assign the StartTask that will be executed when compute nodes join the pool.
// In this case, we copy the StartTask's resource files (that will be automatically downloaded
// to the node by the StartTask) into the shared directory that all tasks will have access to.
pool.StartTask = new StartTask
{
// Specify a command line for the StartTask that copies the task application files to the
// node's shared directory. Every compute node in a Batch pool is configured with a number
// of pre-defined environment variables that can be referenced by commands or applications
// run by tasks.
// Since a successful execution of robocopy can return a non-zero exit code (e.g. 1 when one or
// more files were successfully copied) we need to manually exit with a 0 for Batch to recognize
// StartTask execution success.
CommandLine = "cmd /c (robocopy %AZ_BATCH_TASK_WORKING_DIR% %AZ_BATCH_NODE_SHARED_DIR%) ^& IF %ERRORLEVEL% LEQ 1 exit 0",
ResourceFiles = applicationFiles,
WaitForSuccess = true
};
pool.Commit();
}
catch (BatchException be)
{
// Accept the specific error code PoolExists as that is expected if the pool already exists
if (be.RequestInformation?.BatchError?.Code == BatchErrorCodeStrings.PoolExists)
{
Console.WriteLine("The pool {0} already existed when we tried to create it", PoolId);
}
else
{
throw; // Any other exception is unexpected
}
}
// Create the job that will run the tasks.
await CreateJobAsync(batchClient, JobId, PoolId);
// Add the tasks to the job. We need to supply a container shared access signature for the
// tasks so that they can upload their output to Azure Storage.
await AddTasksAsync(batchClient, JobId, inputFiles, outputContainerSasUrl);
// Monitor task success/failure, specifying a maximum amount of time to wait for the tasks to complete
await MonitorTasks(batchClient, JobId, TimeSpan.FromMinutes(30));
// Download the task output files from the output Storage container to a local directory
await DownloadBlobsFromContainerAsync(blobClient, outputContainerName, Environment.GetEnvironmentVariable("TEMP"));
// Clean up Storage resources
await DeleteContainerAsync(blobClient, appContainerName);
await DeleteContainerAsync(blobClient, inputContainerName);
await DeleteContainerAsync(blobClient, outputContainerName);
// Print out some timing info
timer.Stop();
Console.WriteLine();
Console.WriteLine("Sample end: {0}", DateTime.Now);
Console.WriteLine("Elapsed time: {0}", timer.Elapsed);
// Clean up Batch resources (if the user so chooses)
Console.WriteLine();
Console.Write("Delete job? [yes] no: ");
string response = Console.ReadLine().ToLower();
if (response != "n" && response != "no")
{
await batchClient.JobOperations.DeleteJobAsync(JobId);
}
Console.Write("Delete pool? [yes] no: ");
response = Console.ReadLine().ToLower();
if (response != "n" && response != "no")
{
await batchClient.PoolOperations.DeletePoolAsync(PoolId);
}
}
}
catch
{
}
}
static void Main(string[] args)
{
if (String.IsNullOrEmpty(BatchAccountName) ||
String.IsNullOrEmpty(BatchAccountKey) ||
String.IsNullOrEmpty(BatchAccountUrl) ||
String.IsNullOrEmpty(StorageAccountName) ||
String.IsNullOrEmpty(StorageAccountKey))
{
throw new InvalidOperationException("One or more account credential strings have not been populated. Please ensure that your Batch and Storage account credentials have been specified.");
}
try
{
Console.WriteLine("Sample start: {0}", DateTime.Now);
Console.WriteLine();
Stopwatch timer = new Stopwatch();
timer.Start();
// Create the blob client, for use in obtaining references to blob storage containers
(CloudBlobClient, CloudStorageAccount)cc = CreateCloudBlobClient(StorageAccountName, StorageAccountKey);
CloudBlobClient blobClient = cc.Item1;
CloudStorageAccount linkedStorageAccount = cc.Item2;
// Use the blob client to create the input container in Azure Storage
const string inputContainerName = "input";
CloudBlobContainer container = blobClient.GetContainerReference(inputContainerName);
container.CreateIfNotExistsAsync().Wait();
// The collection of data files that are to be processed by the tasks
List <string> inputFilePaths = new List <string> {
"input.txt"
};
// Upload the data files to Azure Storage. This is the data that will be processed by each of the tasks that are
// executed on the compute nodes within the pool.
UploadFileToContainer(blobClient, inputContainerName, "input.txt");
// Get a Batch client using account creds
BatchSharedKeyCredentials cred = new BatchSharedKeyCredentials(BatchAccountUrl, BatchAccountName, BatchAccountKey);
using (BatchClient batchClient = BatchClient.Open(cred))
{
Console.WriteLine("Creating pool [{0}]...", PoolId);
// Create a Windows Server image, VM configuration, Batch pool
ImageReference imageReference = CreateImageReference();
VirtualMachineConfiguration vmConfiguration = CreateVirtualMachineConfiguration(imageReference);
CreateBatchPool(batchClient, vmConfiguration);
// Create a Batch job
Console.WriteLine("Creating job [{0}]...", JobId);
try
{
CloudJob job = batchClient.JobOperations.CreateJob();
job.Id = JobId;
job.PoolInformation = new PoolInformation {
PoolId = PoolId
};
job.UsesTaskDependencies = true;
// Create the blob storage container for the outputs.
Task t1 = job.PrepareOutputStorageAsync(linkedStorageAccount);
Task.WaitAll(t1);
string containerName = job.OutputStorageContainerName();
string containerUrl = job.GetOutputStorageContainerUrl(linkedStorageAccount);
job.CommonEnvironmentSettings = new[] { new EnvironmentSetting("JOB_CONTAINER_URL", containerUrl) };
job.Commit();
}
catch (BatchException be)
{
// Accept the specific error code JobExists as that is expected if the job already exists
if (be.RequestInformation?.BatchError?.Code == BatchErrorCodeStrings.JobExists)
{
Console.WriteLine("The job {0} already existed when we tried to create it", JobId);
}
else
{
throw; // Any other exception is unexpected
}
}
// Create a collection to hold the tasks that we'll be adding to the job
List <CloudTask> tasks = new List <CloudTask>();
// Create each of the tasks to process one of the input files.
string lastTaskId = null;
for (int i = 0; i < 3; i++)
{
string taskId = String.Format("Task{0}", i);
CloudTask task;
// first task
if (lastTaskId == null)
{
//task = new CloudTask(taskId, $"cmd /c %AZ_BATCH_APP_PACKAGE_TESTBATCHAPP%\\testbatchapp.exe {i}")
task = new CloudTask(taskId, $"cmd /c %AZ_BATCH_APP_PACKAGE_HPCAPP%\\testbatchapp.exe {i}")
{
ApplicationPackageReferences = new List <ApplicationPackageReference>
{
new ApplicationPackageReference
{
//ApplicationId = "testbatchapp",
ApplicationId = "hpcapp" //,
//Version = "1.1"
}
}
};
}
else
{
// task = new CloudTask(taskId, $"cmd /c %AZ_BATCH_APP_PACKAGE_TESTBATCHAPP%\\testbatchapp.exe {i}")
task = new CloudTask(taskId, $"cmd /c %AZ_BATCH_APP_PACKAGE_HPCAPP%\\testbatchapp.exe {i}")
{
ApplicationPackageReferences = new List <ApplicationPackageReference>
{
new ApplicationPackageReference
{
// ApplicationId = "testbatchapp",
ApplicationId = "hpcapp" //,
//Version = "1.1"
}
}
};
task.DependsOn = TaskDependencies.OnId(lastTaskId);
}
lastTaskId = taskId;
tasks.Add(task);
}
// Add all tasks to the job.
batchClient.JobOperations.AddTask(JobId, tasks);
// Monitor task success/failure, specifying a maximum amount of time to wait for the tasks to complete.
TimeSpan timeout = TimeSpan.FromMinutes(30);
Console.WriteLine("Monitoring all tasks for 'Completed' state, timeout in {0}...", timeout);
IEnumerable <CloudTask> addedTasks = batchClient.JobOperations.ListTasks(JobId);
batchClient.Utilities.CreateTaskStateMonitor().WaitAll(addedTasks, TaskState.Completed, timeout);
Console.WriteLine("All tasks reached state Completed.");
// Print task output
Console.WriteLine();
Console.WriteLine("Printing task output...");
IEnumerable <CloudTask> completedtasks = batchClient.JobOperations.ListTasks(JobId);
foreach (CloudTask task in completedtasks)
{
List <Task> readTasks = new List <Task>();
foreach (OutputFileReference output in task.OutputStorage(linkedStorageAccount).ListOutputs(TaskOutputKind.TaskOutput))
{
// Console.WriteLine($"output file: {output.FilePath}");
readTasks.Add(output.DownloadToFileAsync($"{JobId}-{output.FilePath}", System.IO.FileMode.Create));
}
Task.WaitAll(readTasks.ToArray());
}
// Print out some timing info
timer.Stop();
Console.WriteLine();
Console.WriteLine("Sample end: {0}", DateTime.Now);
Console.WriteLine("Elapsed time: {0}", timer.Elapsed);
// Clean up Storage resources
container.DeleteIfExistsAsync().Wait();
Console.WriteLine("Container [{0}] deleted.", inputContainerName);
// Clean up Batch resources (if the user so chooses)
Console.WriteLine();
Console.Write("Delete job? [yes] no: ");
string response = Console.ReadLine().ToLower();
if (response != "n" && response != "no")
{
batchClient.JobOperations.DeleteJob(JobId);
}
Console.Write("Delete pool? [yes] no: ");
response = Console.ReadLine().ToLower();
if (response != "n" && response != "no")
{
batchClient.PoolOperations.DeletePool(PoolId);
}
}
}
finally
{
Console.WriteLine();
Console.WriteLine("Sample complete, hit ENTER to exit...");
Console.ReadLine();
}
}
static void Main(string[] args)
{
string storageConnectionString =
$"DefaultEndpointsProtocol=https;AccountName={StorageAccountName};AccountKey={StorageAccountKey}";
// Retrieve the storage account
CloudStorageAccount storageAccount = CloudStorageAccount.Parse(storageConnectionString);
// Create the blob client
CloudBlobClient blobClient = storageAccount.CreateCloudBlobClient();
const string inputContainerName = "batchinput";
List <string> inputFilePaths = new List <string>
{
"taskdata0.txt",
"taskdata1.txt",
"taskdata2.txt"
};
// Upload the input files to blob storage
List <ResourceFile> inputFiles = new List <ResourceFile>();
foreach (string filePath in inputFilePaths)
{
inputFiles.Add(UploadFileToContainer(blobClient, inputContainerName, filePath));
}
// Get a SAS Url for the output container
const string outputContainerName = "batchoutput";
string outputContainerSasUrl = GetOutputContainerSasUrl(blobClient, outputContainerName);
// Create the virtual machine image reference
ImageReference imageReference = new ImageReference(
publisher: "MicrosoftWindowsServer",
offer: "WindowsServer",
sku: "2016-datacenter-smalldisk",
version: "latest");
// Create the virtual machine configuration for the pool
VirtualMachineConfiguration virtualMachineConfiguration = new VirtualMachineConfiguration(
imageReference: imageReference,
nodeAgentSkuId: "batch.node.windows amd64");
BatchSharedKeyCredentials cred = new BatchSharedKeyCredentials(BatchAccountUrl, BatchAccountName, BatchAccountKey);
using (BatchClient batchClient = BatchClient.Open(cred))
{
Console.WriteLine("Creating pool [{0}]...", PoolId);
try
{
CloudPool pool = batchClient.PoolOperations.CreatePool(
poolId: PoolId,
targetDedicatedComputeNodes: PoolNodeCount,
virtualMachineSize: PoolVMSize,
virtualMachineConfiguration: virtualMachineConfiguration);
// Specify the application packages for the tasks
pool.ApplicationPackageReferences = new List <ApplicationPackageReference>
{
new ApplicationPackageReference {
ApplicationId = "ReadWriteFile", Version = "1"
}
};
pool.Commit();
}
catch (BatchException be)
{
// Accept the specific error code PoolExists as that is expected if the pool already exists
if (be.RequestInformation?.BatchError?.Code == BatchErrorCodeStrings.PoolExists)
{
Console.WriteLine("The pool {0} already existed when we tried to create it", PoolId);
}
else
{
throw; // Any other exception is unexpected
}
}
Console.WriteLine("Creating job [{0}]...", JobId);
try
{
CloudJob job = batchClient.JobOperations.CreateJob();
job.Id = JobId;
job.PoolInformation = new PoolInformation {
PoolId = PoolId
};
job.Commit();
}
catch (BatchException be)
{
// Accept the specific error code JobExists as that is expected if the job already exists
if (be.RequestInformation?.BatchError?.Code == BatchErrorCodeStrings.JobExists)
{
Console.WriteLine("The job {0} already existed when we tried to create it", JobId);
}
else
{
throw; // Any other exception is unexpected
}
}
// Create a collection to hold the tasks that we'll be adding to the job
Console.WriteLine("Adding {0} tasks to job [{1}]...", inputFiles.Count, JobId);
List <CloudTask> tasks = new List <CloudTask>();
// Create each of the tasks to process one of the input files.
for (int i = 0; i < inputFiles.Count; i++)
{
string taskId = String.Format("Task{0}", i);
string inputFilename = inputFiles[i].FilePath;
string outputFileName = string.Format("out{0}", inputFilename);
string taskCommandLine = string.Format("cmd /c %AZ_BATCH_APP_PACKAGE_READWRITEFILE%\\ReadWriteFile.exe {0} {1}", inputFilename, outputFileName);
CloudTask task = new CloudTask(taskId, taskCommandLine);
// Set the resource files and output files for the task
task.ResourceFiles = new List <ResourceFile> {
inputFiles[i]
};
task.OutputFiles = new List <OutputFile>
{
new OutputFile(
filePattern: outputFileName,
destination: new OutputFileDestination(new OutputFileBlobContainerDestination(containerUrl: outputContainerSasUrl, path: outputFileName)),
uploadOptions: new OutputFileUploadOptions(OutputFileUploadCondition.TaskCompletion))
};
tasks.Add(task);
}
// Add all tasks to the job.
batchClient.JobOperations.AddTask(JobId, tasks);
// Monitor task success/failure, specifying a maximum amount of time to wait for the tasks to complete.
TimeSpan timeout = TimeSpan.FromMinutes(30);
Console.WriteLine("Monitoring all tasks for 'Completed' state, timeout in {0}...", timeout);
IEnumerable <CloudTask> addedTasks = batchClient.JobOperations.ListTasks(JobId);
batchClient.Utilities.CreateTaskStateMonitor().WaitAll(addedTasks, TaskState.Completed, timeout);
Console.WriteLine("All tasks reached state Completed.");
// Print task output
Console.WriteLine();
Console.WriteLine("Printing task output...");
IEnumerable <CloudTask> completedtasks = batchClient.JobOperations.ListTasks(JobId);
foreach (CloudTask task in completedtasks)
{
string nodeId = String.Format(task.ComputeNodeInformation.ComputeNodeId);
Console.WriteLine("Task: {0}", task.Id);
Console.WriteLine("Node: {0}", nodeId);
Console.WriteLine("Standard out:");
Console.WriteLine(task.GetNodeFile(Constants.StandardOutFileName).ReadAsString());
}
// Clean up Batch resources (if the user so chooses)
Console.WriteLine();
Console.Write("Delete job? [yes] no: ");
string response = Console.ReadLine().ToLower();
if (response != "n" && response != "no")
{
batchClient.JobOperations.DeleteJob(JobId);
}
Console.Write("Delete pool? [yes] no: ");
response = Console.ReadLine().ToLower();
if (response != "n" && response != "no")
{
batchClient.PoolOperations.DeletePool(PoolId);
}
}
}
private static void StartBatchTasks(List <ResourceFile> inputFiles)
{
BatchSharedKeyCredentials cred = new BatchSharedKeyCredentials(BatchAccountUrl, BatchAccountName, BatchAccountKey);
using (BatchClient batchClient = BatchClient.Open(cred))
{
// Create a Windows Server image, VM configuration, Batch pool
ImageReference imageReference = CreateImageReference();
VirtualMachineConfiguration vmConfiguration = CreateVirtualMachineConfiguration(imageReference);
CreateBatchPool(batchClient, vmConfiguration);
CreateJob(batchClient);
List <CloudTask> tasks = new List <CloudTask>();
for (int i = 0; i < inputFiles.Count; i++)
{
string taskId = String.Format("Task{0}", i);
string inputFilename = inputFiles[i].FilePath;
string taskCommandLine = String.Format("cmd /c type {0}", inputFilename);
CloudTask task = new CloudTask(taskId, taskCommandLine);
task.ResourceFiles = new List <ResourceFile> {
inputFiles[i]
};
tasks.Add(task);
}
batchClient.JobOperations.AddTask(JobId, tasks);
TimeSpan timeout = TimeSpan.FromMinutes(30);
Console.WriteLine("Monitoring all tasks for 'Completed' state, timeout in {0}...", timeout);
IEnumerable <CloudTask> addedTasks = batchClient.JobOperations.ListTasks(JobId);
batchClient.Utilities.CreateTaskStateMonitor().WaitAll(addedTasks, TaskState.Completed, timeout);
// List tasks
IEnumerable <CloudTask> completedtasks = batchClient.JobOperations.ListTasks(JobId);
foreach (CloudTask task in completedtasks)
{
string nodeId = String.Format(task.ComputeNodeInformation.ComputeNodeId);
Console.WriteLine("Task: {0}", task.Id);
Console.WriteLine("Node: {0}", nodeId);
Console.WriteLine("Standard out:");
Console.WriteLine(task.GetNodeFile(Constants.StandardOutFileName).ReadAsString());
}
// delete job
batchClient.JobOperations.DeleteJob(JobId);
// delete pool
batchClient.PoolOperations.DeletePool(PoolId);
}
}
public void Start()
{
CloudStorageAccount storageAccount = createCloudStorageAccount();
CloudBlobClient cloudBlobClient = storageAccount.CreateCloudBlobClient();
string contName = ConfigurationManager.AppSettings["ContainerInput"];
CloudBlobContainer cloudBlobContainer = getBlobContainer(cloudBlobClient, contName);
List <string> files = getInputDataFiles();
List <ResourceFile> inputFiles = new List <ResourceFile>();
foreach (string f in files)
{
if (!String.IsNullOrEmpty(f))
{
inputFiles.Add(GetResource(f, cloudBlobClient, cloudBlobContainer));
}
}
using (BatchClient client = getBatchClient())
{
// Create a Batch pool, VM configuration, Windows Server image
string poolId = ConfigurationManager.AppSettings["PoolId"];
int poolNodeCount = 1;
string poolVMSize = "STANDARD_A1_v2";
string jobId = "DotNetQuickstartJob";
Console.WriteLine("Creating pool [{0}]...", poolId);
ImageReference imageReference = new ImageReference(
publisher: "MicrosoftWindowsServer",
offer: "WindowsServer",
sku: "2012-R2-datacenter-smalldisk",
version: "latest");
VirtualMachineConfiguration virtualMachineConfiguration =
new VirtualMachineConfiguration(
imageReference: imageReference,
nodeAgentSkuId: "batch.node.windows amd64");
try
{
if (client.PoolOperations.ListPools() != null &&
client.PoolOperations.ListPools().Count() > 0 &&
client.PoolOperations.ListPools().First(p => p.Id == poolId) != null)
{
client.PoolOperations.DeletePoolAsync(poolId).Wait();
}
CloudPool pool = client.PoolOperations.CreatePool(
poolId: poolId,
targetDedicatedComputeNodes: poolNodeCount,
virtualMachineSize: poolVMSize,
virtualMachineConfiguration: virtualMachineConfiguration);
pool.Commit();
}
catch (BatchException be)
{
// Accept the specific error code PoolExists as that is expected if the pool already exists
if (be.RequestInformation?.BatchError?.Code == BatchErrorCodeStrings.PoolExists)
{
Console.WriteLine("The pool {0} already existed when we tried to create it", poolId);
}
else
{
throw; // Any other exception is unexpected
}
}
catch (Exception ex)
{
}
try
{
client.JobOperations.DeleteJob(jobId);
CloudJob job = client.JobOperations.CreateJob();
job.Id = jobId;
job.PoolInformation = new PoolInformation {
PoolId = poolId
};
job.Commit();
}
catch (BatchException be)
{
// Accept the specific error code JobExists as that is expected if the job already exists
if (be.RequestInformation?.BatchError?.Code == BatchErrorCodeStrings.JobExists)
{
Console.WriteLine("The job {0} already existed when we tried to create it", jobId);
}
else
{
throw; // Any other exception is unexpected
}
}
// Create a collection to hold the tasks that we'll be adding to the job
Console.WriteLine("Adding {0} tasks to job [{1}]...", inputFiles.Count, jobId);
List <CloudTask> tasks = new List <CloudTask>();
// Create each of the tasks to process one of the input files.
for (int i = 0; i < inputFiles.Count; i++)
{
string taskId = String.Format("Task{0}", i);
string inputFilename = inputFiles[i].FilePath;
string taskCommandLine = String.Format("cmd /c type {0}", inputFilename);
CloudTask task = new CloudTask(taskId, taskCommandLine);
task.ResourceFiles = new List <ResourceFile> {
inputFiles[i]
};
tasks.Add(task);
}
// Add all tasks to the job.
client.JobOperations.AddTask(jobId, tasks);
// Monitor task success/failure, specifying a maximum amount of time to wait for the tasks to complete.
TimeSpan timeout = TimeSpan.FromMinutes(10);
Console.WriteLine("Monitoring all tasks for 'Completed' state, timeout in {0}...", timeout);
IEnumerable <CloudTask> addedTasks = client.JobOperations.ListTasks(jobId);
client.Utilities.CreateTaskStateMonitor().WaitAll(addedTasks, TaskState.Completed, timeout);
Console.WriteLine("All tasks reached state Completed.");
// Print task output
Console.WriteLine();
Console.WriteLine("Printing task output...");
IEnumerable <CloudTask> completedtasks = client.JobOperations.ListTasks(jobId);
Stopwatch timer = new Stopwatch();
timer.Start();
foreach (CloudTask task in completedtasks)
{
string nodeId = String.Format(task.ComputeNodeInformation.ComputeNodeId);
Console.WriteLine("Task: {0}", task.Id);
Console.WriteLine("Node: {0}", nodeId);
Console.WriteLine("Standard out:");
}
// Print out some timing info
timer.Stop();
Console.WriteLine();
Console.WriteLine("Sample end: {0}", DateTime.Now);
Console.WriteLine("Elapsed time: {0}", timer.Elapsed);
// Clean up Storage resources
if (cloudBlobContainer != null)
{
cloudBlobContainer.DeleteIfExistsAsync().Wait();
Console.WriteLine("Container [{0}] deleted.", contName);
}
else
{
Console.WriteLine("Container [{0}] does not exist, skipping deletion.", contName);
}
// Clean up Batch resources (if the user so chooses)
Console.WriteLine();
Console.Write("Delete job? [yes] no: ");
string response = Console.ReadLine().ToLower();
if (response != "n" && response != "no")
{
client.JobOperations.DeleteJob(jobId);
}
Console.Write("Delete pool? [yes] no: ");
response = Console.ReadLine().ToLower();
if (response != "n" && response != "no")
{
client.PoolOperations.DeletePool(poolId);
}
}
}
static void Main()
{
if (String.IsNullOrEmpty(BatchAccountName) ||
String.IsNullOrEmpty(BatchAccountKey) ||
String.IsNullOrEmpty(BatchAccountUrl) ||
String.IsNullOrEmpty(StorageAccountName) ||
String.IsNullOrEmpty(StorageAccountKey))
{
throw new InvalidOperationException("One or more account credential strings have not been populated. Please ensure that your Batch and Storage account credentials have been specified.");
}
try
{
Console.WriteLine("Sample start: {0}", DateTime.Now);
Console.WriteLine();
Stopwatch timer = new Stopwatch();
timer.Start();
// Create the blob client, for use in obtaining references to blob storage containers
CloudBlobClient blobClient = CreateCloudBlobClient(StorageAccountName, StorageAccountKey);
// Use the blob client to create the input container in Azure Storage
const string inputContainerName = "input";
CloudBlobContainer container = blobClient.GetContainerReference(inputContainerName);
container.CreateIfNotExistsAsync().Wait();
// The collection of data files that are to be processed by the tasks
List <string> inputFilePaths = new List <string>
{
"taskdata0.txt",
"taskdata1.txt",
"taskdata2.txt"
};
// Upload the data files to Azure Storage. This is the data that will be processed by each of the tasks that are
// executed on the compute nodes within the pool.
List <ResourceFile> inputFiles = new List <ResourceFile>();
foreach (string filePath in inputFilePaths)
{
inputFiles.Add(UploadFileToContainer(blobClient, inputContainerName, filePath));
}
// Get a Batch client using account creds
BatchSharedKeyCredentials cred = new BatchSharedKeyCredentials(BatchAccountUrl, BatchAccountName, BatchAccountKey);
using (BatchClient batchClient = BatchClient.Open(cred))
{
Console.WriteLine("Creating pool [{0}]...", PoolId);
// Create a Windows Server image, VM configuration, Batch pool
ImageReference imageReference = CreateImageReference();
VirtualMachineConfiguration vmConfiguration = CreateVirtualMachineConfiguration(imageReference);
CreateBatchPool(batchClient, vmConfiguration);
// Create a Batch job
Console.WriteLine("Creating job [{0}]...", JobId);
try
{
CloudJob job = batchClient.JobOperations.CreateJob();
job.Id = JobId;
job.PoolInformation = new PoolInformation {
PoolId = PoolId
};
job.Commit();
}
catch (BatchException be)
{
// Accept the specific error code JobExists as that is expected if the job already exists
if (be.RequestInformation?.BatchError?.Code == BatchErrorCodeStrings.JobExists)
{
Console.WriteLine("The job {0} already existed when we tried to create it", JobId);
}
else
{
throw; // Any other exception is unexpected
}
}
// Create a collection to hold the tasks that we'll be adding to the job
Console.WriteLine("Adding {0} tasks to job [{1}]...", inputFiles.Count, JobId);
List <CloudTask> tasks = new List <CloudTask>();
// Create each of the tasks to process one of the input files.
for (int i = 0; i < inputFiles.Count; i++)
{
string taskId = String.Format("Task{0}", i);
string inputFilename = inputFiles[i].FilePath;
string taskCommandLine = String.Format("cmd /c type {0}", inputFilename);
CloudTask task = new CloudTask(taskId, taskCommandLine);
task.ResourceFiles = new List <ResourceFile> {
inputFiles[i]
};
tasks.Add(task);
}
// Add all tasks to the job.
batchClient.JobOperations.AddTask(JobId, tasks);
// Monitor task success/failure, specifying a maximum amount of time to wait for the tasks to complete.
TimeSpan timeout = TimeSpan.FromMinutes(30);
Console.WriteLine("Monitoring all tasks for 'Completed' state, timeout in {0}...", timeout);
IEnumerable <CloudTask> addedTasks = batchClient.JobOperations.ListTasks(JobId);
batchClient.Utilities.CreateTaskStateMonitor().WaitAll(addedTasks, TaskState.Completed, timeout);
Console.WriteLine("All tasks reached state Completed.");
// Print task output
Console.WriteLine();
Console.WriteLine("Printing task output...");
IEnumerable <CloudTask> completedtasks = batchClient.JobOperations.ListTasks(JobId);
foreach (CloudTask task in completedtasks)
{
string nodeId = String.Format(task.ComputeNodeInformation.ComputeNodeId);
Console.WriteLine("Task: {0}", task.Id);
Console.WriteLine("Node: {0}", nodeId);
Console.WriteLine("Standard out:");
Console.WriteLine(task.GetNodeFile(Constants.StandardOutFileName).ReadAsString());
}
// Print out some timing info
timer.Stop();
Console.WriteLine();
Console.WriteLine("Sample end: {0}", DateTime.Now);
Console.WriteLine("Elapsed time: {0}", timer.Elapsed);
// Clean up Storage resources
container.DeleteIfExistsAsync().Wait();
Console.WriteLine("Container [{0}] deleted.", inputContainerName);
// Clean up Batch resources (if the user so chooses)
Console.WriteLine();
Console.Write("Delete job? [yes] no: ");
string response = Console.ReadLine().ToLower();
if (response != "n" && response != "no")
{
batchClient.JobOperations.DeleteJob(JobId);
}
Console.Write("Delete pool? [yes] no: ");
response = Console.ReadLine().ToLower();
if (response != "n" && response != "no")
{
batchClient.PoolOperations.DeletePool(PoolId);
}
}
}
finally
{
Console.WriteLine();
Console.WriteLine("Sample complete, hit ENTER to exit...");
Console.ReadLine();
}
}
