private CloudPool CreatePoolIfNotExists(String poolID, String vmSize, int computerNodes = 2, ImageReference imageReference = null) { messageContainer.AddInformationMessage("Configuring pool..."); CloudPool cPool = null; try { cPool = batchClient.PoolOperations.GetPool(poolID); } catch { } if (cPool == null) { //imageReference = (imageReference == null ? CreateImageReference("MicrosoftWindowsServer", "WindowsServer", "2016-Datacenter", "latest") : imageReference); imageReference = (imageReference == null ? CreateImageReference() : imageReference); VirtualMachineConfiguration vmConfig = new VirtualMachineConfiguration(imageReference, SKUReference(imageReference.Offer)); messageContainer.AddInformationMessage("Creating pool..."); cPool = batchClient.PoolOperations.CreatePool(poolID, vmSize, vmConfig, computerNodes); cPool.ApplicationPackageReferences = new List <ApplicationPackageReference> { new ApplicationPackageReference { ApplicationId = "radiance", Version = "1.0" } }; cPool.Commit(); messageContainer.AddInformationMessage("Pool created..."); } messageContainer.AddInformationMessage("Pool configured..."); return(cPool); }
private static void CreateBatchPool(BatchClient batchClient, VirtualMachineConfiguration vmConfiguration) { try { CloudPool pool = batchClient.PoolOperations.CreatePool( poolId: PoolId, targetDedicatedComputeNodes: PoolNodeCount, virtualMachineSize: PoolVMSize, virtualMachineConfiguration: vmConfiguration); pool.Commit(); } catch (BatchException be) { // Accept the specific error code PoolExists as that is expected if the pool already exists if (be.RequestInformation?.BatchError?.Code == BatchErrorCodeStrings.PoolExists) { Console.WriteLine("The pool {0} already existed when we tried to create it", PoolId); } else { throw; // Any other exception is unexpected } } }
// BATCH CLIENT OPERATIONS - FUNCTION IMPLEMENTATIONS /// <summary> /// Creates the Batch pool. /// </summary> /// <param name="batchClient">A BatchClient object</param> /// <param name="poolId">ID of the CloudPool object to create.</param> private static async Task CreatePoolIfNotExistAsync(BatchClient batchClient, string poolId) { CloudPool pool = null; try { Console.WriteLine("Creating pool [{0}]...", poolId); ImageReference imageReference = new ImageReference( publisher: "MicrosoftWindowsServer", offer: "WindowsServer", sku: "2012-R2-Datacenter-smalldisk", version: "latest"); VirtualMachineConfiguration virtualMachineConfiguration = new VirtualMachineConfiguration( imageReference: imageReference, nodeAgentSkuId: "batch.node.windows amd64"); // Create an unbound pool. No pool is actually created in the Batch service until we call // CloudPool.Commit(). This CloudPool instance is therefore considered "unbound," and we can // modify its properties. pool = batchClient.PoolOperations.CreatePool( poolId: poolId, targetDedicatedComputeNodes: DedicatedNodeCount, targetLowPriorityComputeNodes: LowPriorityNodeCount, virtualMachineSize: PoolVMSize, virtualMachineConfiguration: virtualMachineConfiguration); // Specify the application and version to install on the compute nodes // This assumes that a Windows 64-bit zipfile of ffmpeg has been added to Batch account // with Application Id of "ffmpeg" and Version of "4.3.1". // Download the zipfile https://www.gyan.dev/ffmpeg/builds/packages/ffmpeg-4.3.1-2020-09-21-full_build.zip // to upload as application package pool.ApplicationPackageReferences = new List <ApplicationPackageReference> { new ApplicationPackageReference { ApplicationId = appPackageId, Version = appPackageVersion } }; await pool.CommitAsync(); } catch (BatchException be) { // Accept the specific error code PoolExists as that is expected if the pool already exists if (be.RequestInformation?.BatchError?.Code == BatchErrorCodeStrings.PoolExists) { Console.WriteLine("The pool {0} already existed when we tried to create it", poolId); } else { throw; // Any other exception is unexpected } } }
private static async Task CreateBatchPoolAsync(BatchClient batchClient, string poolId) { CloudPool pool = null; Console.WriteLine("Creating pool [{0}]...", poolId); // Create an image reference object to store the settings for the nodes to be added to the pool ImageReference imageReference = new ImageReference( publisher: "MicrosoftWindowsServer", offer: "WindowsServer", sku: "2012-R2-Datacenter-smalldisk", version: "latest"); // Use the image reference to create a VirtualMachineConfiguration object VirtualMachineConfiguration virtualMachineConfiguration = new VirtualMachineConfiguration( imageReference: imageReference, nodeAgentSkuId: "batch.node.windows amd64"); try { // Create an unbound pool. No pool is actually created in the Batch service until we call // CloudPool.CommitAsync(). This CloudPool instance is therefore considered "unbound," and we can // modify its properties. pool = batchClient.PoolOperations.CreatePool( poolId: poolId, targetDedicatedComputeNodes: DedicatedNodeCount, targetLowPriorityComputeNodes: LowPriorityNodeCount, virtualMachineSize: PoolVMSize, virtualMachineConfiguration: virtualMachineConfiguration); // Specify the application and version to install on the compute nodes pool.ApplicationPackageReferences = new List <ApplicationPackageReference> { new ApplicationPackageReference { ApplicationId = appPackageId, Version = appPackageVersion } }; // Create the pool await pool.CommitAsync(); } catch (BatchException be) { // Accept the specific error code PoolExists as that is expected if the pool already exists if (be.RequestInformation?.BatchError?.Code == BatchErrorCodeStrings.PoolExists) { Console.WriteLine("The pool [{0}] already existed when we tried to create it", poolId); } else { throw; // Any other exception is unexpected } } }
private static void CreatePool(BatchClient batchClient, VirtualMachineConfiguration vmConfiguration) { CloudPool pool = batchClient.PoolOperations.CreatePool(poolId: PoolId, virtualMachineSize: PoolVMSize, virtualMachineConfiguration: vmConfiguration, targetDedicatedComputeNodes: PoolNodeCount); pool.Commit(); }
// Create the Compute Pool of the Batch Account public static async Task CreateBatchPoolIfNotExist(BatchClient batchClient, VirtualMachineConfiguration vmConfiguration, string vnetSubnetId) { Console.WriteLine("Creating pool [{0}]...", PoolId); try { CloudPool pool = batchClient.PoolOperations.CreatePool( poolId: PoolId, targetDedicatedComputeNodes: PoolNodeCount, virtualMachineSize: PoolVMSize, virtualMachineConfiguration: vmConfiguration); // Specify the application and version to install on the compute nodes pool.ApplicationPackageReferences = new List <ApplicationPackageReference> { new ApplicationPackageReference { ApplicationId = AppPackageName, Version = AppPackageVersion } }; // Initial the first data disk for each VM in the pool StartTask startTask = new StartTask("cmd /c Powershell -command \"Get-Disk | Where partitionstyle -eq 'raw' | sort number | Select-Object -first 1 |" + " Initialize-Disk -PartitionStyle MBR -PassThru | New-Partition -UseMaximumSize -DriveLetter F |" + " Format-Volume -FileSystem NTFS -NewFileSystemLabel data1 -Confirm:$false -Force\""); startTask.MaxTaskRetryCount = 1; startTask.UserIdentity = new UserIdentity(new AutoUserSpecification(AutoUserScope.Pool, ElevationLevel.Admin)); startTask.WaitForSuccess = true; pool.StartTask = startTask; // Create the Pool within the vnet subnet if it's specified. if (vnetSubnetId != null) { pool.NetworkConfiguration = new NetworkConfiguration(); pool.NetworkConfiguration.SubnetId = vnetSubnetId; } await pool.CommitAsync(); await pool.RefreshAsync(); } catch (BatchException be) { // Accept the specific error code PoolExists as that is expected if the pool already exists if (be.RequestInformation?.BatchError?.Code == BatchErrorCodeStrings.PoolExists) { Console.WriteLine("The pool {0} already existed when we tried to create it", PoolId); } else { throw; // Any other exception is unexpected } } }
// BATCH CLIENT SETUP - METHOD IMPLEMENTATIONS // CreatePoolIfNoneExist(): Creates the Batch pool. // batchClient: A BatchClient object. // PoolId: ID of the CloudPool object to create. // resourceFiles: A collection of ResourceFile objects representing blobs in a Storage account container. private static void CreatePoolIfNoneExist(BatchClient batchClient, string poolId) { CloudPool pool = null; try { Console.WriteLine("Creating pool [{0}]...", poolId); ImageReference imageReference = new ImageReference( publisher: "MicrosoftWindowsServer", offer: "WindowsServer", sku: "2012-R2-Datacenter", version: "latest"); VirtualMachineConfiguration virtualMachineConfiguration = new VirtualMachineConfiguration( imageReference: imageReference, nodeAgentSkuId: "batch.node.windows amd64" ); // Create an unbound pool. No pool is actually created in the Batch service until we call // CloudPool.Commit(). This CloudPool instance is therefore considered "unbound," and we can // modify its properties. pool = batchClient.PoolOperations.CreatePool( poolId: poolId, targetDedicatedComputeNodes: 5, // 5 compute nodes virtualMachineSize: "STANDARD_A1_v2", // Single-core, 1.75 GB memory, 225 GB disk virtualMachineConfiguration: virtualMachineConfiguration); // Windows Server 2012 R2 // Specify the application and version to install on the compute nodes // This assumes that a Windows 64-bit zipfile of ffmpeg has been added to Batch account // with Application Id of "ffmpeg" and Version of "3.4". // Download the zipfile https://ffmpeg.zeranoe.com/builds/win64/static/ffmpeg-3.4-win64-static.zip // to upload as application package pool.ApplicationPackageReferences = new List <ApplicationPackageReference> { new ApplicationPackageReference { ApplicationId = appPackageId, Version = appPackageVersion } }; pool.Commit(); } catch (BatchException be) { // Accept the specific error code PoolExists as that is expected if the pool already exists if (be.RequestInformation?.BatchError != null && be.RequestInformation.BatchError.Code == BatchErrorCodeStrings.PoolExists) { Console.WriteLine("The pool {0} already existed when we tried to create it", poolId); } else { throw; // Any other exception is unexpected } } }
public static VirtualMachineConfiguration CreateVirtualMachineConfiguration(ImageReference imageReference) { VirtualMachineConfiguration config = new VirtualMachineConfiguration( imageReference: imageReference, nodeAgentSkuId: "batch.node.windows amd64"); config.DataDisks = new List <DataDisk>(); config.DataDisks.Add(new DataDisk(0, 2048, CachingType.ReadOnly, StorageAccountType.PremiumLrs)); return(config); }
private static async Task CreateBatchPoolAsync(BatchClient batchClient, string poolId) { CloudPool pool = null; Console.WriteLine("Creating Pool [{0}]", poolId); // create image reference object for nodes var imageReference = new ImageReference( publisher: "MicrosoftWindowsServer", offer: "WindowsServer", sku: "2012-R2-Datacenter-smalldisk", version: "latest"); // create VirtualMachineConfiguration object from image reference var virtualMachineConfiguration = new VirtualMachineConfiguration( imageReference, "batch.node.windows amd64"); try { // create unbound pool, only created when we call CommitAsync() pool = batchClient.PoolOperations.CreatePool( poolId, targetDedicatedComputeNodes: DedicatedNodeCount, targetLowPriorityComputeNodes: LowPriorityNodeCount, virtualMachineSize: PoolVMSize, virtualMachineConfiguration: virtualMachineConfiguration ); pool.ApplicationPackageReferences = new List <ApplicationPackageReference> { new ApplicationPackageReference { ApplicationId = appPackageId, Version = appPackageVersion } }; await pool.CommitAsync(); } catch (BatchException be) { // accept if poolexists if (be.RequestInformation?.BatchError?.Code == BatchErrorCodeStrings.PoolExists) { Console.WriteLine("The pool [{0}] already existed", poolId); } else { throw; } } }
protected override async Task ExecuteAsync(CancellationToken stoppingToken) { var accountUrl = "***"; var accountName = "***"; var accountSecret = "**"; var jobtoDelete = "as_1bf795a45fda4ad8ab0f79bc24a942eb"; var sharedKeyCredentials = new BatchSharedKeyCredentials(accountUrl, accountName, accountSecret); var imageReference = new ImageReference( publisher: "microsoft-azure-batch", offer: "ubuntu-server-container", sku: "16-04-lts", version: "latest"); var virtualMachineConfiguration = new VirtualMachineConfiguration( imageReference, "batch.node.ubuntu 16.04"); using (var bc = await BatchClient.OpenAsync(sharedKeyCredentials)) { try { var newJob = bc.JobOperations.CreateJob("test", new PoolInformation() { AutoPoolSpecification = new AutoPoolSpecification() { AutoPoolIdPrefix = "test", PoolLifetimeOption = PoolLifetimeOption.Job, KeepAlive = false, PoolSpecification = new PoolSpecification() { TargetDedicatedComputeNodes = 1, VirtualMachineSize = "Standard_A1_v2", VirtualMachineConfiguration = virtualMachineConfiguration }, } }); newJob.Commit(); // never works fails with forbidden await bc.JobOperations.DeleteJobAsync(newJob.Id, cancellationToken : stoppingToken); } catch (Exception e) { Console.WriteLine($"Could not delete the job because {e}"); throw; } Console.WriteLine("Could delete job!"); } }
private async Task CreatePoolIfNotExistAsync(BatchClient batchClient) { try { Console.WriteLine("Creating pool [{0}]...", PoolKeys.PoolId); var imageReference = new ImageReference( publisher: "MicrosoftWindowsServer", offer: "WindowsServer", sku: "2012-R2-Datacenter-smalldisk", version: "latest"); var virtualMachineConfiguration = new VirtualMachineConfiguration( imageReference, "batch.node.windows amd64"); // Create an unbound pool. No pool is actually created in the Batch service until we call // CloudPool.Commit(). This CloudPool instance is therefore considered "unbound," and we can // modify its properties. var pool = batchClient.PoolOperations.CreatePool( PoolKeys.PoolId, targetDedicatedComputeNodes: PoolKeys.DedicatedNodeCount, targetLowPriorityComputeNodes: PoolKeys.LowPriorityNodeCount, virtualMachineSize: PoolKeys.PoolVMSize, virtualMachineConfiguration: virtualMachineConfiguration); pool.ApplicationPackageReferences = new List <ApplicationPackageReference> { new ApplicationPackageReference { ApplicationId = AppInformation.PackageId, Version = AppInformation.Version } }; await pool.CommitAsync(); } catch (BatchException be) { // Accept the specific error code PoolExists as that is expected if the pool already exists if (be.RequestInformation?.BatchError?.Code == BatchErrorCodeStrings.PoolExists) { Console.WriteLine("The pool {0} already existed when we tried to create it", PoolKeys.PoolId); } else { throw; // Any other exception is unexpected } } }
private static CloudPool CreatePool(BatchClient batchCli, string poolId) { var ubuntuImageDetails = IaasLinuxPoolFixture.GetUbuntuImageDetails(batchCli); VirtualMachineConfiguration virtualMachineConfiguration = new VirtualMachineConfiguration( ubuntuImageDetails.ImageReference, nodeAgentSkuId: ubuntuImageDetails.NodeAgentSku.Id); return(batchCli.PoolOperations.CreatePool( poolId: poolId, virtualMachineSize: PoolFixture.VMSize, virtualMachineConfiguration: virtualMachineConfiguration, targetDedicatedComputeNodes: 1)); }
/// <summary> /// Constructs an Azure Batch PoolInformation instance /// </summary> /// <param name="image">The image name for the current <see cref="TesTask"/></param> /// <param name="vmSize">The Azure VM sku</param> /// <param name="preemptible">True if preemptible machine should be used</param> /// <returns></returns> private async Task <PoolInformation> CreatePoolInformation(string image, string vmSize, bool preemptible) { var vmConfig = new VirtualMachineConfiguration( imageReference: new ImageReference("ubuntu-server-container", "microsoft-azure-batch", "16-04-lts", "latest"), nodeAgentSkuId: "batch.node.ubuntu 16.04"); var containerRegistryInfo = await azureProxy.GetContainerRegistryInfoAsync(image); if (containerRegistryInfo != null) { var containerRegistry = new ContainerRegistry( userName: containerRegistryInfo.Username, registryServer: containerRegistryInfo.RegistryServer, password: containerRegistryInfo.Password); // Download private images at node startup, since those cannot be downloaded in the main task that runs multiple containers. // Doing this also requires that the main task runs inside a container, hence downloading the "docker" image (contains docker client) as well. vmConfig.ContainerConfiguration = new ContainerConfiguration { ContainerImageNames = new List <string> { image, DockerInDockerImageName }, ContainerRegistries = new List <ContainerRegistry> { containerRegistry } }; } var poolSpecification = new PoolSpecification { VirtualMachineConfiguration = vmConfig, VirtualMachineSize = vmSize, ResizeTimeout = TimeSpan.FromMinutes(30), TargetLowPriorityComputeNodes = preemptible ? 1 : 0, TargetDedicatedComputeNodes = preemptible ? 0 : 1 }; var poolInformation = new PoolInformation { AutoPoolSpecification = new AutoPoolSpecification { AutoPoolIdPrefix = "TES", PoolLifetimeOption = PoolLifetimeOption.Job, PoolSpecification = poolSpecification, KeepAlive = false } }; return(poolInformation); }
/// <summary> /// This method can create a pool, it is not used in our example. /// The code remains for documentation purposes. /// </summary> public static void PreparePool() { BatchClient batchClient = PrepareConnection(); if (Environment.GetEnvironmentVariable("REGISTRYNAME") != null) { ContainerRegistry containerRegistry = new ContainerRegistry( registryServer: Environment.GetEnvironmentVariable("REGISTRYNAME"), userName: Environment.GetEnvironmentVariable("REGISTRYUSERNAME"), password: Environment.GetEnvironmentVariable("REGISTRYPASSWORD") ); // Create container configuration, prefetching Docker images from the container registry ContainerConfiguration containerConfig = new ContainerConfiguration() { ContainerImageNames = new List <string>() { Environment.GetEnvironmentVariable("WORKERIMAGENAME") }, ContainerRegistries = new List <ContainerRegistry> { containerRegistry } }; ImageReference imageReference = new ImageReference( publisher: "microsoft-azure-batch", offer: "ubuntu-server-container", sku: "16-04-lts", version: "latest"); // VM configuration VirtualMachineConfiguration virtualMachineConfiguration = new VirtualMachineConfiguration( imageReference: imageReference, nodeAgentSkuId: "batch.node.ubuntu 16.04") { ContainerConfiguration = containerConfig, }; //Create pool CloudPool pool = batchClient.PoolOperations.CreatePool( poolId: "docker", targetDedicatedComputeNodes: 1, virtualMachineSize: "Standard_A2_v2", virtualMachineConfiguration: virtualMachineConfiguration); pool.Commit(); } }
/// <summary> /// Creates a <see cref="CloudPool"/> associated with the specified Batch account. If an existing pool with the /// specified ID is found, the pool is resized to match the specified node count. /// </summary> /// <param name="batchClient">A fully initialized <see cref="BatchClient"/>.</param> /// <param name="poolId">The ID of the <see cref="CloudPool"/>.</param> /// <param name="nodeSize">The size of the nodes within the pool.</param> /// <param name="nodeCount">The number of nodes to create within the pool.</param> /// <param name="maxTasksPerNode">The maximum number of tasks to run concurrently on each node.</param> /// <returns>A bound <see cref="CloudPool"/> with the specified properties.</returns> public async Task <CloudPool> CreatePoolIfNotExistAsync(BatchClient batchClient, string poolId, string nodeSize = null, int nodeCount = 0, int maxTasksPerNode = 0) { try { // Provide a reference to an Azure Marketplace image for // "Windows Server 2016 Datacenter with Containers" ImageReference imageReference = new ImageReference( publisher: "MicrosoftWindowsServer", offer: "WindowsServer", sku: "2016-Datacenter-with-Containers", version: "latest"); ContainerConfiguration containerConfig = new ContainerConfiguration { ContainerImageNames = new List <string> { "ContainerImage" } }; // VM configuration VirtualMachineConfiguration virtualMachineConfiguration = new VirtualMachineConfiguration( imageReference: imageReference, nodeAgentSkuId: "batch.node.windows amd64"); virtualMachineConfiguration.ContainerConfiguration = containerConfig; // You can learn more about os families and versions at: // https://azure.microsoft.com/en-us/documentation/articles/cloud-services-guestos-update-matrix/ CloudPool pool = batchClient.PoolOperations.CreatePool(); pool.Id = poolId; pool.TargetDedicatedComputeNodes = 1; pool.VirtualMachineSize = "Standard_A1"; pool.VirtualMachineConfiguration = virtualMachineConfiguration; pool.StartTask = new StartTask { CommandLine = "cmd /C echo start task" }; await this.CreatePoolIfNotExistAsync(batchClient, pool).ConfigureAwait(continueOnCapturedContext: false); } catch (Exception ex) { } return(await batchClient.PoolOperations.GetPoolAsync(poolId).ConfigureAwait(continueOnCapturedContext: false)); }
private void CreateBatchPool(VirtualMachineConfiguration vmc, string vmsize = "Standard_E2_v3") { try { CloudPool pool = Client.PoolOperations.CreatePool(PoolId, vmsize, vmc, PoolSize); pool.Commit(); } catch (BatchException be) { if (be.RequestInformation?.BatchError?.Code == BatchErrorCodeStrings.PoolExists) { Logger.Warning("The pool " + PoolId + " already exist"); } else { throw; } } }
/// <summary> /// Creates a <see cref="CloudPool"/> with the specified id and configures its StartTask with the /// specified <see cref="ResourceFile"/> collection. /// </summary> /// <param name="batchClient">A <see cref="BatchClient"/>.</param> /// <param name="poolId">The id of the <see cref="CloudPool"/> to create.</param> /// <param name="resourceFiles">A collection of <see cref="ResourceFile"/> objects representing blobs within /// a Storage account container. The StartTask will download these files from Storage prior to execution.</param> /// <returns>A <see cref="System.Threading.Tasks.Task"/> object that represents the asynchronous operation.</returns> private static async Task CreatePoolAsync(BatchClient batchClient, string poolId, IList <ResourceFile> resourceFiles) { Console.WriteLine("Creating pool [{0}]...", poolId); // Set the OS and version of the VM image ImageReference imageReference = new ImageReference( publisher: "MicrosoftWindowsServer", offer: "WindowsServer", sku: "2016-Datacenter", version: "latest"); VirtualMachineConfiguration virtualMachineConfiguration = new VirtualMachineConfiguration( imageReference: imageReference, nodeAgentSkuId: "batch.node.windows amd64"); // Create the unbound pool. Until we call CloudPool.Commit() or CommitAsync(), no pool is actually created in the // Batch service. This CloudPool instance is therefore considered "unbound," and we can modify its properties. CloudPool pool = batchClient.PoolOperations.CreatePool( poolId: poolId, targetDedicatedComputeNodes: nodeNumberDedicated, // Number of dedicated compute nodes targetLowPriorityComputeNodes: nodeNumberLowPriority, // Number of low priority compute nodes virtualMachineSize: NodeSize, // Node instance size virtualMachineConfiguration: virtualMachineConfiguration); // Windows Server 2012 R2 // Create and assign the StartTask that will be executed when compute nodes join the pool. // In this case, we copy the StartTask's resource files (that will be automatically downloaded // to the node by the StartTask) into the shared directory that all tasks will have access to. pool.StartTask = new StartTask { // Specify a command line for the StartTask that copies the task application files to the // node's shared directory. Every compute node in a Batch pool is configured with a number // of pre-defined environment variables that can be referenced by commands or applications // run by tasks. CommandLine = "cmd /c starttask.cmd", ResourceFiles = resourceFiles, WaitForSuccess = true }; await pool.CommitAsync(); }
public static async Task <string> CreateBatchPoolAndImportJob([ActivityTrigger] ImportRequest request, ILogger log) { var azureServiceTokenProvider = new AzureServiceTokenProvider(); // Get a Batch client using function identity BatchTokenCredentials batchCred = new BatchTokenCredentials(request.BatchAccountUrl, await azureServiceTokenProvider.GetAccessTokenAsync("https://batch.core.windows.net/")); string jobId = request.TargetSqlServerName + "-Import-" + DateTime.UtcNow.ToString("MMddHHmmss"); using (BatchClient batchClient = BatchClient.Open(batchCred)) { ImageReference imageReference = CreateImageReference(); VirtualMachineConfiguration vmConfiguration = CreateVirtualMachineConfiguration(imageReference); await CreateBatchPoolIfNotExist(batchClient, vmConfiguration, request.VNetSubnetId); await CreateBatchJob(batchClient, jobId, log); } return(jobId); }
private static async Task PoolCreation(BatchClient p_batchClient, string p_poolId) { Console.WriteLine("Creating the pool of virtual machines"); try { ImageReference demo_image = new ImageReference( publisher: "MicrosoftWindowsServer", offer: "WindowsServer", sku: "2016-Datacenter", version: "latest"); VirtualMachineConfiguration demo_configuration = new VirtualMachineConfiguration( imageReference: demo_image, nodeAgentSkuId: "batch.node.windows amd64"); CloudPool demo_pool = null; demo_pool = p_batchClient.PoolOperations.CreatePool( poolId: p_poolId, targetDedicatedComputeNodes: 1, targetLowPriorityComputeNodes: 0, virtualMachineSize: "STANDARD_A1_v2", virtualMachineConfiguration: demo_configuration); demo_pool.ApplicationPackageReferences = new List <ApplicationPackageReference> { new ApplicationPackageReference { ApplicationId = demo_packageid, Version = demo_packageversion } }; await demo_pool.CommitAsync(); } catch (BatchException pool_error) { Console.WriteLine(pool_error.Message); } }
/// <summary> /// Constructs an Azure Batch PoolInformation instance /// </summary> /// <param name="image">The image name for the current <see cref="TesTask"/></param> /// <param name="vmSize">The Azure VM sku</param> /// <param name="preemptible">True if preemptible machine should be used</param> /// <returns></returns> private async Task <PoolInformation> CreatePoolInformation(string image, string vmSize, bool preemptible) { var vmConfig = new VirtualMachineConfiguration( imageReference: new ImageReference("ubuntu-server-container", "microsoft-azure-batch", "16-04-lts", "latest"), nodeAgentSkuId: "batch.node.ubuntu 16.04"); var containerRegistry = await GetContainerRegistry(image); vmConfig.ContainerConfiguration = new ContainerConfiguration { ContainerImageNames = new List <string> { image }, ContainerRegistries = containerRegistry != null ? new List <ContainerRegistry> { containerRegistry } : null }; var poolSpecification = new PoolSpecification { VirtualMachineConfiguration = vmConfig, VirtualMachineSize = vmSize, ResizeTimeout = TimeSpan.FromMinutes(30), TargetLowPriorityComputeNodes = preemptible ? 1 : 0, TargetDedicatedComputeNodes = preemptible ? 0 : 1 }; var poolInformation = new PoolInformation { AutoPoolSpecification = new AutoPoolSpecification { AutoPoolIdPrefix = "TES", PoolLifetimeOption = PoolLifetimeOption.Job, PoolSpecification = poolSpecification, KeepAlive = false } }; return(poolInformation); }
public void Execute() { string reqfile = _config.ResourceFolder + _config.ResourceName; //> CREATE try { ResourceFile req = CreateResourceFile(reqfile); BatchTokenCredentials cred = new BatchTokenCredentials(_config.BatchAccountUrl, GetToken()); Client = BatchClient.Open(cred); ImageReference imgref = new ImageReference(_config.VmImage); VirtualMachineConfiguration vmc = new VirtualMachineConfiguration(imageReference: imgref, nodeAgentSkuId: "batch.node.windows amd64"); CreateBatchPool(vmc, _config.VmSKU); // Create Job CloudJob job = Client.JobOperations.CreateJob(); job.Id = _jobId; job.PoolInformation = new PoolInformation { PoolId = _poolId }; job.Commit(); // Create Tasks List <CloudTask> tasks = new List <CloudTask>(); string taskId = "task" + System.DateTime.Now.Ticks.ToString(); string cmd = String.Format("cmd /c type {0}", req.FilePath); CloudTask task = new CloudTask(taskId, cmd); task.ResourceFiles = new List <ResourceFile> { req }; tasks.Add(task); Client.JobOperations.AddTask(_jobId, tasks); } catch (Exception x) { //TODO: Log Somewhere throw; } }
protected CloudPool CreatePool() { CloudPool currentPool = this.FindPoolIfExists(); // gotta create a new pool if (currentPool == null) { var ubuntuImageDetails = GetUbuntuImageDetails(this.client); VirtualMachineConfiguration virtualMachineConfiguration = new VirtualMachineConfiguration( ubuntuImageDetails.ImageReference, nodeAgentSkuId: ubuntuImageDetails.NodeAgentSku.Id); currentPool = this.client.PoolOperations.CreatePool( poolId: this.PoolId, virtualMachineSize: VMSize, virtualMachineConfiguration: virtualMachineConfiguration, targetDedicatedComputeNodes: 1); currentPool.Commit(); } return(WaitForPoolAllocation(this.client, this.PoolId)); }
static void Main(string[] args) { var config = new VirtualMachineConfiguration( true, true, true, true, true); using (var container = new Win64Container(config)) { var intType = container.VirtualMachine.TypeProvider.FindPrimitiveType(PrimitiveTypes.Int); var voidType = container.VirtualMachine.TypeProvider.FindPrimitiveType(PrimitiveTypes.Void); var pointMetadata = new ClassMetadata("Point"); pointMetadata.DefineField(new FieldDefinition("x", intType, AccessModifier.Public)); pointMetadata.DefineField(new FieldDefinition("y", intType, AccessModifier.Public)); pointMetadata.CreateFields(); container.VirtualMachine.ClassMetadataProvider.Add(pointMetadata); var pointType = container.VirtualMachine.TypeProvider.FindClassType("Point"); var pointArrayType = container.VirtualMachine.TypeProvider.FindArrayType(pointType); void Println(long objectReference) { Console.WriteLine($"0x{objectReference.ToString("x8")}"); } container.VirtualMachine.Binder.Define(FunctionDefinition.NewExternal <PrintlnPoint>( "std.println", new List <BaseType>() { pointType }, voidType, Println)); var constructorFunction = new ManagedFunction( new FunctionDefinition(".constructor", new List <BaseType>(), voidType, pointType, true), new List <BaseType>(), new List <Instruction>() { //new Instruction(OpCodes.LoadArgument, 0), //new Instruction(OpCodes.LoadInt, 1337), //new Instruction(OpCodes.StoreField, "Point::x"), new Instruction(OpCodes.Return) }); var mainFunction = new ManagedFunction( new FunctionDefinition("main", new List <BaseType>(), intType), new List <BaseType>() { pointArrayType }, //new List<BaseType>() { pointType }, new List <Instruction> { //new Instruction(OpCodes.LoadInt, 10), //new Instruction(OpCodes.NewArray, pointType.Name), //new Instruction(OpCodes.StoreLocal, 0), //new Instruction(OpCodes.LoadLocal, 0), //new Instruction(OpCodes.LoadInt, 0), //new Instruction(OpCodes.NewObject, ".constructor", pointType, new List<BaseType>()), //new Instruction(OpCodes.StoreElement, pointType.Name), //new Instruction(OpCodes.Call, "std.gc.collect", new List<BaseType>()), //new Instruction(OpCodes.LoadInt, 0), //new Instruction(OpCodes.Return), new Instruction(OpCodes.LoadInt, 10), new Instruction(OpCodes.NewArray, pointType.Name), new Instruction(OpCodes.StoreLocal, 0), new Instruction(OpCodes.NewObject, ".constructor", pointType, new List <BaseType>()), new Instruction(OpCodes.Pop), new Instruction(OpCodes.Call, "std.gc.collect", new List <BaseType>()), new Instruction(OpCodes.Call, "std.gc.collect", new List <BaseType>()), new Instruction(OpCodes.LoadInt, 0), new Instruction(OpCodes.Return), }); var functions = new List <ManagedFunction>() { mainFunction, constructorFunction }; container.VirtualMachine.LoadFunctionsAsAssembly(functions); container.VirtualMachine.Compile(); foreach (var function in functions) { var disassembler = new Disassembler( container.VirtualMachine.Compiler.GetCompilationData(function), x => new Compiler.Win64.Disassembler(x)); Console.WriteLine(disassembler.Disassemble()); } int returnValue = container.VirtualMachine.GetEntryPoint()(); Console.WriteLine(returnValue); } Console.ReadLine(); }
static void Main(string[] args) { string storageConnectionString = $"DefaultEndpointsProtocol=https;AccountName={StorageAccountName};AccountKey={StorageAccountKey}"; // Retrieve the storage account CloudStorageAccount storageAccount = CloudStorageAccount.Parse(storageConnectionString); // Create the blob client CloudBlobClient blobClient = storageAccount.CreateCloudBlobClient(); // Upload the input files to blob storage const string inputContainerName = "batchinput"; List <string> inputFilePaths = new List <string> { "taskdata0.txt", "taskdata1.txt", "taskdata2.txt" }; List <ResourceFile> inputFiles = new List <ResourceFile>(); foreach (string filePath in inputFilePaths) { inputFiles.Add(UploadFileToContainer(blobClient, inputContainerName, filePath)); } // Get a SAS Url for the output container const string outputContainerName = "batchoutput"; string outputContainerSasUrl = GetOutputContainerSasUrl(blobClient, outputContainerName); // Specify a container registry ContainerRegistry containerRegistry = new ContainerRegistry( registryServer: RegistryServer, userName: RegistryUserName, password: RegistryPassword); // Create container configuration, prefetching Docker images from the container registry ContainerConfiguration containerConfig = new ContainerConfiguration() { ContainerImageNames = ContainerImageNames, ContainerRegistries = new List <ContainerRegistry> { containerRegistry } }; // Create the virtual machine image reference - make sure to choose an image that supports containers ImageReference imageReference = new ImageReference( publisher: "MicrosoftWindowsServer", offer: "WindowsServer", sku: "2016-datacenter-with-containers", version: "latest"); // Create the virtual machine configuration for the pool and set the container configuration VirtualMachineConfiguration virtualMachineConfiguration = new VirtualMachineConfiguration( imageReference: imageReference, nodeAgentSkuId: "batch.node.windows amd64"); virtualMachineConfiguration.ContainerConfiguration = containerConfig; BatchSharedKeyCredentials cred = new BatchSharedKeyCredentials(BatchAccountUrl, BatchAccountName, BatchAccountKey); using (BatchClient batchClient = BatchClient.Open(cred)) { Console.WriteLine("Creating pool [{0}]...", PoolId); try { CloudPool pool = batchClient.PoolOperations.CreatePool( poolId: PoolId, targetDedicatedComputeNodes: PoolNodeCount, virtualMachineSize: PoolVMSize, virtualMachineConfiguration: virtualMachineConfiguration); pool.Commit(); } catch (BatchException be) { // Accept the specific error code PoolExists as that is expected if the pool already exists if (be.RequestInformation?.BatchError?.Code == BatchErrorCodeStrings.PoolExists) { Console.WriteLine("The pool {0} already existed when we tried to create it", PoolId); } else { throw; // Any other exception is unexpected } } Console.WriteLine("Creating job [{0}]...", JobId); CloudJob job = null; try { job = batchClient.JobOperations.CreateJob(); job.Id = JobId; job.PoolInformation = new PoolInformation { PoolId = PoolId }; // Add job preparation task to remove existing Docker image Console.WriteLine("Adding job preparation task to job [{0}]...", JobId); string jobPreparationCmdLine = $"cmd /c docker rmi -f {ContainerImageNames[0]}:latest"; JobPreparationTask jobPreparationTask = new JobPreparationTask(jobPreparationCmdLine) { UserIdentity = new UserIdentity(new AutoUserSpecification(elevationLevel: ElevationLevel.Admin, scope: AutoUserScope.Task)) }; job.JobPreparationTask = jobPreparationTask; job.Commit(); } catch (BatchException be) { // Accept the specific error code JobExists as that is expected if the job already exists if (be.RequestInformation?.BatchError?.Code == BatchErrorCodeStrings.JobExists) { Console.WriteLine("The job {0} already existed when we tried to create it", JobId); } else { throw; // Any other exception is unexpected } } if (job != null) { // Create a collection to hold the tasks that we'll be adding to the job Console.WriteLine("Adding {0} tasks to job [{1}]...", inputFiles.Count, JobId); List <CloudTask> tasks = new List <CloudTask>(); // Create each of the tasks to process one of the input files. for (int i = 0; i < inputFiles.Count; i++) { string taskId = String.Format("Task{0}", i); string inputFilename = inputFiles[i].FilePath; string outputFileName = string.Format("out{0}", inputFilename); // Override the default entrypoint of the container string taskCommandLine = string.Format("C:\\ReadWriteFile\\ReadWriteFile.exe {0} {1}", inputFilename, outputFileName); // Specify the container the task will run TaskContainerSettings cmdContainerSettings = new TaskContainerSettings( imageName: "nimccollftacr.azurecr.io/batch/readwritefile" ); CloudTask task = new CloudTask(taskId, taskCommandLine); task.ContainerSettings = cmdContainerSettings; // Set the resource files and output files for the task task.ResourceFiles = new List <ResourceFile> { inputFiles[i] }; task.OutputFiles = new List <OutputFile> { new OutputFile( filePattern: outputFileName, destination: new OutputFileDestination(new OutputFileBlobContainerDestination(containerUrl: outputContainerSasUrl, path: outputFileName)), uploadOptions: new OutputFileUploadOptions(OutputFileUploadCondition.TaskCompletion)) }; // You must elevate the identity of the task in order to run a container task.UserIdentity = new UserIdentity(new AutoUserSpecification(elevationLevel: ElevationLevel.Admin, scope: AutoUserScope.Task)); tasks.Add(task); } // Add all tasks to the job. batchClient.JobOperations.AddTask(JobId, tasks); // Monitor task success/failure, specifying a maximum amount of time to wait for the tasks to complete. TimeSpan timeout = TimeSpan.FromMinutes(30); Console.WriteLine("Monitoring all tasks for 'Completed' state, timeout in {0}...", timeout); IEnumerable <CloudTask> addedTasks = batchClient.JobOperations.ListTasks(JobId); batchClient.Utilities.CreateTaskStateMonitor().WaitAll(addedTasks, TaskState.Completed, timeout); Console.WriteLine("All tasks reached state Completed."); // Print task output Console.WriteLine(); Console.WriteLine("Printing task output..."); IEnumerable <CloudTask> completedtasks = batchClient.JobOperations.ListTasks(JobId); foreach (CloudTask task in completedtasks) { string nodeId = String.Format(task.ComputeNodeInformation.ComputeNodeId); Console.WriteLine("Task: {0}", task.Id); Console.WriteLine("Node: {0}", nodeId); Console.WriteLine("Standard out:"); Console.WriteLine(task.GetNodeFile(Constants.StandardOutFileName).ReadAsString()); } } // Clean up Batch resources (if the user so chooses) Console.WriteLine(); Console.Write("Delete job? [yes] no: "); string response = Console.ReadLine().ToLower(); if (response != "n" && response != "no") { batchClient.JobOperations.DeleteJob(JobId); } Console.Write("Delete pool? [yes] no: "); response = Console.ReadLine().ToLower(); if (response != "n" && response != "no") { batchClient.PoolOperations.DeletePool(PoolId); } } }
private static async Task MainAsync() { if (String.IsNullOrEmpty(BatchAccountName) || String.IsNullOrEmpty(BatchAccountKey) || String.IsNullOrEmpty(BatchAccountUrl) || String.IsNullOrEmpty(StorageAccountName) || String.IsNullOrEmpty(StorageAccountKey)) { throw new InvalidOperationException("One or more account credential strings have not been populated. Please ensure that your Batch and Storage account credentials have been specified."); } try { Console.WriteLine("Sample start: {0}", DateTime.Now); Console.WriteLine(); Stopwatch timer = new Stopwatch(); timer.Start(); // Construct the Storage account connection string string storageConnectionString = String.Format("DefaultEndpointsProtocol=https;AccountName={0};AccountKey={1}", StorageAccountName, StorageAccountKey); // Retrieve the storage account CloudStorageAccount storageAccount = CloudStorageAccount.Parse(storageConnectionString); // Create the blob client, for use in obtaining references to blob storage containers CloudBlobClient blobClient = storageAccount.CreateCloudBlobClient(); // Use the blob client to create the input container in Azure Storage const string appContainerName = "application"; const string inputContainerName = "input"; const string outputContainerName = "output"; await CreateContainerIfNotExistAsync(blobClient, appContainerName); await CreateContainerIfNotExistAsync(blobClient, inputContainerName); await CreateContainerIfNotExistAsync(blobClient, outputContainerName); // The collection of data files that are to be processed by the tasks // The collection of data files that are to be processed by the tasks List <string> inputFilePaths = new List <string> { @"..\..\taskdata1.txt", @"..\..\taskdata2.txt", @"..\..\taskdata3.txt" }; // Paths to the executable and its dependencies that will be executed by the tasks List <string> applicationFilePaths = new List <string> { // The DotNetTutorial project includes a project reference to TaskApplication, allowing us to // determine the path of the task application binary dynamically typeof(WordCounter.Program).Assembly.Location, "Microsoft.WindowsAzure.Storage.dll" }; // Upload the application and its dependencies to Azure Storage. This is the application that will // process the data files, and will be executed by each of the tasks on the compute nodes. List <ResourceFile> applicationFiles = await UploadFilesToContainerAsync(blobClient, appContainerName, applicationFilePaths); // Upload the data files. This is the data that will be processed by each of the tasks that are // executed on the compute nodes within the pool. List <ResourceFile> inputFiles = await UploadFilesToContainerAsync(blobClient, inputContainerName, inputFilePaths); // Obtain a shared access signature that provides write access to the output container to which // the tasks will upload their output. string outputContainerSasUrl = GetContainerSasUrl(blobClient, outputContainerName, SharedAccessBlobPermissions.Write); BatchSharedKeyCredentials cred = new BatchSharedKeyCredentials(BatchAccountUrl, BatchAccountName, BatchAccountKey); using (BatchClient batchClient = BatchClient.Open(cred)) { // Create a Batch pool, VM configuration, Windows Server image Console.WriteLine("Creating pool [{0}]...", PoolId); ImageReference imageReference = new ImageReference( publisher: "MicrosoftWindowsServer", offer: "WindowsServer", sku: "2012-R2-datacenter-smalldisk", version: "latest"); VirtualMachineConfiguration virtualMachineConfiguration = new VirtualMachineConfiguration( imageReference: imageReference, nodeAgentSkuId: "batch.node.windows amd64"); try { CloudPool pool = batchClient.PoolOperations.CreatePool( poolId: PoolId, targetDedicatedComputeNodes: PoolNodeCount, virtualMachineSize: PoolVMSize, virtualMachineConfiguration: virtualMachineConfiguration); // Create and assign the StartTask that will be executed when compute nodes join the pool. // In this case, we copy the StartTask's resource files (that will be automatically downloaded // to the node by the StartTask) into the shared directory that all tasks will have access to. pool.StartTask = new StartTask { // Specify a command line for the StartTask that copies the task application files to the // node's shared directory. Every compute node in a Batch pool is configured with a number // of pre-defined environment variables that can be referenced by commands or applications // run by tasks. // Since a successful execution of robocopy can return a non-zero exit code (e.g. 1 when one or // more files were successfully copied) we need to manually exit with a 0 for Batch to recognize // StartTask execution success. CommandLine = "cmd /c (robocopy %AZ_BATCH_TASK_WORKING_DIR% %AZ_BATCH_NODE_SHARED_DIR%) ^& IF %ERRORLEVEL% LEQ 1 exit 0", ResourceFiles = applicationFiles, WaitForSuccess = true }; pool.Commit(); } catch (BatchException be) { // Accept the specific error code PoolExists as that is expected if the pool already exists if (be.RequestInformation?.BatchError?.Code == BatchErrorCodeStrings.PoolExists) { Console.WriteLine("The pool {0} already existed when we tried to create it", PoolId); } else { throw; // Any other exception is unexpected } } // Create the job that will run the tasks. await CreateJobAsync(batchClient, JobId, PoolId); // Add the tasks to the job. We need to supply a container shared access signature for the // tasks so that they can upload their output to Azure Storage. await AddTasksAsync(batchClient, JobId, inputFiles, outputContainerSasUrl); // Monitor task success/failure, specifying a maximum amount of time to wait for the tasks to complete await MonitorTasks(batchClient, JobId, TimeSpan.FromMinutes(30)); // Download the task output files from the output Storage container to a local directory await DownloadBlobsFromContainerAsync(blobClient, outputContainerName, Environment.GetEnvironmentVariable("TEMP")); // Clean up Storage resources await DeleteContainerAsync(blobClient, appContainerName); await DeleteContainerAsync(blobClient, inputContainerName); await DeleteContainerAsync(blobClient, outputContainerName); // Print out some timing info timer.Stop(); Console.WriteLine(); Console.WriteLine("Sample end: {0}", DateTime.Now); Console.WriteLine("Elapsed time: {0}", timer.Elapsed); // Clean up Batch resources (if the user so chooses) Console.WriteLine(); Console.Write("Delete job? [yes] no: "); string response = Console.ReadLine().ToLower(); if (response != "n" && response != "no") { await batchClient.JobOperations.DeleteJobAsync(JobId); } Console.Write("Delete pool? [yes] no: "); response = Console.ReadLine().ToLower(); if (response != "n" && response != "no") { await batchClient.PoolOperations.DeletePoolAsync(PoolId); } } } catch { } }
static void Main(string[] args) { if (String.IsNullOrEmpty(BatchAccountName) || String.IsNullOrEmpty(BatchAccountKey) || String.IsNullOrEmpty(BatchAccountUrl) || String.IsNullOrEmpty(StorageAccountName) || String.IsNullOrEmpty(StorageAccountKey)) { throw new InvalidOperationException("One or more account credential strings have not been populated. Please ensure that your Batch and Storage account credentials have been specified."); } try { Console.WriteLine("Sample start: {0}", DateTime.Now); Console.WriteLine(); Stopwatch timer = new Stopwatch(); timer.Start(); // Create the blob client, for use in obtaining references to blob storage containers (CloudBlobClient, CloudStorageAccount)cc = CreateCloudBlobClient(StorageAccountName, StorageAccountKey); CloudBlobClient blobClient = cc.Item1; CloudStorageAccount linkedStorageAccount = cc.Item2; // Use the blob client to create the input container in Azure Storage const string inputContainerName = "input"; CloudBlobContainer container = blobClient.GetContainerReference(inputContainerName); container.CreateIfNotExistsAsync().Wait(); // The collection of data files that are to be processed by the tasks List <string> inputFilePaths = new List <string> { "input.txt" }; // Upload the data files to Azure Storage. This is the data that will be processed by each of the tasks that are // executed on the compute nodes within the pool. UploadFileToContainer(blobClient, inputContainerName, "input.txt"); // Get a Batch client using account creds BatchSharedKeyCredentials cred = new BatchSharedKeyCredentials(BatchAccountUrl, BatchAccountName, BatchAccountKey); using (BatchClient batchClient = BatchClient.Open(cred)) { Console.WriteLine("Creating pool [{0}]...", PoolId); // Create a Windows Server image, VM configuration, Batch pool ImageReference imageReference = CreateImageReference(); VirtualMachineConfiguration vmConfiguration = CreateVirtualMachineConfiguration(imageReference); CreateBatchPool(batchClient, vmConfiguration); // Create a Batch job Console.WriteLine("Creating job [{0}]...", JobId); try { CloudJob job = batchClient.JobOperations.CreateJob(); job.Id = JobId; job.PoolInformation = new PoolInformation { PoolId = PoolId }; job.UsesTaskDependencies = true; // Create the blob storage container for the outputs. Task t1 = job.PrepareOutputStorageAsync(linkedStorageAccount); Task.WaitAll(t1); string containerName = job.OutputStorageContainerName(); string containerUrl = job.GetOutputStorageContainerUrl(linkedStorageAccount); job.CommonEnvironmentSettings = new[] { new EnvironmentSetting("JOB_CONTAINER_URL", containerUrl) }; job.Commit(); } catch (BatchException be) { // Accept the specific error code JobExists as that is expected if the job already exists if (be.RequestInformation?.BatchError?.Code == BatchErrorCodeStrings.JobExists) { Console.WriteLine("The job {0} already existed when we tried to create it", JobId); } else { throw; // Any other exception is unexpected } } // Create a collection to hold the tasks that we'll be adding to the job List <CloudTask> tasks = new List <CloudTask>(); // Create each of the tasks to process one of the input files. string lastTaskId = null; for (int i = 0; i < 3; i++) { string taskId = String.Format("Task{0}", i); CloudTask task; // first task if (lastTaskId == null) { //task = new CloudTask(taskId, $"cmd /c %AZ_BATCH_APP_PACKAGE_TESTBATCHAPP%\\testbatchapp.exe {i}") task = new CloudTask(taskId, $"cmd /c %AZ_BATCH_APP_PACKAGE_HPCAPP%\\testbatchapp.exe {i}") { ApplicationPackageReferences = new List <ApplicationPackageReference> { new ApplicationPackageReference { //ApplicationId = "testbatchapp", ApplicationId = "hpcapp" //, //Version = "1.1" } } }; } else { // task = new CloudTask(taskId, $"cmd /c %AZ_BATCH_APP_PACKAGE_TESTBATCHAPP%\\testbatchapp.exe {i}") task = new CloudTask(taskId, $"cmd /c %AZ_BATCH_APP_PACKAGE_HPCAPP%\\testbatchapp.exe {i}") { ApplicationPackageReferences = new List <ApplicationPackageReference> { new ApplicationPackageReference { // ApplicationId = "testbatchapp", ApplicationId = "hpcapp" //, //Version = "1.1" } } }; task.DependsOn = TaskDependencies.OnId(lastTaskId); } lastTaskId = taskId; tasks.Add(task); } // Add all tasks to the job. batchClient.JobOperations.AddTask(JobId, tasks); // Monitor task success/failure, specifying a maximum amount of time to wait for the tasks to complete. TimeSpan timeout = TimeSpan.FromMinutes(30); Console.WriteLine("Monitoring all tasks for 'Completed' state, timeout in {0}...", timeout); IEnumerable <CloudTask> addedTasks = batchClient.JobOperations.ListTasks(JobId); batchClient.Utilities.CreateTaskStateMonitor().WaitAll(addedTasks, TaskState.Completed, timeout); Console.WriteLine("All tasks reached state Completed."); // Print task output Console.WriteLine(); Console.WriteLine("Printing task output..."); IEnumerable <CloudTask> completedtasks = batchClient.JobOperations.ListTasks(JobId); foreach (CloudTask task in completedtasks) { List <Task> readTasks = new List <Task>(); foreach (OutputFileReference output in task.OutputStorage(linkedStorageAccount).ListOutputs(TaskOutputKind.TaskOutput)) { // Console.WriteLine($"output file: {output.FilePath}"); readTasks.Add(output.DownloadToFileAsync($"{JobId}-{output.FilePath}", System.IO.FileMode.Create)); } Task.WaitAll(readTasks.ToArray()); } // Print out some timing info timer.Stop(); Console.WriteLine(); Console.WriteLine("Sample end: {0}", DateTime.Now); Console.WriteLine("Elapsed time: {0}", timer.Elapsed); // Clean up Storage resources container.DeleteIfExistsAsync().Wait(); Console.WriteLine("Container [{0}] deleted.", inputContainerName); // Clean up Batch resources (if the user so chooses) Console.WriteLine(); Console.Write("Delete job? [yes] no: "); string response = Console.ReadLine().ToLower(); if (response != "n" && response != "no") { batchClient.JobOperations.DeleteJob(JobId); } Console.Write("Delete pool? [yes] no: "); response = Console.ReadLine().ToLower(); if (response != "n" && response != "no") { batchClient.PoolOperations.DeletePool(PoolId); } } } finally { Console.WriteLine(); Console.WriteLine("Sample complete, hit ENTER to exit..."); Console.ReadLine(); } }
static void Main(string[] args) { string storageConnectionString = $"DefaultEndpointsProtocol=https;AccountName={StorageAccountName};AccountKey={StorageAccountKey}"; // Retrieve the storage account CloudStorageAccount storageAccount = CloudStorageAccount.Parse(storageConnectionString); // Create the blob client CloudBlobClient blobClient = storageAccount.CreateCloudBlobClient(); const string inputContainerName = "batchinput"; List <string> inputFilePaths = new List <string> { "taskdata0.txt", "taskdata1.txt", "taskdata2.txt" }; // Upload the input files to blob storage List <ResourceFile> inputFiles = new List <ResourceFile>(); foreach (string filePath in inputFilePaths) { inputFiles.Add(UploadFileToContainer(blobClient, inputContainerName, filePath)); } // Get a SAS Url for the output container const string outputContainerName = "batchoutput"; string outputContainerSasUrl = GetOutputContainerSasUrl(blobClient, outputContainerName); // Create the virtual machine image reference ImageReference imageReference = new ImageReference( publisher: "MicrosoftWindowsServer", offer: "WindowsServer", sku: "2016-datacenter-smalldisk", version: "latest"); // Create the virtual machine configuration for the pool VirtualMachineConfiguration virtualMachineConfiguration = new VirtualMachineConfiguration( imageReference: imageReference, nodeAgentSkuId: "batch.node.windows amd64"); BatchSharedKeyCredentials cred = new BatchSharedKeyCredentials(BatchAccountUrl, BatchAccountName, BatchAccountKey); using (BatchClient batchClient = BatchClient.Open(cred)) { Console.WriteLine("Creating pool [{0}]...", PoolId); try { CloudPool pool = batchClient.PoolOperations.CreatePool( poolId: PoolId, targetDedicatedComputeNodes: PoolNodeCount, virtualMachineSize: PoolVMSize, virtualMachineConfiguration: virtualMachineConfiguration); // Specify the application packages for the tasks pool.ApplicationPackageReferences = new List <ApplicationPackageReference> { new ApplicationPackageReference { ApplicationId = "ReadWriteFile", Version = "1" } }; pool.Commit(); } catch (BatchException be) { // Accept the specific error code PoolExists as that is expected if the pool already exists if (be.RequestInformation?.BatchError?.Code == BatchErrorCodeStrings.PoolExists) { Console.WriteLine("The pool {0} already existed when we tried to create it", PoolId); } else { throw; // Any other exception is unexpected } } Console.WriteLine("Creating job [{0}]...", JobId); try { CloudJob job = batchClient.JobOperations.CreateJob(); job.Id = JobId; job.PoolInformation = new PoolInformation { PoolId = PoolId }; job.Commit(); } catch (BatchException be) { // Accept the specific error code JobExists as that is expected if the job already exists if (be.RequestInformation?.BatchError?.Code == BatchErrorCodeStrings.JobExists) { Console.WriteLine("The job {0} already existed when we tried to create it", JobId); } else { throw; // Any other exception is unexpected } } // Create a collection to hold the tasks that we'll be adding to the job Console.WriteLine("Adding {0} tasks to job [{1}]...", inputFiles.Count, JobId); List <CloudTask> tasks = new List <CloudTask>(); // Create each of the tasks to process one of the input files. for (int i = 0; i < inputFiles.Count; i++) { string taskId = String.Format("Task{0}", i); string inputFilename = inputFiles[i].FilePath; string outputFileName = string.Format("out{0}", inputFilename); string taskCommandLine = string.Format("cmd /c %AZ_BATCH_APP_PACKAGE_READWRITEFILE%\\ReadWriteFile.exe {0} {1}", inputFilename, outputFileName); CloudTask task = new CloudTask(taskId, taskCommandLine); // Set the resource files and output files for the task task.ResourceFiles = new List <ResourceFile> { inputFiles[i] }; task.OutputFiles = new List <OutputFile> { new OutputFile( filePattern: outputFileName, destination: new OutputFileDestination(new OutputFileBlobContainerDestination(containerUrl: outputContainerSasUrl, path: outputFileName)), uploadOptions: new OutputFileUploadOptions(OutputFileUploadCondition.TaskCompletion)) }; tasks.Add(task); } // Add all tasks to the job. batchClient.JobOperations.AddTask(JobId, tasks); // Monitor task success/failure, specifying a maximum amount of time to wait for the tasks to complete. TimeSpan timeout = TimeSpan.FromMinutes(30); Console.WriteLine("Monitoring all tasks for 'Completed' state, timeout in {0}...", timeout); IEnumerable <CloudTask> addedTasks = batchClient.JobOperations.ListTasks(JobId); batchClient.Utilities.CreateTaskStateMonitor().WaitAll(addedTasks, TaskState.Completed, timeout); Console.WriteLine("All tasks reached state Completed."); // Print task output Console.WriteLine(); Console.WriteLine("Printing task output..."); IEnumerable <CloudTask> completedtasks = batchClient.JobOperations.ListTasks(JobId); foreach (CloudTask task in completedtasks) { string nodeId = String.Format(task.ComputeNodeInformation.ComputeNodeId); Console.WriteLine("Task: {0}", task.Id); Console.WriteLine("Node: {0}", nodeId); Console.WriteLine("Standard out:"); Console.WriteLine(task.GetNodeFile(Constants.StandardOutFileName).ReadAsString()); } // Clean up Batch resources (if the user so chooses) Console.WriteLine(); Console.Write("Delete job? [yes] no: "); string response = Console.ReadLine().ToLower(); if (response != "n" && response != "no") { batchClient.JobOperations.DeleteJob(JobId); } Console.Write("Delete pool? [yes] no: "); response = Console.ReadLine().ToLower(); if (response != "n" && response != "no") { batchClient.PoolOperations.DeletePool(PoolId); } } }
private static void StartBatchTasks(List <ResourceFile> inputFiles) { BatchSharedKeyCredentials cred = new BatchSharedKeyCredentials(BatchAccountUrl, BatchAccountName, BatchAccountKey); using (BatchClient batchClient = BatchClient.Open(cred)) { // Create a Windows Server image, VM configuration, Batch pool ImageReference imageReference = CreateImageReference(); VirtualMachineConfiguration vmConfiguration = CreateVirtualMachineConfiguration(imageReference); CreateBatchPool(batchClient, vmConfiguration); CreateJob(batchClient); List <CloudTask> tasks = new List <CloudTask>(); for (int i = 0; i < inputFiles.Count; i++) { string taskId = String.Format("Task{0}", i); string inputFilename = inputFiles[i].FilePath; string taskCommandLine = String.Format("cmd /c type {0}", inputFilename); CloudTask task = new CloudTask(taskId, taskCommandLine); task.ResourceFiles = new List <ResourceFile> { inputFiles[i] }; tasks.Add(task); } batchClient.JobOperations.AddTask(JobId, tasks); TimeSpan timeout = TimeSpan.FromMinutes(30); Console.WriteLine("Monitoring all tasks for 'Completed' state, timeout in {0}...", timeout); IEnumerable <CloudTask> addedTasks = batchClient.JobOperations.ListTasks(JobId); batchClient.Utilities.CreateTaskStateMonitor().WaitAll(addedTasks, TaskState.Completed, timeout); // List tasks IEnumerable <CloudTask> completedtasks = batchClient.JobOperations.ListTasks(JobId); foreach (CloudTask task in completedtasks) { string nodeId = String.Format(task.ComputeNodeInformation.ComputeNodeId); Console.WriteLine("Task: {0}", task.Id); Console.WriteLine("Node: {0}", nodeId); Console.WriteLine("Standard out:"); Console.WriteLine(task.GetNodeFile(Constants.StandardOutFileName).ReadAsString()); } // delete job batchClient.JobOperations.DeleteJob(JobId); // delete pool batchClient.PoolOperations.DeletePool(PoolId); } }
public void Start() { CloudStorageAccount storageAccount = createCloudStorageAccount(); CloudBlobClient cloudBlobClient = storageAccount.CreateCloudBlobClient(); string contName = ConfigurationManager.AppSettings["ContainerInput"]; CloudBlobContainer cloudBlobContainer = getBlobContainer(cloudBlobClient, contName); List <string> files = getInputDataFiles(); List <ResourceFile> inputFiles = new List <ResourceFile>(); foreach (string f in files) { if (!String.IsNullOrEmpty(f)) { inputFiles.Add(GetResource(f, cloudBlobClient, cloudBlobContainer)); } } using (BatchClient client = getBatchClient()) { // Create a Batch pool, VM configuration, Windows Server image string poolId = ConfigurationManager.AppSettings["PoolId"]; int poolNodeCount = 1; string poolVMSize = "STANDARD_A1_v2"; string jobId = "DotNetQuickstartJob"; Console.WriteLine("Creating pool [{0}]...", poolId); ImageReference imageReference = new ImageReference( publisher: "MicrosoftWindowsServer", offer: "WindowsServer", sku: "2012-R2-datacenter-smalldisk", version: "latest"); VirtualMachineConfiguration virtualMachineConfiguration = new VirtualMachineConfiguration( imageReference: imageReference, nodeAgentSkuId: "batch.node.windows amd64"); try { if (client.PoolOperations.ListPools() != null && client.PoolOperations.ListPools().Count() > 0 && client.PoolOperations.ListPools().First(p => p.Id == poolId) != null) { client.PoolOperations.DeletePoolAsync(poolId).Wait(); } CloudPool pool = client.PoolOperations.CreatePool( poolId: poolId, targetDedicatedComputeNodes: poolNodeCount, virtualMachineSize: poolVMSize, virtualMachineConfiguration: virtualMachineConfiguration); pool.Commit(); } catch (BatchException be) { // Accept the specific error code PoolExists as that is expected if the pool already exists if (be.RequestInformation?.BatchError?.Code == BatchErrorCodeStrings.PoolExists) { Console.WriteLine("The pool {0} already existed when we tried to create it", poolId); } else { throw; // Any other exception is unexpected } } catch (Exception ex) { } try { client.JobOperations.DeleteJob(jobId); CloudJob job = client.JobOperations.CreateJob(); job.Id = jobId; job.PoolInformation = new PoolInformation { PoolId = poolId }; job.Commit(); } catch (BatchException be) { // Accept the specific error code JobExists as that is expected if the job already exists if (be.RequestInformation?.BatchError?.Code == BatchErrorCodeStrings.JobExists) { Console.WriteLine("The job {0} already existed when we tried to create it", jobId); } else { throw; // Any other exception is unexpected } } // Create a collection to hold the tasks that we'll be adding to the job Console.WriteLine("Adding {0} tasks to job [{1}]...", inputFiles.Count, jobId); List <CloudTask> tasks = new List <CloudTask>(); // Create each of the tasks to process one of the input files. for (int i = 0; i < inputFiles.Count; i++) { string taskId = String.Format("Task{0}", i); string inputFilename = inputFiles[i].FilePath; string taskCommandLine = String.Format("cmd /c type {0}", inputFilename); CloudTask task = new CloudTask(taskId, taskCommandLine); task.ResourceFiles = new List <ResourceFile> { inputFiles[i] }; tasks.Add(task); } // Add all tasks to the job. client.JobOperations.AddTask(jobId, tasks); // Monitor task success/failure, specifying a maximum amount of time to wait for the tasks to complete. TimeSpan timeout = TimeSpan.FromMinutes(10); Console.WriteLine("Monitoring all tasks for 'Completed' state, timeout in {0}...", timeout); IEnumerable <CloudTask> addedTasks = client.JobOperations.ListTasks(jobId); client.Utilities.CreateTaskStateMonitor().WaitAll(addedTasks, TaskState.Completed, timeout); Console.WriteLine("All tasks reached state Completed."); // Print task output Console.WriteLine(); Console.WriteLine("Printing task output..."); IEnumerable <CloudTask> completedtasks = client.JobOperations.ListTasks(jobId); Stopwatch timer = new Stopwatch(); timer.Start(); foreach (CloudTask task in completedtasks) { string nodeId = String.Format(task.ComputeNodeInformation.ComputeNodeId); Console.WriteLine("Task: {0}", task.Id); Console.WriteLine("Node: {0}", nodeId); Console.WriteLine("Standard out:"); } // Print out some timing info timer.Stop(); Console.WriteLine(); Console.WriteLine("Sample end: {0}", DateTime.Now); Console.WriteLine("Elapsed time: {0}", timer.Elapsed); // Clean up Storage resources if (cloudBlobContainer != null) { cloudBlobContainer.DeleteIfExistsAsync().Wait(); Console.WriteLine("Container [{0}] deleted.", contName); } else { Console.WriteLine("Container [{0}] does not exist, skipping deletion.", contName); } // Clean up Batch resources (if the user so chooses) Console.WriteLine(); Console.Write("Delete job? [yes] no: "); string response = Console.ReadLine().ToLower(); if (response != "n" && response != "no") { client.JobOperations.DeleteJob(jobId); } Console.Write("Delete pool? [yes] no: "); response = Console.ReadLine().ToLower(); if (response != "n" && response != "no") { client.PoolOperations.DeletePool(poolId); } } }
static void Main() { if (String.IsNullOrEmpty(BatchAccountName) || String.IsNullOrEmpty(BatchAccountKey) || String.IsNullOrEmpty(BatchAccountUrl) || String.IsNullOrEmpty(StorageAccountName) || String.IsNullOrEmpty(StorageAccountKey)) { throw new InvalidOperationException("One or more account credential strings have not been populated. Please ensure that your Batch and Storage account credentials have been specified."); } try { Console.WriteLine("Sample start: {0}", DateTime.Now); Console.WriteLine(); Stopwatch timer = new Stopwatch(); timer.Start(); // Create the blob client, for use in obtaining references to blob storage containers CloudBlobClient blobClient = CreateCloudBlobClient(StorageAccountName, StorageAccountKey); // Use the blob client to create the input container in Azure Storage const string inputContainerName = "input"; CloudBlobContainer container = blobClient.GetContainerReference(inputContainerName); container.CreateIfNotExistsAsync().Wait(); // The collection of data files that are to be processed by the tasks List <string> inputFilePaths = new List <string> { "taskdata0.txt", "taskdata1.txt", "taskdata2.txt" }; // Upload the data files to Azure Storage. This is the data that will be processed by each of the tasks that are // executed on the compute nodes within the pool. List <ResourceFile> inputFiles = new List <ResourceFile>(); foreach (string filePath in inputFilePaths) { inputFiles.Add(UploadFileToContainer(blobClient, inputContainerName, filePath)); } // Get a Batch client using account creds BatchSharedKeyCredentials cred = new BatchSharedKeyCredentials(BatchAccountUrl, BatchAccountName, BatchAccountKey); using (BatchClient batchClient = BatchClient.Open(cred)) { Console.WriteLine("Creating pool [{0}]...", PoolId); // Create a Windows Server image, VM configuration, Batch pool ImageReference imageReference = CreateImageReference(); VirtualMachineConfiguration vmConfiguration = CreateVirtualMachineConfiguration(imageReference); CreateBatchPool(batchClient, vmConfiguration); // Create a Batch job Console.WriteLine("Creating job [{0}]...", JobId); try { CloudJob job = batchClient.JobOperations.CreateJob(); job.Id = JobId; job.PoolInformation = new PoolInformation { PoolId = PoolId }; job.Commit(); } catch (BatchException be) { // Accept the specific error code JobExists as that is expected if the job already exists if (be.RequestInformation?.BatchError?.Code == BatchErrorCodeStrings.JobExists) { Console.WriteLine("The job {0} already existed when we tried to create it", JobId); } else { throw; // Any other exception is unexpected } } // Create a collection to hold the tasks that we'll be adding to the job Console.WriteLine("Adding {0} tasks to job [{1}]...", inputFiles.Count, JobId); List <CloudTask> tasks = new List <CloudTask>(); // Create each of the tasks to process one of the input files. for (int i = 0; i < inputFiles.Count; i++) { string taskId = String.Format("Task{0}", i); string inputFilename = inputFiles[i].FilePath; string taskCommandLine = String.Format("cmd /c type {0}", inputFilename); CloudTask task = new CloudTask(taskId, taskCommandLine); task.ResourceFiles = new List <ResourceFile> { inputFiles[i] }; tasks.Add(task); } // Add all tasks to the job. batchClient.JobOperations.AddTask(JobId, tasks); // Monitor task success/failure, specifying a maximum amount of time to wait for the tasks to complete. TimeSpan timeout = TimeSpan.FromMinutes(30); Console.WriteLine("Monitoring all tasks for 'Completed' state, timeout in {0}...", timeout); IEnumerable <CloudTask> addedTasks = batchClient.JobOperations.ListTasks(JobId); batchClient.Utilities.CreateTaskStateMonitor().WaitAll(addedTasks, TaskState.Completed, timeout); Console.WriteLine("All tasks reached state Completed."); // Print task output Console.WriteLine(); Console.WriteLine("Printing task output..."); IEnumerable <CloudTask> completedtasks = batchClient.JobOperations.ListTasks(JobId); foreach (CloudTask task in completedtasks) { string nodeId = String.Format(task.ComputeNodeInformation.ComputeNodeId); Console.WriteLine("Task: {0}", task.Id); Console.WriteLine("Node: {0}", nodeId); Console.WriteLine("Standard out:"); Console.WriteLine(task.GetNodeFile(Constants.StandardOutFileName).ReadAsString()); } // Print out some timing info timer.Stop(); Console.WriteLine(); Console.WriteLine("Sample end: {0}", DateTime.Now); Console.WriteLine("Elapsed time: {0}", timer.Elapsed); // Clean up Storage resources container.DeleteIfExistsAsync().Wait(); Console.WriteLine("Container [{0}] deleted.", inputContainerName); // Clean up Batch resources (if the user so chooses) Console.WriteLine(); Console.Write("Delete job? [yes] no: "); string response = Console.ReadLine().ToLower(); if (response != "n" && response != "no") { batchClient.JobOperations.DeleteJob(JobId); } Console.Write("Delete pool? [yes] no: "); response = Console.ReadLine().ToLower(); if (response != "n" && response != "no") { batchClient.PoolOperations.DeletePool(PoolId); } } } finally { Console.WriteLine(); Console.WriteLine("Sample complete, hit ENTER to exit..."); Console.ReadLine(); } }