public void CloudJob_WhenSendingToTheServer_HasExpectedUnboundProperties()
        {
            const string jobId              = "id-123";
            const string displayName        = "DisplayNameFoo";
            MetadataItem metadataItem       = new MetadataItem("foo", "bar");
            const int    priority           = 0;
            const string applicationId      = "testApp";
            const string applicationVersion = "beta";

            using BatchClient client = ClientUnitTestCommon.CreateDummyClient();
            CloudJob cloudJob = client.JobOperations.CreateJob(jobId, new PoolInformation {
                AutoPoolSpecification = new AutoPoolSpecification {
                    KeepAlive = false
                }
            });

            cloudJob.Id          = jobId;
            cloudJob.DisplayName = displayName;
            cloudJob.Metadata    = new List <MetadataItem> {
                metadataItem
            };
            cloudJob.Priority       = priority;
            cloudJob.JobManagerTask = new JobManagerTask
            {
                ApplicationPackageReferences = new List <ApplicationPackageReference>
                {
                    new ApplicationPackageReference {
                        ApplicationId = applicationId, Version = applicationVersion
                    }
                },
                AuthenticationTokenSettings = new AuthenticationTokenSettings()
                {
                    Access = AccessScope.Job
                }
            };

            cloudJob.OnAllTasksComplete = OnAllTasksComplete.NoAction;
            cloudJob.OnTaskFailure      = OnTaskFailure.NoAction;


            Assert.Throws <InvalidOperationException>(() => cloudJob.Url); // cannot read a Url since it's unbound at this point.
            Assert.Equal(cloudJob.Id, jobId);                              // can set an unbound object
            Assert.Equal(cloudJob.Metadata.First().Name, metadataItem.Name);
            Assert.Equal(cloudJob.Metadata.First().Value, metadataItem.Value);
            Assert.Equal(OnAllTasksComplete.NoAction, cloudJob.OnAllTasksComplete);
            Assert.Equal(OnTaskFailure.NoAction, cloudJob.OnTaskFailure);

            cloudJob.Commit(additionalBehaviors: InterceptorFactory.CreateAddJobRequestInterceptor());

            // writing isn't allowed for a job that is in an invalid state.
            Assert.Throws <InvalidOperationException>(() => cloudJob.Id          = "cannot-change-id");
            Assert.Throws <InvalidOperationException>(() => cloudJob.DisplayName = "cannot-change-display-name");

            AuthenticationTokenSettings authenticationTokenSettings = new AuthenticationTokenSettings {
                Access = AccessScope.Job
            };

            Assert.Throws <InvalidOperationException>(() => { cloudJob.JobManagerTask.AuthenticationTokenSettings = authenticationTokenSettings; });
        }
Ejemplo n.º 2
0
        public void Bug2338301_CheckStreamPositionAfterFileRead()
        {
            void test()
            {
                using BatchClient batchCli = TestUtilities.OpenBatchClient(TestUtilities.GetCredentialsFromEnvironment());
                JobOperations jobOperations = batchCli.JobOperations;
                {
                    string jobId = "Bug2338301Job-" + TestUtilities.GetMyName();

                    try
                    {
                        const string taskId = "hiWorld";

                        //
                        // Create the job
                        //
                        CloudJob unboundJob = jobOperations.CreateJob(jobId, new PoolInformation()
                        {
                            PoolId = poolFixture.PoolId
                        });
                        unboundJob.Commit();

                        CloudJob  boundJob = jobOperations.GetJob(jobId);
                        CloudTask myTask   = new CloudTask(taskId, "cmd /c echo hello world");

                        boundJob.AddTask(myTask);

                        testOutputHelper.WriteLine("Initial job commit()");

                        //
                        // Wait for task to go to completion
                        //
                        Utilities        utilities        = batchCli.Utilities;
                        TaskStateMonitor taskStateMonitor = utilities.CreateTaskStateMonitor();
                        taskStateMonitor.WaitAll(
                            boundJob.ListTasks(),
                            Microsoft.Azure.Batch.Common.TaskState.Completed,
                            TimeSpan.FromMinutes(3));

                        CloudTask boundTask = boundJob.GetTask(taskId);

                        //Get the task file
                        const string fileToGet = "stdout.txt";
                        NodeFile     file      = boundTask.GetNodeFile(fileToGet);

                        //Download the file data
                        string result = file.ReadAsString();
                        Assert.True(result.Length > 0);
                    }
                    finally
                    {
                        jobOperations.DeleteJob(jobId);
                    }
                }
            }

            SynchronizationContextHelper.RunTest(test, TestTimeout);
        }
Ejemplo n.º 3
0
        public void TestOMJobPrepReleaseRunOnNaiveComputeNode()
        {
            string jobId = "TestOMJobPrepReleaseRunOnNaiveComputeNode-" + TestUtilities.GetMyName();
            Action test  = () =>
            {
                using (BatchClient client = TestUtilities.OpenBatchClient(TestUtilities.GetCredentialsFromEnvironment()))
                {
                    try
                    {
                        // create job with prep/release
                        {
                            CloudJob unboundJob = client.JobOperations.CreateJob(jobId, null);
                            unboundJob.PoolInformation        = new PoolInformation();
                            unboundJob.PoolInformation.PoolId = this.poolFixture.PoolId;

                            // add the jobPrep task to the job
                            {
                                JobPreparationTask prep = new JobPreparationTask("cmd /c echo JobPrep!!");
                                unboundJob.JobPreparationTask = prep;

                                prep.WaitForSuccess = true; // be explicit even though this is the default.  need JP/JP to not run
                            }


                            // add a jobRelease task to the job
                            {
                                JobReleaseTask relTask = new JobReleaseTask(JobReleaseTaskCommandLine);
                                unboundJob.JobReleaseTask = relTask;
                            }

                            // add the job to the service
                            unboundJob.Commit();
                        }

                        // the victim nodes.  pool should have size 1.
                        List <ComputeNode> computeNodes = client.PoolOperations.ListComputeNodes(this.poolFixture.PoolId).ToList();

                        Assert.Single(computeNodes);
                        // now we have a job with zero tasks... lets call get-status methods

                        // call it
                        List <JobPreparationAndReleaseTaskExecutionInformation> jrStatusList =
                            client.JobOperations.ListJobPreparationAndReleaseTaskStatus(jobId).ToList();

                        JobPreparationAndReleaseTaskExecutionInformation prepAndReleaseStatus = jrStatusList.FirstOrDefault();

                        Assert.Null(prepAndReleaseStatus);
                    }
                    finally
                    {
                        // cleanup
                        TestUtilities.DeleteJobIfExistsAsync(client, jobId).Wait();
                    }
                }
            };

            SynchronizationContextHelper.RunTest(test, TestTimeout);
        }
Ejemplo n.º 4
0
        public static void Submit()
        {
            Log("Start submission process.");
            state = null;

            BatchSharedKeyCredentials cred = new BatchSharedKeyCredentials(Settings.batchEndpoint, Settings.batchAccount, Settings.batchKey);

            using (BatchClient client = BatchClient.Open(cred)) // <-- connect to the cluster
            {
                #region job submission

                string          jobname = prefix + Environment.GetEnvironmentVariable("USERNAME") + "_" + DateTime.Now.ToString("yyyyMMdd-HHmmss");
                PoolInformation pool    = new PoolInformation();
                pool.PoolId = Settings.poolname;

                CloudJob job = client.JobOperations.CreateJob(jobname, pool); // <-- create a workitem that runs on pool "trdemo"

                Log("Submitting...");
                job.Commit();
                jobName = jobname;
                Log(string.Format("Job {0} created.", jobname));

                job = client.JobOperations.GetJob(jobname);

                Log("Analyzing input blobs...");
                string inputcontainersas  = StorageHelper.GetContainerSAS(Settings.inputContainer);
                string outputcontainersas = StorageHelper.GetContainerSAS(Settings.outputContainer);
                foreach (string blob in StorageHelper.ListBlobs(Settings.inputContainer))
                {
                    string filename = System.IO.Path.GetFileName((new Uri(blob)).LocalPath);
                    string taskname = "task_" + System.IO.Path.GetFileNameWithoutExtension(filename);

                    // prepare the command line
                    string cli;
                    cli  = ". robocopy.exe ${env:WATASK_TVM_ROOT_DIR}\\shared\\ . *.*;";
                    cli += "ffmpeg.exe -i {0} -vf 'movie=microsoft.png [watermark]; [in][watermark] overlay=10:main_h-overlay_h-10 [out]' {0}.output.avi;".Replace("{0}", filename);
                    cli += "azcopy.exe . {0} *.output.avi /destsas:'{1}' /y".Replace("{0}", Settings.outputContainer).Replace("{1}", outputcontainersas);

                    cli = string.Format("powershell -command \"{0}\"", cli);

                    // prepare task object
                    CloudTask task = new CloudTask(taskname, cli);
                    task.ResourceFiles = new List <ResourceFile>();
                    task.ResourceFiles.Add(new ResourceFile(blob + inputcontainersas, filename));

                    job.AddTask(task); // <-- add Task
                }


                #endregion job submission

                ThreadPool.QueueUserWorkItem((x) => { Monitor(); });

                client.Utilities.CreateTaskStateMonitor().WaitAll(client.JobOperations.ListTasks(jobname), TaskState.Completed, new TimeSpan(0, 60, 0));
                client.JobOperations.GetJob(jobname).Terminate();
            }
        }
        private static void CreateJob(BatchClient batchClient)
        {
            CloudJob job = batchClient.JobOperations.CreateJob();

            job.Id = JobId;
            job.PoolInformation = new PoolInformation {
                PoolId = PoolId
            };
            job.Commit();
        }
        private CloudJob CreateBoundJob(BatchClient client, string jobId, Action <CloudJob> jobSetup = null)
        {
            CloudJob cloudJob = client.JobOperations.CreateJob(jobId, new PoolInformation());

            cloudJob.PoolInformation.PoolId = poolFixture.PoolId;
            jobSetup?.Invoke(cloudJob);
            cloudJob.Commit();

            return(client.JobOperations.GetJob(jobId));
        }
Ejemplo n.º 7
0
        public void SetUpdateJobConditionalHeader()
        {
            Action test = () =>
            {
                using (BatchClient batchCli = TestUtilities.OpenBatchClientFromEnvironmentAsync().Result)
                {
                    string jobId = "JobConditionalHeaders-" + TestUtilities.GetMyName();
                    try
                    {
                        PoolInformation poolInfo = new PoolInformation()
                        {
                            PoolId = "Fake"
                        };

                        CloudJob unboundJob = batchCli.JobOperations.CreateJob(jobId, poolInfo);
                        unboundJob.Commit();

                        CloudJob boundJob = batchCli.JobOperations.GetJob(jobId);

                        string capturedEtag1 = boundJob.ETag;
                        this.testOutputHelper.WriteLine("Etag is: {0}", capturedEtag1);
                        Assert.NotNull(capturedEtag1);

                        boundJob.Constraints = new JobConstraints(TimeSpan.FromMinutes(60), 0);

                        BatchClientBehavior updateInterceptor = new Protocol.RequestInterceptor(
                            (req) =>
                        {
                            var typedParams = req.Options as Protocol.Models.JobUpdateOptions;
                            if (typedParams != null)
                            {
                                typedParams.IfMatch = capturedEtag1;
                            }
                        });

                        //Update bound job with if-match header, it should succeed
                        boundJob.Commit(additionalBehaviors: new[] { updateInterceptor });

                        boundJob = batchCli.JobOperations.GetJob(jobId);

                        boundJob.Constraints = new JobConstraints(TimeSpan.FromMinutes(30), 1);

                        //Update bound job with if-match header, it should fail
                        Exception e = TestUtilities.AssertThrows <BatchException>(() => boundJob.Commit(additionalBehaviors: new[] { updateInterceptor }));
                        TestUtilities.AssertIsBatchExceptionAndHasCorrectAzureErrorCode(e, BatchErrorCodeStrings.ConditionNotMet, this.testOutputHelper);
                    }
                    finally
                    {
                        TestUtilities.DeleteJobIfExistsAsync(batchCli, jobId).Wait();
                    }
                }
            };

            SynchronizationContextHelper.RunTest(test, LongTestTimeout);
        }
Ejemplo n.º 8
0
        public void CanSpecifyTaskDependencyIds()
        {
            using (BatchClient batchCli = TestUtilities.OpenBatchClientFromEnvironmentAsync().Result)
            {
                string jobId = GenerateJobId();

                try
                {
                    CloudJob unboundJob = batchCli.JobOperations.CreateJob(jobId, new PoolInformation());
                    unboundJob.PoolInformation = new PoolInformation()
                    {
                        PoolId = this.poolFixture.PoolId
                    };

                    unboundJob.UsesTaskDependencies = true;
                    unboundJob.Commit();
                    var taskId = Guid.NewGuid().ToString();

                    IList <TaskIdRange> taskIdRanges = new List <TaskIdRange>
                    {
                        new TaskIdRange(1, 5),
                        new TaskIdRange(8, 8)
                    };

                    IList <string> taskIds = new List <string> {
                        "1"
                    };

                    var boundJob = batchCli.JobOperations.GetJob(jobId);

                    CloudTask taskToAdd = new CloudTask(taskId, "cmd.exe")
                    {
                        DependsOn = new TaskDependencies(taskIds, taskIdRanges),
                    };

                    boundJob.AddTask(taskToAdd);

                    CloudTask task             = boundJob.GetTask(taskId);
                    var       dependedOnRange  = task.DependsOn.TaskIdRanges.First();
                    var       dependedOnTaskId = task.DependsOn.TaskIds.First();

                    Assert.Equal(1, dependedOnRange.Start);
                    Assert.Equal(5, dependedOnRange.End);
                    Assert.Equal("1", dependedOnTaskId);

                    Assert.True(boundJob.UsesTaskDependencies);
                }
                finally
                {
                    TestUtilities.DeleteJobIfExistsAsync(batchCli, jobId).Wait();
                }
            }
        }
Ejemplo n.º 9
0
        // CreateJob(): Creates a job in the specified pool.
        //   batchClient: A BatchClient object.
        //   jobId: ID of the job to create.
        //   poolId: ID of the CloudPool object in which to create the job.
        private static void CreateJob(BatchClient batchClient, string jobId, string poolId)
        {
            Console.WriteLine("Creating job [{0}]...", jobId);

            CloudJob job = batchClient.JobOperations.CreateJob();

            job.Id = jobId;
            job.PoolInformation = new PoolInformation {
                PoolId = poolId
            };

            job.Commit();
        }
Ejemplo n.º 10
0
        /// <summary>
        /// Cria um novo job associado um pool previamente criado.
        /// </summary>
        public CloudJob CreateJob(CloudPool pool, string jobId)
        {
            CloudJob job = client.JobOperations.CreateJob();

            job.Id = jobId;
            job.PoolInformation = new PoolInformation
            {
                PoolId = pool.Id
            };

            job.Commit();

            return(job);
        }
Ejemplo n.º 11
0
        public void CreateJob(string jobId, Uri resourceFile, string commandLine, string storageContainerSAS)
        {
            CloudJob unboundJob = Client.JobOperations.CreateJob();

            unboundJob.Id = jobId;
            unboundJob.PoolInformation = new PoolInformation()
            {
                PoolId = PoolId
            };
            unboundJob.JobPreparationTask = CreateJobPreparationTask();
            unboundJob.JobManagerTask     = new JobManagerTask()
            {
                Id           = jobId,
                CommandLine  = commandLine,
                RunExclusive = false,

                ResourceFiles = resourceFile != null
                    ? new List <ResourceFile>()
                {
                    new ResourceFile(resourceFile.AbsoluteUri, AzureBatchFileNames.GetTaskJarFileName())
                }
                    : new List <ResourceFile>(),

                EnvironmentSettings = new List <EnvironmentSetting>
                {
                    new EnvironmentSetting(AzureStorageContainerSasToken, storageContainerSAS)
                },

                // This setting will signal Batch to generate an access token and pass it
                // to the Job Manager Task (aka the Driver) as an environment variable.
                // For more info, see
                // https://docs.microsoft.com/en-us/dotnet/api/microsoft.azure.batch.cloudtask.authenticationtokensettings
                AuthenticationTokenSettings = new AuthenticationTokenSettings()
                {
                    Access = AccessScope.Job
                },
                ContainerSettings = CreateTaskContainerSettings(jobId),
            };

            if (AreContainersEnabled)
            {
                unboundJob.JobManagerTask.UserIdentity =
                    new UserIdentity(autoUserSpecification: new AutoUserSpecification(elevationLevel: ElevationLevel.Admin));
            }

            unboundJob.Commit();

            LOGGER.Log(Level.Info, "Submitted job {0}, commandLine {1} ", jobId, commandLine);
        }
Ejemplo n.º 12
0
        private static void CreateJob(Settings unzipperSettings, BatchClient client)
        {
            Console.WriteLine("Creating job: " + unzipperSettings.JobId);
            // get an empty unbound Job
            CloudJob unboundJob = client.JobOperations.CreateJob();

            unboundJob.Id = unzipperSettings.JobId;
            unboundJob.PoolInformation = new PoolInformation()
            {
                PoolId = unzipperSettings.PoolId
            };

            // Commit Job to create it in the service
            unboundJob.Commit();
        }
Ejemplo n.º 13
0
        public void TestJobCompletesWhenAllItsTasksComplete()
        {
            Action test = () =>
            {
                using (BatchClient client = TestUtilities.OpenBatchClient(TestUtilities.GetCredentialsFromEnvironment()))
                {
                    //Create a job
                    string jobId        = Constants.DefaultConveniencePrefix + TestUtilities.GetMyName() + "-TestJobCompletesWhenAllItsTasksComplete";
                    string taskIdPrefix = "task-id";

                    try
                    {
                        CloudJob boundJob = CreateBoundJob(client, jobId);

                        var cloudTasks = new List <CloudTask>();
                        for (var i = 0; i < 4; i++)
                        {
                            cloudTasks.Add(new CloudTask(taskIdPrefix + "-" + i, "cmd /c ping 127.0.0.1"));
                        }

                        boundJob.AddTask(cloudTasks);

                        boundJob.OnAllTasksComplete = OnAllTasksComplete.TerminateJob;
                        boundJob.Commit();

                        boundJob.Refresh();

                        TestUtilities.WaitForJobStateAsync(boundJob, TimeSpan.FromMinutes(2), JobState.Completed).Wait();

                        Assert.Equal(JobState.Completed, boundJob.State);
                        Assert.Equal("AllTasksCompleted", boundJob.ExecutionInformation.TerminateReason);
                    }
                    finally
                    {
                        TestUtilities.DeleteJobIfExistsAsync(client, jobId).Wait();
                    }
                }
            };

            SynchronizationContextHelper.RunTest(test, TestTimeout);
        }
Ejemplo n.º 14
0
        public void Execute()
        {
            string reqfile = _config.ResourceFolder + _config.ResourceName;

            //> CREATE
            try
            {
                ResourceFile req = CreateResourceFile(reqfile);

                BatchTokenCredentials cred = new BatchTokenCredentials(_config.BatchAccountUrl, GetToken());
                Client = BatchClient.Open(cred);
                ImageReference imgref           = new ImageReference(_config.VmImage);
                VirtualMachineConfiguration vmc = new VirtualMachineConfiguration(imageReference: imgref, nodeAgentSkuId: "batch.node.windows amd64");
                CreateBatchPool(vmc, _config.VmSKU);
                // Create Job
                CloudJob job = Client.JobOperations.CreateJob();
                job.Id = _jobId;
                job.PoolInformation = new PoolInformation {
                    PoolId = _poolId
                };
                job.Commit();
                // Create Tasks
                List <CloudTask> tasks  = new List <CloudTask>();
                string           taskId = "task" + System.DateTime.Now.Ticks.ToString();
                string           cmd    = String.Format("cmd /c type {0}", req.FilePath);
                CloudTask        task   = new CloudTask(taskId, cmd);
                task.ResourceFiles = new List <ResourceFile> {
                    req
                };
                tasks.Add(task);
                Client.JobOperations.AddTask(_jobId, tasks);
            }

            catch (Exception x)
            {
                //TODO: Log Somewhere
                throw;
            }
        }
        public void TestContainerTask()
        {
            void test()
            {
                using BatchClient client = TestUtilities.OpenBatchClient(TestUtilities.GetCredentialsFromEnvironment());
                string jobId = "ContainerJob" + TestUtilities.GetMyName();

                try
                {
                    CloudJob job = client.JobOperations.CreateJob(jobId, new PoolInformation {
                        PoolId = poolFixture.PoolId
                    });
                    job.Commit();

                    CloudTask newTask = new CloudTask("a", "cat /etc/centos-release")
                    {
                        ContainerSettings = new TaskContainerSettings("centos")
                    };
                    client.JobOperations.AddTask(jobId, newTask);

                    IPagedEnumerable <CloudTask> tasks = client.JobOperations.ListTasks(jobId);

                    TaskStateMonitor monitor = client.Utilities.CreateTaskStateMonitor();
                    monitor.WaitAll(tasks, TaskState.Completed, TimeSpan.FromMinutes(7));

                    CloudTask task = tasks.Single();
                    task.Refresh();

                    Assert.Equal("ContainerPoolNotSupported", task.ExecutionInformation.FailureInformation.Code);
                }
                finally
                {
                    TestUtilities.DeleteJobIfExistsAsync(client, jobId).Wait();
                }
            }

            SynchronizationContextHelper.RunTest(test, TimeSpan.FromMinutes(10));
        }
        protected void CreateJob(
            BatchClient batchClient,
            string jobName,
            Dictionary <string, string> jobMetadata)
        {
            // Create a Batch job
            Console.WriteLine("Creating job [{0}]...", jobName);
            try
            {
                // Create a Batch job
                CloudJob job = batchClient.JobOperations.CreateJob();
                job.Id = jobName;
                job.PoolInformation = new PoolInformation {
                    PoolId = PoolName
                };                                                            // Associate the job to a PoolId

                if (jobMetadata != null)
                {
                    job.Metadata = GetMetadataItems(jobMetadata);
                }

                // Commit the job to Batch Service
                job.Commit();
            }
            catch (BatchException be)
            {
                // Accept the specific error code JobExists as that is expected if the job already exists
                if (be.RequestInformation?.BatchError?.Code == BatchErrorCodeStrings.JobExists)
                {
                    Console.WriteLine("The job [{0}] already exists", jobName);
                }
                else
                {
                    throw; // Any other exception is unexpected
                }
            };
        }
Ejemplo n.º 17
0
        private CloudJob CreateJobIfNotExists(String jobID, String poolID)
        {
            messageContainer.AddInformationMessage("Configuring job...");

            CloudJob cJob = null;

            try { cJob = batchClient.JobOperations.GetJob(jobID); } catch { }

            if (cJob == null)
            {
                messageContainer.AddInformationMessage("Creating job...");
                cJob    = batchClient.JobOperations.CreateJob();
                cJob.Id = jobID;
                cJob.PoolInformation = new PoolInformation {
                    PoolId = poolID
                };

                cJob.Commit();
                messageContainer.AddInformationMessage("Job created...");
            }
            messageContainer.AddInformationMessage("Job configured...");

            return(cJob);
        }
Ejemplo n.º 18
0
 private static void CreateJob(BatchClient batchClient)
 {
     try
     {
         CloudJob job = batchClient.JobOperations.CreateJob();
         job.Id = JobId;
         job.PoolInformation = new PoolInformation {
             PoolId = PoolId
         };
         job.Commit();
     }
     catch (BatchException be)
     {
         // Accept the specific error code JobExists as that is expected if the job already exists
         if (be.RequestInformation?.BatchError?.Code == BatchErrorCodeStrings.JobExists)
         {
             Console.WriteLine("The job {0} already existed when we tried to create it", JobId);
         }
         else
         {
             throw;         // Any other exception is unexpected
         }
     }
 }
Ejemplo n.º 19
0
        public void Bug2329884_ComputeNodeRecentTasksAndComputeNodeError()
        {
            Action test = () =>
            {
                using (BatchClient batchCli = TestUtilities.OpenBatchClientAsync(TestUtilities.GetCredentialsFromEnvironment()).Result)
                {
                    string jobId = "Bug2329884Job-" + TestUtilities.GetMyName();
                    Protocol.RequestInterceptor interceptor = null;

                    try
                    {
                        const string taskId = "hiWorld";

                        //
                        // Create the job
                        //
                        CloudJob unboundJob = batchCli.JobOperations.CreateJob(jobId, new PoolInformation());
                        unboundJob.PoolInformation.PoolId = this.poolFixture.PoolId;

                        unboundJob.Commit();

                        CloudJob  boundJob = batchCli.JobOperations.GetJob(jobId);
                        CloudTask myTask   = new CloudTask(taskId, "cmd /c echo hello world");

                        boundJob.AddTask(myTask);

                        this.testOutputHelper.WriteLine("Initial job commit()");

                        //
                        // Wait for task to go to completion
                        //
                        Utilities        utilities        = batchCli.Utilities;
                        TaskStateMonitor taskStateMonitor = utilities.CreateTaskStateMonitor();

                        taskStateMonitor.WaitAll(
                            boundJob.ListTasks(),
                            Microsoft.Azure.Batch.Common.TaskState.Completed,
                            new TimeSpan(0, 3 /*min*/, 0));

                        CloudTask boundTask = boundJob.GetTask(taskId);

                        //Since the compute node name comes back as "Node:<computeNodeId>" we need to split on : to get the actual compute node name
                        string computeNodeId = boundTask.ComputeNodeInformation.AffinityId.Split(':')[1];

                        //
                        // Check recent tasks
                        //
                        ComputeNode computeNode = batchCli.PoolOperations.GetComputeNode(this.poolFixture.PoolId, computeNodeId);

                        this.testOutputHelper.WriteLine("Recent tasks:");

                        foreach (TaskInformation recentTask in computeNode.RecentTasks)
                        {
                            this.testOutputHelper.WriteLine("Compute node has recent task Job: {0}, Task: {1}, State: {2}, Subtask: {3}",
                                                            recentTask.JobId,
                                                            recentTask.TaskId,
                                                            recentTask.TaskState,
                                                            recentTask.SubtaskId);
                        }

                        TaskInformation myTaskInfo = computeNode.RecentTasks.First(taskInfo => taskInfo.JobId.Equals(
                                                                                       jobId, StringComparison.InvariantCultureIgnoreCase) &&
                                                                                   taskInfo.TaskId.Equals(taskId, StringComparison.InvariantCultureIgnoreCase));

                        Assert.Equal(TaskState.Completed, myTaskInfo.TaskState);
                        Assert.NotNull(myTaskInfo.ExecutionInformation);
                        Assert.Equal(0, myTaskInfo.ExecutionInformation.ExitCode);

                        //
                        // Check compute node Error
                        //
                        const string expectedErrorCode    = "TestErrorCode";
                        const string expectedErrorMessage = "Test error message";
                        const string nvpValue             = "Test";

                        //We use mocking to return a fake compute node object here to test Compute Node Error because we cannot force one easily
                        interceptor = new Protocol.RequestInterceptor((req =>
                        {
                            if (req is ComputeNodeGetBatchRequest)
                            {
                                var typedRequest = req as ComputeNodeGetBatchRequest;

                                typedRequest.ServiceRequestFunc = (token) =>
                                {
                                    var response = new AzureOperationResponse <Protocol.Models.ComputeNode, Protocol.Models.ComputeNodeGetHeaders>();

                                    List <Protocol.Models.ComputeNodeError> errors =
                                        new List <Protocol.Models.ComputeNodeError>();

                                    //Generate first Compute Node Error
                                    List <Protocol.Models.NameValuePair> nvps =
                                        new List <Protocol.Models.NameValuePair>();
                                    nvps.Add(new Protocol.Models.NameValuePair()
                                    {
                                        Name = nvpValue, Value = nvpValue
                                    });

                                    Protocol.Models.ComputeNodeError error1 = new Protocol.Models.ComputeNodeError();
                                    error1.Code = expectedErrorCode;
                                    error1.Message = expectedErrorMessage;
                                    error1.ErrorDetails = nvps;

                                    errors.Add(error1);

                                    //Generate second Compute Node Error
                                    nvps = new List <Protocol.Models.NameValuePair>();
                                    nvps.Add(new Protocol.Models.NameValuePair()
                                    {
                                        Name = nvpValue, Value = nvpValue
                                    });

                                    Protocol.Models.ComputeNodeError error2 = new Protocol.Models.ComputeNodeError();
                                    error2.Code = expectedErrorCode;
                                    error2.Message = expectedErrorMessage;
                                    error2.ErrorDetails = nvps;

                                    errors.Add(error2);

                                    Protocol.Models.ComputeNode protoComputeNode = new Protocol.Models.ComputeNode();
                                    protoComputeNode.Id = computeNodeId;
                                    protoComputeNode.State = Protocol.Models.ComputeNodeState.Idle;
                                    protoComputeNode.Errors = errors;

                                    response.Body = protoComputeNode;

                                    return(Task.FromResult(response));
                                };
                            }
                        }));

                        batchCli.PoolOperations.CustomBehaviors.Add(interceptor);

                        computeNode = batchCli.PoolOperations.GetComputeNode(this.poolFixture.PoolId, computeNodeId);

                        Assert.Equal(computeNodeId, computeNode.Id);
                        Assert.NotNull(computeNode.Errors);
                        Assert.Equal(2, computeNode.Errors.Count());

                        foreach (ComputeNodeError computeNodeError in computeNode.Errors)
                        {
                            Assert.Equal(expectedErrorCode, computeNodeError.Code);
                            Assert.Equal(expectedErrorMessage, computeNodeError.Message);
                            Assert.NotNull(computeNodeError.ErrorDetails);
                            Assert.Equal(1, computeNodeError.ErrorDetails.Count());
                            Assert.Contains(nvpValue, computeNodeError.ErrorDetails.First().Name);
                        }
                    }
                    finally
                    {
                        batchCli.JobOperations.DeleteJob(jobId);
                    }
                }
            };

            SynchronizationContextHelper.RunTest(test, TestTimeout);
        }
        public void CanAddTaskWithFilesToStage()
        {
            StagingStorageAccount storageCreds = TestUtilities.GetStorageCredentialsFromEnvironment();

            using (BatchClient batchCli = TestUtilities.OpenBatchClient(TestUtilities.GetCredentialsFromEnvironment()))
            {
                string jobId = "TestTaskWithFilesToStage-" + TestUtilities.GetMyName();

                try
                {
                    CloudJob job = batchCli.JobOperations.CreateJob(jobId, new PoolInformation()
                    {
                        PoolId = this.poolFixture.PoolId
                    });
                    job.Commit();
                    CloudJob boundJob = batchCli.JobOperations.GetJob(jobId);

                    CloudTask myTask = new CloudTask(id: "CountWordsTask", commandline: @"cmd /c dir /s .. & dir & wc localwords.txt");

                    myTask.FilesToStage = new List <IFileStagingProvider>
                    {
                        new FileToStage(Resources.LocalWordsDotText, storageCreds)
                    };

                    // add the task to the job
                    var artifacts        = boundJob.AddTask(myTask);
                    var specificArtifact = artifacts[typeof(FileToStage)];
                    SequentialFileStagingArtifact sfsa = specificArtifact as SequentialFileStagingArtifact;

                    Assert.NotNull(sfsa);

                    // Open the new Job as bound.
                    CloudPool boundPool = batchCli.PoolOperations.GetPool(boundJob.ExecutionInformation.PoolId);

                    // wait for the task to complete
                    TaskStateMonitor taskStateMonitor = batchCli.Utilities.CreateTaskStateMonitor();

                    taskStateMonitor.WaitAll(
                        boundJob.ListTasks(),
                        Microsoft.Azure.Batch.Common.TaskState.Completed,
                        TimeSpan.FromMinutes(10),
                        controlParams: null,
                        additionalBehaviors: new[]
                    {
                        // spam/logging interceptor
                        new Microsoft.Azure.Batch.Protocol.RequestInterceptor((x) =>
                        {
                            this.testOutputHelper.WriteLine("Issuing request type: " + x.GetType().ToString());

                            try
                            {
                                // print out the compute node states... we are actually waiting on the compute nodes
                                List <ComputeNode> allComputeNodes = boundPool.ListComputeNodes().ToList();

                                this.testOutputHelper.WriteLine("    #compute nodes: " + allComputeNodes.Count);

                                allComputeNodes.ForEach(
                                    (icn) =>
                                {
                                    this.testOutputHelper.WriteLine("  computeNode.id: " + icn.Id + ", state: " + icn.State);
                                });
                            }
                            catch (Exception ex)
                            {
                                // there is a race between the pool-life-job and the end of the job.. and the ListComputeNodes above
                                Assert.True(false, "SampleWithFilesAndPool probably can ignore this if its pool not found: " + ex.ToString());
                            }
                        })
                    });

                    List <CloudTask> tasks           = boundJob.ListTasks().ToList();
                    CloudTask        myCompletedTask = tasks.Single();

                    foreach (CloudTask curTask in tasks)
                    {
                        this.testOutputHelper.WriteLine("Task Id: " + curTask.Id + ", state: " + curTask.State);
                    }

                    boundPool.Refresh();

                    this.testOutputHelper.WriteLine("Pool Id: " + boundPool.Id + ", state: " + boundPool.State);

                    string stdOut = myCompletedTask.GetNodeFile(Constants.StandardOutFileName).ReadAsString();
                    string stdErr = myCompletedTask.GetNodeFile(Constants.StandardErrorFileName).ReadAsString();

                    this.testOutputHelper.WriteLine("StdOut: ");
                    this.testOutputHelper.WriteLine(stdOut);

                    this.testOutputHelper.WriteLine("StdErr: ");
                    this.testOutputHelper.WriteLine(stdErr);

                    this.testOutputHelper.WriteLine("Task Files:");

                    foreach (NodeFile curFile in myCompletedTask.ListNodeFiles(recursive: true))
                    {
                        this.testOutputHelper.WriteLine("    File path: " + curFile.Path);
                    }

                    var files = myCompletedTask.ListNodeFiles(recursive: true).ToList();

                    // confirm the files are there
                    Assert.True(files.Any(file => file.Path.Contains("localWords.txt")), "missing file: localWords.txt");
                }
                finally
                {
                    TestUtilities.DeleteJobIfExistsAsync(batchCli, jobId).Wait();
                }
            }
        }
Ejemplo n.º 21
0
        static void Main(string[] args)
        {
            string storageConnectionString =
                $"DefaultEndpointsProtocol=https;AccountName={StorageAccountName};AccountKey={StorageAccountKey}";

            // Retrieve the storage account
            CloudStorageAccount storageAccount = CloudStorageAccount.Parse(storageConnectionString);

            // Create the blob client
            CloudBlobClient blobClient = storageAccount.CreateCloudBlobClient();

            // Upload the input files to blob storage
            const string  inputContainerName = "batchinput";
            List <string> inputFilePaths     = new List <string>
            {
                "taskdata0.txt",
                "taskdata1.txt",
                "taskdata2.txt"
            };
            List <ResourceFile> inputFiles = new List <ResourceFile>();

            foreach (string filePath in inputFilePaths)
            {
                inputFiles.Add(UploadFileToContainer(blobClient, inputContainerName, filePath));
            }

            // Get a SAS Url for the output container
            const string outputContainerName   = "batchoutput";
            string       outputContainerSasUrl = GetOutputContainerSasUrl(blobClient, outputContainerName);

            // Specify a container registry
            ContainerRegistry containerRegistry = new ContainerRegistry(
                registryServer: RegistryServer,
                userName: RegistryUserName,
                password: RegistryPassword);

            // Create container configuration, prefetching Docker images from the container registry
            ContainerConfiguration containerConfig = new ContainerConfiguration()
            {
                ContainerImageNames = ContainerImageNames,
                ContainerRegistries = new List <ContainerRegistry> {
                    containerRegistry
                }
            };

            // Create the virtual machine image reference - make sure to choose an image that supports containers
            ImageReference imageReference = new ImageReference(
                publisher: "MicrosoftWindowsServer",
                offer: "WindowsServer",
                sku: "2016-datacenter-with-containers",
                version: "latest");

            // Create the virtual machine configuration for the pool and set the container configuration
            VirtualMachineConfiguration virtualMachineConfiguration = new VirtualMachineConfiguration(
                imageReference: imageReference,
                nodeAgentSkuId: "batch.node.windows amd64");

            virtualMachineConfiguration.ContainerConfiguration = containerConfig;

            BatchSharedKeyCredentials cred = new BatchSharedKeyCredentials(BatchAccountUrl, BatchAccountName, BatchAccountKey);

            using (BatchClient batchClient = BatchClient.Open(cred))
            {
                Console.WriteLine("Creating pool [{0}]...", PoolId);

                try
                {
                    CloudPool pool = batchClient.PoolOperations.CreatePool(
                        poolId: PoolId,
                        targetDedicatedComputeNodes: PoolNodeCount,
                        virtualMachineSize: PoolVMSize,
                        virtualMachineConfiguration: virtualMachineConfiguration);

                    pool.Commit();
                }
                catch (BatchException be)
                {
                    // Accept the specific error code PoolExists as that is expected if the pool already exists
                    if (be.RequestInformation?.BatchError?.Code == BatchErrorCodeStrings.PoolExists)
                    {
                        Console.WriteLine("The pool {0} already existed when we tried to create it", PoolId);
                    }
                    else
                    {
                        throw; // Any other exception is unexpected
                    }
                }

                Console.WriteLine("Creating job [{0}]...", JobId);
                CloudJob job = null;
                try
                {
                    job    = batchClient.JobOperations.CreateJob();
                    job.Id = JobId;
                    job.PoolInformation = new PoolInformation {
                        PoolId = PoolId
                    };

                    // Add job preparation task to remove existing Docker image
                    Console.WriteLine("Adding job preparation task to job [{0}]...", JobId);
                    string             jobPreparationCmdLine = $"cmd /c docker rmi -f {ContainerImageNames[0]}:latest";
                    JobPreparationTask jobPreparationTask    = new JobPreparationTask(jobPreparationCmdLine)
                    {
                        UserIdentity = new UserIdentity(new AutoUserSpecification(elevationLevel: ElevationLevel.Admin, scope: AutoUserScope.Task))
                    };
                    job.JobPreparationTask = jobPreparationTask;

                    job.Commit();
                }
                catch (BatchException be)
                {
                    // Accept the specific error code JobExists as that is expected if the job already exists
                    if (be.RequestInformation?.BatchError?.Code == BatchErrorCodeStrings.JobExists)
                    {
                        Console.WriteLine("The job {0} already existed when we tried to create it", JobId);
                    }
                    else
                    {
                        throw; // Any other exception is unexpected
                    }
                }

                if (job != null)
                {
                    // Create a collection to hold the tasks that we'll be adding to the job
                    Console.WriteLine("Adding {0} tasks to job [{1}]...", inputFiles.Count, JobId);

                    List <CloudTask> tasks = new List <CloudTask>();

                    // Create each of the tasks to process one of the input files.
                    for (int i = 0; i < inputFiles.Count; i++)
                    {
                        string taskId         = String.Format("Task{0}", i);
                        string inputFilename  = inputFiles[i].FilePath;
                        string outputFileName = string.Format("out{0}", inputFilename);

                        // Override the default entrypoint of the container
                        string taskCommandLine = string.Format("C:\\ReadWriteFile\\ReadWriteFile.exe {0} {1}", inputFilename, outputFileName);

                        // Specify the container the task will run
                        TaskContainerSettings cmdContainerSettings = new TaskContainerSettings(
                            imageName: "nimccollftacr.azurecr.io/batch/readwritefile"
                            );

                        CloudTask task = new CloudTask(taskId, taskCommandLine);
                        task.ContainerSettings = cmdContainerSettings;

                        // Set the resource files and output files for the task
                        task.ResourceFiles = new List <ResourceFile> {
                            inputFiles[i]
                        };
                        task.OutputFiles = new List <OutputFile>
                        {
                            new OutputFile(
                                filePattern: outputFileName,
                                destination: new OutputFileDestination(new OutputFileBlobContainerDestination(containerUrl: outputContainerSasUrl, path: outputFileName)),
                                uploadOptions: new OutputFileUploadOptions(OutputFileUploadCondition.TaskCompletion))
                        };

                        // You must elevate the identity of the task in order to run a container
                        task.UserIdentity = new UserIdentity(new AutoUserSpecification(elevationLevel: ElevationLevel.Admin, scope: AutoUserScope.Task));
                        tasks.Add(task);
                    }

                    // Add all tasks to the job.
                    batchClient.JobOperations.AddTask(JobId, tasks);

                    // Monitor task success/failure, specifying a maximum amount of time to wait for the tasks to complete.
                    TimeSpan timeout = TimeSpan.FromMinutes(30);
                    Console.WriteLine("Monitoring all tasks for 'Completed' state, timeout in {0}...", timeout);

                    IEnumerable <CloudTask> addedTasks = batchClient.JobOperations.ListTasks(JobId);

                    batchClient.Utilities.CreateTaskStateMonitor().WaitAll(addedTasks, TaskState.Completed, timeout);

                    Console.WriteLine("All tasks reached state Completed.");

                    // Print task output
                    Console.WriteLine();
                    Console.WriteLine("Printing task output...");

                    IEnumerable <CloudTask> completedtasks = batchClient.JobOperations.ListTasks(JobId);

                    foreach (CloudTask task in completedtasks)
                    {
                        string nodeId = String.Format(task.ComputeNodeInformation.ComputeNodeId);
                        Console.WriteLine("Task: {0}", task.Id);
                        Console.WriteLine("Node: {0}", nodeId);
                        Console.WriteLine("Standard out:");
                        Console.WriteLine(task.GetNodeFile(Constants.StandardOutFileName).ReadAsString());
                    }
                }

                // Clean up Batch resources (if the user so chooses)
                Console.WriteLine();
                Console.Write("Delete job? [yes] no: ");
                string response = Console.ReadLine().ToLower();
                if (response != "n" && response != "no")
                {
                    batchClient.JobOperations.DeleteJob(JobId);
                }

                Console.Write("Delete pool? [yes] no: ");
                response = Console.ReadLine().ToLower();
                if (response != "n" && response != "no")
                {
                    batchClient.PoolOperations.DeletePool(PoolId);
                }
            }
        }
Ejemplo n.º 22
0
        public void TestGetNodeFileByTask()
        {
            Action test = () =>
            {
                using (BatchClient batchCli = TestUtilities.OpenBatchClientAsync(TestUtilities.GetCredentialsFromEnvironment()).Result)
                {
                    JobOperations jobOperations = batchCli.JobOperations;

                    string jobId = Constants.DefaultConveniencePrefix + TestUtilities.GetMyName() + "-" + nameof(TestGetNodeFileByTask);
                    try
                    {
                        //
                        // Create the job
                        //
                        CloudJob job = jobOperations.CreateJob(jobId, new PoolInformation());
                        job.PoolInformation = new PoolInformation()
                        {
                            PoolId = this.poolFixture.PoolId
                        };

                        this.testOutputHelper.WriteLine("Initial job schedule commit()");

                        job.Commit();

                        //
                        // Wait for the job
                        //
                        this.testOutputHelper.WriteLine("Waiting for job");
                        CloudJob boundJob = jobOperations.GetJob(jobId);

                        //
                        // Add task to the job
                        //
                        const string taskId      = "T1";
                        const string taskMessage = "This is a test";

                        this.testOutputHelper.WriteLine("Adding task: {0}", taskId);
                        CloudTask task = new CloudTask(taskId, string.Format("cmd /c echo {0}", taskMessage));
                        boundJob.AddTask(task);

                        //
                        // Wait for the task to complete
                        //
                        this.testOutputHelper.WriteLine("Waiting for the task to complete");
                        Utilities        utilities        = batchCli.Utilities;
                        TaskStateMonitor taskStateMonitor = utilities.CreateTaskStateMonitor();

                        //Wait for the task state to be running
                        taskStateMonitor.WaitAll(
                            jobOperations.ListTasks(jobId),
                            TaskState.Completed,
                            TimeSpan.FromSeconds(30));

                        //Download the data
                        this.testOutputHelper.WriteLine("Downloading the stdout for the file");
                        NodeFile file = jobOperations.GetNodeFile(jobId, taskId, Constants.StandardOutFileName);
                        string   data = file.ReadAsString();
                        this.testOutputHelper.WriteLine("Data: {0}", data);
                        Assert.Contains(taskMessage, data);

                        // Download the data again using the JobOperations read file content helper
                        data = batchCli.JobOperations.CopyNodeFileContentToString(jobId, taskId, Constants.StandardOutFileName);
                        this.testOutputHelper.WriteLine("Data: {0}", data);
                        Assert.Contains(taskMessage, data);
                    }
                    finally
                    {
                        jobOperations.DeleteJob(jobId);
                    }
                }
            };

            SynchronizationContextHelper.RunTest(test, TestTimeout);
        }
Ejemplo n.º 23
0
        public void Bug1480491NodeFileFileProperties()
        {
            Action test = () =>
            {
                using (BatchClient batchCli = TestUtilities.OpenBatchClientAsync(TestUtilities.GetCredentialsFromEnvironment()).Result)
                {
                    string jobId = "Bug1480491Job-" + TestUtilities.GetMyName();

                    try
                    {
                        const string taskId = "hiWorld";

                        //
                        // Create the job
                        //
                        CloudJob unboundJob = batchCli.JobOperations.CreateJob(jobId, new PoolInformation());
                        unboundJob.PoolInformation.PoolId = this.poolFixture.PoolId;
                        unboundJob.Commit();

                        CloudJob  boundJob = batchCli.JobOperations.GetJob(jobId);
                        CloudTask myTask   = new CloudTask(taskId, "cmd /c echo hello world");

                        boundJob.AddTask(myTask);

                        this.testOutputHelper.WriteLine("Initial job commit()");

                        //
                        // Wait for task to go to completion
                        //
                        Utilities        utilities        = batchCli.Utilities;
                        TaskStateMonitor taskStateMonitor = utilities.CreateTaskStateMonitor();

                        taskStateMonitor.WaitAll(
                            boundJob.ListTasks(),
                            Microsoft.Azure.Batch.Common.TaskState.Completed,
                            TimeSpan.FromMinutes(3));

                        const int expectedFileSize = 13; //Magic number based on output generated by the task

                        //
                        // NodeFile by task
                        //
                        NodeFile file = batchCli.JobOperations.GetNodeFile(jobId, taskId, Constants.StandardOutFileName);

                        this.testOutputHelper.WriteLine("File {0} has content length: {1}", Constants.StandardOutFileName, file.Properties.ContentLength);
                        this.testOutputHelper.WriteLine("File {0} has content type: {1}", Constants.StandardOutFileName, file.Properties.ContentType);

                        this.testOutputHelper.WriteLine("File {0} has creation time: {1}", Constants.StandardOutFileName, file.Properties.CreationTime);
                        this.testOutputHelper.WriteLine("File {0} has last modified time: {1}", Constants.StandardOutFileName, file.Properties.LastModified);

                        Assert.Equal(expectedFileSize, file.Properties.ContentLength);
                        Assert.Equal("text/plain", file.Properties.ContentType);

                        //
                        // NodeFile by node
                        //
                        CloudTask boundTask     = boundJob.GetTask(taskId);
                        string    computeNodeId = boundTask.ComputeNodeInformation.AffinityId.Split(':')[1];

                        ComputeNode computeNode = batchCli.PoolOperations.GetComputeNode(this.poolFixture.PoolId, computeNodeId);

                        this.testOutputHelper.WriteLine("Task ran on compute node: {0}", computeNodeId);

                        List <NodeFile> files = computeNode.ListNodeFiles(recursive: true).ToList();
                        foreach (NodeFile nodeFile in files)
                        {
                            this.testOutputHelper.WriteLine("Found file: {0}", nodeFile.Path);
                        }

                        string filePathToGet = string.Format("workitems/{0}/{1}/{2}/{3}", jobId, "job-1", taskId, Constants.StandardOutFileName);
                        file = computeNode.GetNodeFile(filePathToGet);

                        this.testOutputHelper.WriteLine("File {0} has content length: {1}", filePathToGet, file.Properties.ContentLength);
                        this.testOutputHelper.WriteLine("File {0} has content type: {1}", filePathToGet, file.Properties.ContentType);

                        this.testOutputHelper.WriteLine("File {0} has creation time: {1}", filePathToGet, file.Properties.CreationTime);
                        this.testOutputHelper.WriteLine("File {0} has last modified time: {1}", filePathToGet, file.Properties.LastModified);

                        Assert.Equal(expectedFileSize, file.Properties.ContentLength);
                        Assert.Equal("text/plain", file.Properties.ContentType);
                    }
                    finally
                    {
                        batchCli.JobOperations.DeleteJob(jobId);
                    }
                }
            };

            SynchronizationContextHelper.RunTest(test, TestTimeout);
        }
Ejemplo n.º 24
0
        public void Bug1480489NodeFileMissingIsDirectory()
        {
            Action test = () =>
            {
                using (BatchClient batchCli = TestUtilities.OpenBatchClientAsync(TestUtilities.GetCredentialsFromEnvironment()).Result)
                {
                    string jobId = "Bug1480489Job-" + TestUtilities.GetMyName();

                    try
                    {
                        // here we show how to use an unbound Job + Commit() to run millions of Tasks
                        CloudJob unboundJob = batchCli.JobOperations.CreateJob(jobId, new PoolInformation()
                        {
                            PoolId = this.poolFixture.PoolId
                        });
                        unboundJob.Commit();

                        // Open the new Job as bound.
                        CloudJob boundJob = batchCli.JobOperations.GetJob(jobId);

                        CloudTask myTask = new CloudTask(id: "Bug1480489Task", commandline: @"md Bug1480489Directory");

                        // add the task to the job
                        boundJob.AddTask(myTask);

                        // wait for the task to complete
                        Utilities        utilities        = batchCli.Utilities;
                        TaskStateMonitor taskStateMonitor = utilities.CreateTaskStateMonitor();

                        taskStateMonitor.WaitAll(
                            boundJob.ListTasks(),
                            Microsoft.Azure.Batch.Common.TaskState.Completed,
                            TimeSpan.FromMinutes(3));

                        CloudTask myCompletedTask = new List <CloudTask>(boundJob.ListTasks(null))[0];

                        string stdOut = myCompletedTask.GetNodeFile(Constants.StandardOutFileName).ReadAsString();
                        string stdErr = myCompletedTask.GetNodeFile(Constants.StandardErrorFileName).ReadAsString();

                        this.testOutputHelper.WriteLine("TaskId: " + myCompletedTask.Id);
                        this.testOutputHelper.WriteLine("StdOut: ");
                        this.testOutputHelper.WriteLine(stdOut);

                        this.testOutputHelper.WriteLine("StdErr: ");
                        this.testOutputHelper.WriteLine(stdErr);

                        this.testOutputHelper.WriteLine("Task Files:");

                        bool foundAtLeastOneDir = false;

                        foreach (NodeFile curFile in myCompletedTask.ListNodeFiles())
                        {
                            this.testOutputHelper.WriteLine("    Filepath: " + curFile.Path);
                            this.testOutputHelper.WriteLine("       IsDirectory: " + curFile.IsDirectory.ToString());

                            // turns out wd is created for each task so use it as sentinal
                            if (curFile.Path.Equals("wd") && curFile.IsDirectory.HasValue && curFile.IsDirectory.Value)
                            {
                                foundAtLeastOneDir = true;
                            }
                        }

                        Assert.True(foundAtLeastOneDir);
                    }
                    finally
                    {
                        TestUtilities.DeleteJobIfExistsAsync(batchCli, jobId).Wait();
                    }
                }
            };

            SynchronizationContextHelper.RunTest(test, TestTimeout);
        }
Ejemplo n.º 25
0
        public void TestNode_GetListDeleteFiles()
        {
            Action test = () =>
            {
                using (BatchClient batchCli = TestUtilities.OpenBatchClientAsync(TestUtilities.GetCredentialsFromEnvironment()).Result)
                {
                    string jobId = "TestNodeGetListDeleteFiles-" + TestUtilities.GetMyName();

                    try
                    {
                        const string taskId = "hiWorld";

                        const string directoryCreationTaskId1 = "dirTask1";
                        const string directoryCreationTaskId2 = "dirTask2";

                        const string directoryNameOne = "Foo";
                        const string directoryNameTwo = "Bar";

                        //
                        // Create the job
                        //
                        CloudJob unboundJob = batchCli.JobOperations.CreateJob(jobId, new PoolInformation());
                        unboundJob.PoolInformation.PoolId = this.poolFixture.PoolId;
                        unboundJob.Commit();

                        CloudJob  boundJob = batchCli.JobOperations.GetJob(jobId);
                        CloudTask myTask   = new CloudTask(taskId, "cmd /c echo hello world");
                        CloudTask directoryCreationTask1 = new CloudTask(directoryCreationTaskId1, string.Format("cmd /c mkdir {0} && echo test > {0}/testfile.txt", directoryNameOne));
                        CloudTask directoryCreationTask2 = new CloudTask(directoryCreationTaskId2, string.Format("cmd /c mkdir {0} && echo test > {0}/testfile.txt", directoryNameTwo));

                        boundJob.AddTask(myTask);
                        boundJob.AddTask(directoryCreationTask1);
                        boundJob.AddTask(directoryCreationTask2);

                        this.testOutputHelper.WriteLine("Initial job commit()");

                        //
                        // Wait for task to go to completion
                        //
                        Utilities        utilities        = batchCli.Utilities;
                        TaskStateMonitor taskStateMonitor = utilities.CreateTaskStateMonitor();

                        taskStateMonitor.WaitAll(
                            boundJob.ListTasks(),
                            Microsoft.Azure.Batch.Common.TaskState.Completed,
                            TimeSpan.FromMinutes(3));

                        CloudTask boundTask = boundJob.GetTask(taskId);
                        //Since the compute node name comes back as "Node:<computeNodeId>" we need to split on : to get the actual compute node name
                        string computeNodeId = boundTask.ComputeNodeInformation.AffinityId.Split(':')[1];

                        ComputeNode computeNode = batchCli.PoolOperations.GetComputeNode(this.poolFixture.PoolId, computeNodeId);

                        this.testOutputHelper.WriteLine("Task ran on compute node: {0}", computeNodeId);

                        //Ensure that ListFiles done without a recursive option, or with recursive false return the same values
                        {
                            List <NodeFile> filesByComputeNodeRecursiveOmitted = batchCli.PoolOperations.ListNodeFiles(
                                this.poolFixture.PoolId,
                                computeNodeId).ToList();

                            List <NodeFile> filesByComputeNodeRecursiveFalse = batchCli.PoolOperations.ListNodeFiles(
                                this.poolFixture.PoolId,
                                computeNodeId,
                                recursive: false).ToList();

                            AssertFileListsMatch(filesByComputeNodeRecursiveOmitted, filesByComputeNodeRecursiveFalse);
                        }

                        {
                            List <NodeFile> filesByTaskRecursiveOmitted = batchCli.JobOperations.ListNodeFiles(
                                jobId,
                                taskId).ToList();

                            List <NodeFile> filesByTaskRecursiveFalse = batchCli.JobOperations.ListNodeFiles(
                                jobId,
                                taskId,
                                recursive: false).ToList();

                            AssertFileListsMatch(filesByTaskRecursiveOmitted, filesByTaskRecursiveFalse);
                        }

                        //
                        // List all node files from operations -- recursive true
                        //
                        //TODO: Detail level?
                        List <NodeFile> fileListFromComputeNodeOperations = batchCli.PoolOperations.ListNodeFiles(this.poolFixture.PoolId, computeNodeId, recursive: true).ToList();

                        foreach (NodeFile f in fileListFromComputeNodeOperations)
                        {
                            this.testOutputHelper.WriteLine("Found file: {0}", f.Path);
                        }
                        //Check to make sure the expected folder named "Shared" exists
                        Assert.Contains("shared", fileListFromComputeNodeOperations.Select(f => f.Path));

                        //
                        // List all node files from the compute node -- recursive true
                        //
                        List <NodeFile> fileListFromComputeNode = computeNode.ListNodeFiles(recursive: true).ToList();
                        foreach (NodeFile f in fileListFromComputeNodeOperations)
                        {
                            this.testOutputHelper.WriteLine("Found file: {0}", f.Path);
                        }
                        //Check to make sure the expected folder named "Shared" exists
                        Assert.Contains("shared", fileListFromComputeNode.Select(f => f.Path));

                        //
                        // Get file from operations
                        //
                        string filePathToGet = fileListFromComputeNode.First(f => !f.IsDirectory.Value && f.Properties.ContentLength > 0).Path;
                        this.testOutputHelper.WriteLine("Getting file: {0}", filePathToGet);
                        NodeFile computeNodeFileFromManager = batchCli.PoolOperations.GetNodeFile(this.poolFixture.PoolId, computeNodeId, filePathToGet);
                        this.testOutputHelper.WriteLine("Successfully retrieved file: {0}", filePathToGet);
                        this.testOutputHelper.WriteLine("---- File data: ----");
                        var computeNodeFileContentFromManager = computeNodeFileFromManager.ReadAsString();
                        this.testOutputHelper.WriteLine(computeNodeFileContentFromManager);
                        Assert.NotEmpty(computeNodeFileContentFromManager);

                        //
                        // Get file directly from operations (bypassing the properties call)
                        //
                        var computeNodeFileContentDirect = batchCli.PoolOperations.CopyNodeFileContentToString(this.poolFixture.PoolId, computeNodeId, filePathToGet);
                        this.testOutputHelper.WriteLine("---- File data: ----");
                        this.testOutputHelper.WriteLine(computeNodeFileContentDirect);
                        Assert.NotEmpty(computeNodeFileContentDirect);

                        //
                        // Get file from compute node
                        //
                        this.testOutputHelper.WriteLine("Getting file: {0}", filePathToGet);
                        NodeFile fileFromComputeNode = computeNode.GetNodeFile(filePathToGet);
                        this.testOutputHelper.WriteLine("Successfully retrieved file: {0}", filePathToGet);
                        this.testOutputHelper.WriteLine("---- File data: ----");
                        var computeNodeFileContentFromNode = fileFromComputeNode.ReadAsString();
                        this.testOutputHelper.WriteLine(computeNodeFileContentFromNode);
                        Assert.NotEmpty(computeNodeFileContentFromNode);

                        //
                        // Get file from compute node (bypassing the properties call)
                        //
                        computeNodeFileContentDirect = computeNode.CopyNodeFileContentToString(filePathToGet);
                        this.testOutputHelper.WriteLine("---- File data: ----");
                        this.testOutputHelper.WriteLine(computeNodeFileContentDirect);
                        Assert.NotEmpty(computeNodeFileContentDirect);

                        //
                        // NodeFile delete
                        //
                        string   filePath = Path.Combine(@"workitems", jobId, "job-1", taskId, Constants.StandardOutFileName);
                        NodeFile nodeFile = batchCli.PoolOperations.GetNodeFile(this.poolFixture.PoolId, computeNodeId, filePath);

                        nodeFile.Delete();

                        //Ensure delete succeeded

                        TestUtilities.AssertThrows <BatchException>(() => nodeFile.Refresh());

                        //Delete directory

                        NodeFile directory = batchCli.PoolOperations.ListNodeFiles(this.poolFixture.PoolId, computeNodeId, recursive: true).First(item => item.Path.Contains(directoryNameOne));
                        Assert.True(directory.IsDirectory);
                        TestUtilities.AssertThrows <BatchException>(() => directory.Delete(recursive: false));
                        directory.Delete(recursive: true);

                        Assert.Null(batchCli.PoolOperations.ListNodeFiles(this.poolFixture.PoolId, computeNodeId, recursive: true).FirstOrDefault(item => item.Path.Contains(directoryNameOne)));

                        //
                        // PoolManager delete node file
                        //
                        filePath = Path.Combine(@"workitems", jobId, "job-1", taskId, Constants.StandardErrorFileName);

                        NodeFile file = batchCli.JobOperations.GetNodeFile(jobId, taskId, Constants.StandardErrorFileName);
                        batchCli.PoolOperations.DeleteNodeFile(this.poolFixture.PoolId, computeNodeId, filePath);

                        //Ensure delete succeeded
                        TestUtilities.AssertThrows <BatchException>(() => batchCli.JobOperations.GetNodeFile(jobId, taskId, Constants.StandardErrorFileName));

                        //Delete directory
                        directory = batchCli.PoolOperations.ListNodeFiles(this.poolFixture.PoolId, computeNodeId, recursive: true).First(item => item.Path.Contains(directoryNameTwo));
                        Assert.True(directory.IsDirectory);
                        TestUtilities.AssertThrows <BatchException>(() => batchCli.PoolOperations.DeleteNodeFile(this.poolFixture.PoolId, computeNodeId, directory.Path, recursive: false));
                        batchCli.PoolOperations.DeleteNodeFile(this.poolFixture.PoolId, computeNodeId, directory.Path, recursive: true);

                        Assert.Null(batchCli.PoolOperations.ListNodeFiles(this.poolFixture.PoolId, computeNodeId, recursive: true).FirstOrDefault(item => item.Path.Contains(directoryNameTwo)));
                    }
                    finally
                    {
                        batchCli.JobOperations.DeleteJob(jobId);
                    }
                }
            };

            SynchronizationContextHelper.RunTest(test, TestTimeout);
        }
Ejemplo n.º 26
0
        public void Bug230385SupportDeleteNodeFileByTask()
        {
            Action test = () =>
            {
                using (BatchClient batchCli = TestUtilities.OpenBatchClientAsync(TestUtilities.GetCredentialsFromEnvironment()).Result)
                {
                    string jobId = "Bug230285Job-" + TestUtilities.GetMyName();

                    try
                    {
                        const string taskId = "hiWorld";
                        const string directoryCreationTaskId1 = "dirTask1";
                        const string directoryCreationTaskId2 = "dirTask2";

                        const string directoryNameOne = "Foo";
                        const string directoryNameTwo = "Bar";

                        const string directory2PathOnNode = "wd/" + directoryNameTwo;

                        //
                        // Create the job
                        //
                        CloudJob unboundJob = batchCli.JobOperations.CreateJob(jobId, new PoolInformation());
                        unboundJob.PoolInformation.PoolId = this.poolFixture.PoolId;
                        unboundJob.Commit();

                        CloudJob  boundJob = batchCli.JobOperations.GetJob(jobId);
                        CloudTask myTask   = new CloudTask(taskId, "cmd /c echo hello world");
                        CloudTask directoryCreationTask1 = new CloudTask(directoryCreationTaskId1, string.Format("cmd /c mkdir {0} && echo test > {0}/testfile.txt", directoryNameOne));
                        CloudTask directoryCreationTask2 = new CloudTask(directoryCreationTaskId2, string.Format("cmd /c mkdir {0} && echo test > {0}/testfile.txt", directoryNameTwo));

                        boundJob.AddTask(myTask);
                        boundJob.AddTask(directoryCreationTask1);
                        boundJob.AddTask(directoryCreationTask2);

                        this.testOutputHelper.WriteLine("Initial job commit()");

                        //
                        // Wait for task to go to completion
                        //
                        Utilities        utilities        = batchCli.Utilities;
                        TaskStateMonitor taskStateMonitor = utilities.CreateTaskStateMonitor();

                        taskStateMonitor.WaitAll(
                            boundJob.ListTasks(),
                            Microsoft.Azure.Batch.Common.TaskState.Completed,
                            TimeSpan.FromMinutes(3));

                        //
                        // NodeFile delete
                        //

                        //Delete single file
                        NodeFile file = batchCli.JobOperations.GetNodeFile(jobId, taskId, Constants.StandardOutFileName);
                        file.Delete();

                        //Ensure delete succeeded
                        TestUtilities.AssertThrows <BatchException>(() => batchCli.JobOperations.GetNodeFile(jobId, taskId, Constants.StandardOutFileName));

                        //Delete directory

                        NodeFile directory = batchCli.JobOperations.ListNodeFiles(jobId, directoryCreationTaskId1, recursive: true).First(item => item.Path.Contains(directoryNameOne));
                        Assert.True(directory.IsDirectory);
                        TestUtilities.AssertThrows <BatchException>(() => directory.Delete(recursive: false));
                        directory.Delete(recursive: true);

                        Assert.Null(batchCli.JobOperations.ListNodeFiles(jobId, directoryCreationTaskId1, recursive: true).FirstOrDefault(item => item.Path.Contains(directoryNameOne)));

                        //
                        // JobScheduleOperations delete task file
                        //
                        batchCli.JobOperations.GetNodeFile(jobId, taskId, Constants.StandardErrorFileName);
                        batchCli.JobOperations.DeleteNodeFile(jobId, taskId, Constants.StandardErrorFileName);

                        //Ensure delete succeeded
                        TestUtilities.AssertThrows <BatchException>(() => batchCli.JobOperations.GetNodeFile(jobId, taskId, Constants.StandardErrorFileName));

                        //Delete directory
                        directory = batchCli.JobOperations.ListNodeFiles(jobId, directoryCreationTaskId2, recursive: true).First(item => item.Path.Contains(directoryNameTwo));
                        Assert.True(directory.IsDirectory);
                        TestUtilities.AssertThrows <BatchException>(() => batchCli.JobOperations.DeleteNodeFile(jobId, directoryCreationTaskId2, directory2PathOnNode, recursive: false));
                        batchCli.JobOperations.DeleteNodeFile(jobId, directoryCreationTaskId2, directory2PathOnNode, recursive: true);

                        Assert.Null(batchCli.JobOperations.ListNodeFiles(jobId, directoryCreationTaskId2, recursive: true).FirstOrDefault(item => item.Path.Contains(directoryNameTwo)));
                    }
                    finally
                    {
                        TestUtilities.DeleteJobIfExistsAsync(batchCli, jobId).Wait();
                    }
                }
            };

            SynchronizationContextHelper.RunTest(test, TestTimeout);
        }
Ejemplo n.º 27
0
        public void TestSampleWithFilesAndPool()
        {
            Action test = () =>
            {
                StagingStorageAccount storageCreds = TestUtilities.GetStorageCredentialsFromEnvironment();

                using (BatchClient batchCli = TestUtilities.OpenBatchClientAsync(TestUtilities.GetCredentialsFromEnvironment()).Result)
                {
                    string jobId = "SampleWithFilesJob-" + TestUtilities.GetMyName();


                    try
                    {
                        CloudJob quickJob = batchCli.JobOperations.CreateJob();
                        quickJob.Id = jobId;
                        quickJob.PoolInformation = new PoolInformation()
                        {
                            PoolId = this.poolFixture.PoolId
                        };
                        quickJob.Commit();
                        CloudJob boundJob = batchCli.JobOperations.GetJob(jobId);

                        CloudTask myTask = new CloudTask(id: "CountWordsTask", commandline: @"cmd /c dir /s .. & dir & wc localwords.txt");

                        // first we have local files that we want pushed to the compute node before the commandline is invoked
                        FileToStage wordsDotText = new FileToStage(Resources.LocalWordsDotText, storageCreds);                // use "default" mapping to base name of local file

                        myTask.FilesToStage = new List <IFileStagingProvider>();

                        myTask.FilesToStage.Add(wordsDotText);

                        // add the task to the job
                        var artifacts        = boundJob.AddTask(myTask);
                        var specificArtifact = artifacts[typeof(FileToStage)];
                        SequentialFileStagingArtifact sfsa = specificArtifact as SequentialFileStagingArtifact;

                        Assert.NotNull(sfsa);

                        // add a million more tasks...

                        // test to ensure the task is read only
                        TestUtilities.AssertThrows <InvalidOperationException>(() => myTask.FilesToStage = new List <IFileStagingProvider>());

                        // Open the new Job as bound.
                        CloudPool boundPool = batchCli.PoolOperations.GetPool(boundJob.ExecutionInformation.PoolId);

                        // wait for the task to complete
                        Utilities        utilities        = batchCli.Utilities;
                        TaskStateMonitor taskStateMonitor = utilities.CreateTaskStateMonitor();

                        taskStateMonitor.WaitAll(
                            boundJob.ListTasks(),
                            Microsoft.Azure.Batch.Common.TaskState.Completed,
                            TimeSpan.FromMinutes(10),
                            controlParams: null,
                            additionalBehaviors:
                            new[]
                        {
                            // spam/logging interceptor
                            new Microsoft.Azure.Batch.Protocol.RequestInterceptor((x) =>
                            {
                                this.testOutputHelper.WriteLine("Issuing request type: " + x.GetType().ToString());

                                try
                                {
                                    // print out the compute node states... we are actually waiting on the compute nodes
                                    List <ComputeNode> allComputeNodes = boundPool.ListComputeNodes().ToList();

                                    this.testOutputHelper.WriteLine("    #compute nodes: " + allComputeNodes.Count);

                                    allComputeNodes.ForEach(
                                        (icn) =>
                                    {
                                        this.testOutputHelper.WriteLine("  computeNode.id: " + icn.Id + ", state: " + icn.State);
                                    });
                                }
                                catch (Exception ex)
                                {
                                    // there is a race between the pool-life-job and the end of the job.. and the ListComputeNodes above
                                    Assert.True(false, "SampleWithFilesAndPool probably can ignore this if its pool not found: " + ex.ToString());
                                }
                            })
                        });

                        List <CloudTask> tasks           = boundJob.ListTasks(null).ToList();
                        CloudTask        myCompletedTask = tasks[0];

                        foreach (CloudTask curTask in tasks)
                        {
                            this.testOutputHelper.WriteLine("Task Id: " + curTask.Id + ", state: " + curTask.State);
                        }

                        boundPool.Refresh();

                        this.testOutputHelper.WriteLine("Pool Id: " + boundPool.Id + ", state: " + boundPool.State);

                        string stdOut = myCompletedTask.GetNodeFile(Constants.StandardOutFileName).ReadAsString();
                        string stdErr = myCompletedTask.GetNodeFile(Constants.StandardErrorFileName).ReadAsString();

                        this.testOutputHelper.WriteLine("StdOut: ");
                        this.testOutputHelper.WriteLine(stdOut);

                        this.testOutputHelper.WriteLine("StdErr: ");
                        this.testOutputHelper.WriteLine(stdErr);

                        this.testOutputHelper.WriteLine("Task Files:");

                        foreach (NodeFile curFile in myCompletedTask.ListNodeFiles(recursive: true))
                        {
                            this.testOutputHelper.WriteLine("    Filename: " + curFile.Name);
                        }

                        // confirm the files are there
                        Assert.True(FoundFile("localwords.txt", myCompletedTask.ListNodeFiles(recursive: true)), "mising file: localwords.txt");

                        // test validation of StagingStorageAccount

                        TestUtilities.AssertThrows <ArgumentOutOfRangeException>(() => { new StagingStorageAccount(storageAccount: " ", storageAccountKey: "key", blobEndpoint: "blob"); });
                        TestUtilities.AssertThrows <ArgumentOutOfRangeException>(() => { new StagingStorageAccount(storageAccount: "account", storageAccountKey: " ", blobEndpoint: "blob"); });
                        TestUtilities.AssertThrows <ArgumentOutOfRangeException>(() => { new StagingStorageAccount(storageAccount: "account", storageAccountKey: "key", blobEndpoint: ""); });

                        if (null != sfsa)
                        {
                            // TODO: delete the container!
                        }
                    }
                    finally
                    {
                        TestUtilities.DeleteJobIfExistsAsync(batchCli, jobId).Wait();
                    }
                }
            };

            SynchronizationContextHelper.RunTest(test, TestTimeout);
        }
Ejemplo n.º 28
0
        public void Bug1965363_2384616_Wat7OSVersionFeatures()
        {
            Action test = () =>
            {
                using (BatchClient batchCli = TestUtilities.OpenBatchClientAsync(TestUtilities.GetCredentialsFromEnvironment()).Result)
                {
                    PoolOperations poolOperations = batchCli.PoolOperations;
                    try
                    {
                        this.testOutputHelper.WriteLine("Listing OS Versions:");

                        /* bug 2384616 ListOsVersions hidden for wat 8
                         *
                         *  // test ListOSVersion
                         *  foreach (OSVersion curOSV in poolMgr.ListOSVersions())
                         *  {
                         *      this.testOutputHelper.WriteLine("Label: " + curOSV.Label);
                         *      this.testOutputHelper.WriteLine("    Version: " + curOSV.Version);
                         *      this.testOutputHelper.WriteLine("    Family: " + curOSV.Family);
                         *      this.testOutputHelper.WriteLine("    FamilyLabel: " + curOSV.FamilyLabel);
                         *      this.testOutputHelper.WriteLine("    isDefault: " + curOSV.IsDefault);
                         *      this.testOutputHelper.WriteLine("    IsActive: " + curOSV.IsActive);
                         *
                         *      string expDate;
                         *
                         *      if (curOSV.ExpirationDate.HasValue)
                         *      {
                         *          expDate = curOSV.ExpirationDate.Value.ToString();
                         *      }
                         *      else
                         *      {
                         *          expDate = "<null/novalue>";
                         *      }
                         *
                         *      this.testOutputHelper.WriteLine("    ExpirationDate: " + expDate);
                         *  }
                         *
                         */

                        // create pool tests


                        // forget to set CloudServiceConfiguration on Create, get error
                        {
                            CloudPool noArgs = poolOperations.CreatePool("Bug1965363ButNoOSFamily-" + TestUtilities.GetMyName(), PoolFixture.VMSize, default(CloudServiceConfiguration), targetDedicated: 0);

                            BatchException ex    = TestUtilities.AssertThrows <BatchException>(() => noArgs.Commit());
                            string         exStr = ex.ToString();

                            // we are expecting an exception, assert if the exception is not the correct one.
                            Assert.Contains("cloudServiceConfiguration", exStr);
                        }

                        // create a pool WITH an osFamily
                        {
                            string poolIdHOSF = "Bug1965363HasOSF-" + TestUtilities.GetMyName();
                            try
                            {
                                CloudPool hasOSF = poolOperations.CreatePool(poolIdHOSF, PoolFixture.VMSize, new CloudServiceConfiguration(PoolFixture.OSFamily), targetDedicated: 0);

                                hasOSF.Commit();
                            }
                            finally
                            {
                                poolOperations.DeletePool(poolIdHOSF);
                            }
                        }

                        // TODO: ultimately we will either need to find (via list) a family with more than one version or
                        //       manually update these strings as OS versions are depricated
                        //See here for other OS versions if this test fails: http://azure.microsoft.com/en-us/documentation/articles/cloud-services-guestos-update-matrix/
                        const string familyVersion0 = "*";
                        const string familyVersion1 = "WA-GUEST-OS-4.32_201605-01";

                        // "UpdatePoolOS" tests (ChangeOSVersion in OM)

                        // PoolManager
                        {
                            string poolIdChangeOSV = "Bug1965363ChangeOSVviaMGR-" + TestUtilities.GetMyName();
                            try
                            {
                                CloudPool unboundPool = poolOperations.CreatePool(
                                    poolIdChangeOSV,
                                    PoolFixture.VMSize,
                                    new CloudServiceConfiguration(PoolFixture.OSFamily, familyVersion0), // start with version 0
                                    targetDedicated: 0);

                                unboundPool.Commit();

                                // fetch the bound pool
                                CloudPool boundPool = poolOperations.GetPool(poolIdChangeOSV);

                                Assert.Equal(familyVersion0, boundPool.CloudServiceConfiguration.CurrentOSVersion);

                                // switch to new version
                                poolOperations.ChangeOSVersion(poolIdChangeOSV, familyVersion1);

                                // UpdatePoolOS is has latency???
                                PollForOSVersionChange(boundPool, familyVersion1);

                                // check to make sure the new value is set
                                boundPool.Refresh();

                                Assert.Equal(familyVersion1, boundPool.CloudServiceConfiguration.CurrentOSVersion);
                            }
                            finally
                            {
                                TestUtilities.DeletePoolIfExistsAsync(batchCli, poolIdChangeOSV).Wait();
                            }
                        }

                        // ICloudPool
                        {
                            string poolIdChangeOSV = "Bug1965363ChangeOSVviaPool-" + TestUtilities.GetMyName();
                            try
                            {
                                CloudPool unboundPool = poolOperations.CreatePool(
                                    poolIdChangeOSV,
                                    PoolFixture.VMSize,
                                    new CloudServiceConfiguration(PoolFixture.OSFamily, familyVersion0), // start with version 0
                                    targetDedicated: 0);

                                unboundPool.Commit();

                                // fetch the bound pool
                                CloudPool boundPool = poolOperations.GetPool(poolIdChangeOSV);

                                Assert.Equal(familyVersion0, boundPool.CloudServiceConfiguration.CurrentOSVersion);

                                // switch to new version
                                boundPool.ChangeOSVersion(familyVersion1);

                                // UpdatePoolOS is has latency???
                                PollForOSVersionChange(boundPool, familyVersion1);

                                // check to make sure the new value is set
                                boundPool.Refresh();

                                Assert.Equal(familyVersion1, boundPool.CloudServiceConfiguration.CurrentOSVersion);
                            }
                            finally
                            {
                                TestUtilities.DeletePoolIfExistsAsync(batchCli, poolIdChangeOSV).Wait();
                            }
                        }

                        // autopoolspec tests
                        {
                            string jobId = "Bug1965363WIName-" + TestUtilities.GetMyName();
                            // test not setting osversion
                            try
                            {
                                CloudJob unboundJob       = batchCli.JobOperations.CreateJob(jobId, new PoolInformation());
                                AutoPoolSpecification aps = new AutoPoolSpecification();
                                PoolSpecification     ps  = new PoolSpecification();

                                // test unbound set constraint
                                ps.CloudServiceConfiguration = new CloudServiceConfiguration(PoolFixture.OSFamily);

                                // test unbound get constraint
                                this.testOutputHelper.WriteLine("pus.CloudServiceConfiguration.OSFamily == " + ps.CloudServiceConfiguration.OSFamily);

                                ps.VirtualMachineSize = PoolFixture.VMSize;
                                ps.TargetDedicated    = 0; // trivial size for testing purposes

                                aps.PoolSpecification  = ps;
                                aps.PoolLifetimeOption = PoolLifetimeOption.Job;

                                unboundJob.PoolInformation.AutoPoolSpecification = aps;

                                // commit to test validation
                                unboundJob.Commit();

                                // get bound job
                                CloudJob boundJob = batchCli.JobOperations.GetJob(jobId);

                                // test bound get constraints
                                this.testOutputHelper.WriteLine("    OSFamily == " +
                                                                boundJob.PoolInformation.AutoPoolSpecification.PoolSpecification.CloudServiceConfiguration.OSFamily);

                                string targetOSVersion =
                                    boundJob.PoolInformation.AutoPoolSpecification.PoolSpecification.CloudServiceConfiguration.TargetOSVersion;

                                if (string.IsNullOrEmpty(targetOSVersion))
                                {
                                    targetOSVersion = "<null or empty";
                                }

                                this.testOutputHelper.WriteLine("    TargetOSVersion == " + targetOSVersion);
                            }
                            finally
                            {
                                // cleanup
                                TestUtilities.DeleteJobIfExistsAsync(batchCli, jobId).Wait();
                            }

                            {
                                string jobScheduleId = "Bug1965363WINameSettingAndChanging-" + TestUtilities.GetMyName();
                                // test setting osversion
                                try
                                {
                                    AutoPoolSpecification aps = new AutoPoolSpecification();
                                    PoolSpecification     ps  = new PoolSpecification();
                                    CloudJobSchedule      unboundJobSchedule = batchCli.JobScheduleOperations.CreateJobSchedule(
                                        jobScheduleId,
                                        new Schedule()
                                    {
                                        RecurrenceInterval = TimeSpan.FromDays(7)
                                    },
                                        new JobSpecification(new PoolInformation()
                                    {
                                        AutoPoolSpecification = aps
                                    }));

                                    // test unbound set constraint
                                    ps.CloudServiceConfiguration = new CloudServiceConfiguration(PoolFixture.OSFamily, familyVersion0);

                                    // test unbound get constraint
                                    this.testOutputHelper.WriteLine("pus.CloudServiceConfiguration.OSFamily == " + ps.CloudServiceConfiguration.OSFamily);
                                    this.testOutputHelper.WriteLine("pus.CloudServiceConfiguration.TargetOSVersion == " + ps.CloudServiceConfiguration.TargetOSVersion);

                                    ps.VirtualMachineSize = PoolFixture.VMSize;
                                    ps.TargetDedicated    = 0; // trivial size for testing purposes

                                    aps.PoolSpecification  = ps;
                                    aps.PoolLifetimeOption = PoolLifetimeOption.Job;

                                    unboundJobSchedule.Commit();

                                    // get bound job schedule
                                    CloudJobSchedule boundJobSchedule = batchCli.JobScheduleOperations.GetJobSchedule(jobScheduleId);

                                    // test bound get constraints
                                    this.testOutputHelper.WriteLine("    OSFamily == " + boundJobSchedule.JobSpecification.PoolInformation.AutoPoolSpecification.PoolSpecification.CloudServiceConfiguration.OSFamily);
                                    this.testOutputHelper.WriteLine("    TargetOSVersion == " + boundJobSchedule.JobSpecification.PoolInformation.AutoPoolSpecification.PoolSpecification.CloudServiceConfiguration.TargetOSVersion);

                                    // assert the value is as set above
                                    Assert.Equal(familyVersion0, boundJobSchedule.JobSpecification.PoolInformation.AutoPoolSpecification.PoolSpecification.CloudServiceConfiguration.TargetOSVersion);

                                    // change values
                                    const string altFamily    = "3";
                                    const string altOSVersion = "WA-GUEST-OS-3.39_201605-01";

                                    // change values on the bound PUS
                                    PoolSpecification boundPS = boundJobSchedule.JobSpecification.PoolInformation.AutoPoolSpecification.PoolSpecification;

                                    boundPS.CloudServiceConfiguration = new CloudServiceConfiguration(altFamily, altOSVersion);

                                    // flush changes
                                    boundJobSchedule.Commit();

                                    // confirm changes took
                                    boundJobSchedule.Refresh();

                                    Assert.Equal(altFamily, boundJobSchedule.JobSpecification.PoolInformation.AutoPoolSpecification.PoolSpecification.CloudServiceConfiguration.OSFamily);
                                    Assert.Equal(altOSVersion, boundJobSchedule.JobSpecification.PoolInformation.AutoPoolSpecification.PoolSpecification.CloudServiceConfiguration.TargetOSVersion);
                                }
                                finally
                                {
                                    // cleanup
                                    TestUtilities.DeleteJobScheduleIfExistsAsync(batchCli, jobScheduleId).Wait();
                                }
                            }
                        }
                    }
                    catch (Exception ex)
                    {
                        // special case os version beacuse it is a common failure and requires human intervention/editing
                        // test for expired os version
                        Assert.DoesNotContain("The specified OS Version does not exists", ex.ToString());

                        throw;
                    }
                }
            };

            SynchronizationContextHelper.RunTest(test, TestTimeout);
        }
Ejemplo n.º 29
0
        public void TestOMJobPrepSchedulingError()
        {
            string jobId = "TestOMJobPrepSchedulingError-" + CraftTimeString() + "-" + TestUtilities.GetMyName();

            Action test = () =>
            {
                using (BatchClient client = TestUtilities.OpenBatchClient(TestUtilities.GetCredentialsFromEnvironment()))
                {
                    try
                    {
                        // create job with prep that triggers prep scheduling error
                        {
                            CloudJob unboundJob = client.JobOperations.CreateJob(jobId, new PoolInformation()
                            {
                                PoolId = this.poolFixture.PoolId
                            });
                            // add the jobPrep task to the job
                            {
                                JobPreparationTask prep = new JobPreparationTask("cmd /c JobPrep Task");
                                unboundJob.JobPreparationTask = prep;

                                ResourceFile[] badResFiles = { ResourceFile.FromUrl("https://not.a.domain.invalid/file", "bob.txt") };

                                prep.ResourceFiles = badResFiles;

                                prep.WaitForSuccess = true; // be explicit even though this is the default.  need JP/ to not run
                            }

                            // add the job to the service
                            unboundJob.Commit();
                        }

                        CloudJob boundJob = client.JobOperations.GetJob(jobId);

                        // add a trivial task to force the JP
                        client.JobOperations.AddTask(boundJob.Id, new CloudTask("ForceJobPrep", "cmd /c echo TestOMJobPrepSchedulingError"));

                        // the victim compute node.  pool should have size 1.
                        List <ComputeNode> nodes = client.PoolOperations.ListComputeNodes(this.poolFixture.PoolId).ToList();

                        Assert.Single(nodes);

                        // now we have a job that should be trying to run the JP
                        // poll for the JP to have been run, and it must have a scheduling error
                        bool prepNotCompleted = true;

                        // gotta poll to find out when the jp has been run
                        while (prepNotCompleted)
                        {
                            List <JobPreparationAndReleaseTaskExecutionInformation> jpStatsList =
                                client.JobOperations.ListJobPreparationAndReleaseTaskStatus(jobId).ToList();
                            JobPreparationAndReleaseTaskExecutionInformation jpStatus = jpStatsList.FirstOrDefault();

                            if (jpStatus == null)
                            {
                                Thread.Sleep(2000);
                            }
                            else
                            {
                                if (JobPreparationTaskState.Completed == jpStatus.JobPreparationTaskExecutionInformation.State)
                                {
                                    prepNotCompleted = false; // we see a JP has completed

                                    Assert.NotNull(jpStatus.JobPreparationTaskExecutionInformation.FailureInformation);
                                    Assert.Equal(TaskExecutionResult.Failure, jpStatus.JobPreparationTaskExecutionInformation.Result);

                                    // spew the failure
                                    this.OutputFailureInfo(jpStatus.JobPreparationTaskExecutionInformation.FailureInformation);
                                }

                                this.testOutputHelper.WriteLine("Job Prep is running (waiting for blob dl to timeout)");
                            }
                        }
                    }
                    finally
                    {
                        // cleanup
                        client.JobOperations.DeleteJob(jobId);
                    }
                }
            };

            SynchronizationContextHelper.RunTest(test, TestTimeout);
        }
Ejemplo n.º 30
0
        public void TestOMJobReleaseSchedulingError()
        {
            string jobId = "TestOMJobReleaseSchedulingError-" + CraftTimeString() + "-" + TestUtilities.GetMyName();
            Action test  = () =>
            {
                using (BatchClient client = TestUtilities.OpenBatchClient(TestUtilities.GetCredentialsFromEnvironment()))
                {
                    try
                    {
                        // create job schedule with prep that succeeds and release the triggers scheduling error
                        {
                            PoolInformation poolInfo = new PoolInformation()
                            {
                                PoolId = this.poolFixture.PoolId
                            };
                            CloudJob unboundJob = client.JobOperations.CreateJob(jobId, poolInfo);

                            // add the jobPrep task to the job
                            {
                                JobPreparationTask prep = new JobPreparationTask("cmd /c echo the quick job prep jumped over the...");
                                unboundJob.JobPreparationTask = prep;

                                prep.WaitForSuccess = false; // we don't really care but why not set this
                            }

                            // add a jobRelease task to the job
                            {
                                JobReleaseTask relTask = new JobReleaseTask("cmd /c echo Job Release Task");
                                unboundJob.JobReleaseTask = relTask;

                                ResourceFile[] badResFiles = { ResourceFile.FromUrl("https://not.a.domain.invalid/file", "bob.txt") };

                                relTask.ResourceFiles = badResFiles;

                                relTask.Id = "jobRelease";
                            }

                            // add the job to the service
                            unboundJob.Commit();
                        }

                        // add a trivial task to force the JP
                        client.JobOperations.AddTask(jobId, new CloudTask("ForceJobPrep", "cmd /c echo TestOMJobReleaseSchedulingError"));

                        // wait for the task to complete

                        TaskStateMonitor tsm = client.Utilities.CreateTaskStateMonitor();

                        tsm.WaitAll(
                            client.JobOperations.ListTasks(jobId),
                            TaskState.Completed,
                            TimeSpan.FromMinutes(10),
                            additionalBehaviors:
                            new[]
                        {
                            // spam/logging interceptor
                            new Protocol.RequestInterceptor((x) =>
                            {
                                this.testOutputHelper.WriteLine("Issuing request type: " + x.GetType().ToString());

                                // print out the compute node states... we are actually waiting on the compute nodes
                                List <ComputeNode> allComputeNodes = client.PoolOperations.ListComputeNodes(this.poolFixture.PoolId).ToList();

                                this.testOutputHelper.WriteLine("    #compute nodes: " + allComputeNodes.Count);

                                allComputeNodes.ForEach((icn) =>
                                {
                                    this.testOutputHelper.WriteLine("  computeNode.id: " + icn.Id + ", state: " + icn.State);
                                });
                                this.testOutputHelper.WriteLine("");
                            })
                        }
                            );

                        // ok terminate job to trigger job release
                        client.JobOperations.TerminateJob(jobId, "BUG: Server will throw 500 if I don't provide reason");

                        // the victim compute node.  pool should have size 1.
                        List <ComputeNode> computeNodes = client.PoolOperations.ListComputeNodes(this.poolFixture.PoolId).ToList();

                        Assert.Single(computeNodes);

                        // now we have a job that should be trying to run the JP
                        // poll for the JP to have been run, and it must have a scheduling error
                        bool releaseNotCompleted = true;

                        // gotta poll to find out when the jp has been run
                        while (releaseNotCompleted)
                        {
                            List <JobPreparationAndReleaseTaskExecutionInformation> jrStatusList =
                                client.JobOperations.ListJobPreparationAndReleaseTaskStatus(jobId).ToList();

                            JobPreparationAndReleaseTaskExecutionInformation prepAndReleaseStatus = jrStatusList.FirstOrDefault();

                            if (prepAndReleaseStatus != null && null != prepAndReleaseStatus.JobReleaseTaskExecutionInformation)
                            {
                                if (JobReleaseTaskState.Completed == prepAndReleaseStatus.JobReleaseTaskExecutionInformation.State)
                                {
                                    releaseNotCompleted = false; // we see a JP has been run

                                    // now assert the failure info
                                    Assert.NotNull(prepAndReleaseStatus);
                                    Assert.NotNull(prepAndReleaseStatus.JobReleaseTaskExecutionInformation.FailureInformation);
                                    Assert.Equal(TaskExecutionResult.Failure, prepAndReleaseStatus.JobReleaseTaskExecutionInformation.Result);

                                    // spew the failure info
                                    this.OutputFailureInfo(prepAndReleaseStatus.JobReleaseTaskExecutionInformation.FailureInformation);
                                }
                            }
                            Thread.Sleep(2000);
                            this.testOutputHelper.WriteLine("Job Release tasks still running (waiting for blob dl to timeout).");
                        }
                    }
                    finally
                    {
                        client.JobOperations.DeleteJob(jobId);
                    }
                }
            };

            SynchronizationContextHelper.RunTest(test, LongTestTimeout);
        }
        static void AddTasks(BatchClient client, CloudJob cloudJob, string jobId, IEnumerable<IComputeTask> computeTasks)
        {
            foreach (var computeTask in computeTasks)
            {
                var definition = computeTask.Definition;
                var executable = new ResourceFile($"{definition.StorageUri}/{definition.ExecutableName}", definition.ExecutableName);
                var resources = definition.Resources.Select(resource => new ResourceFile($"{definition.StorageUri}/{resource}", resource));
                var inputs = computeTask.Inputs.Select(input => new ResourceFile($"{definition.StorageUri}/{input}", input));

                var resourceFiles = new List<ResourceFile> { executable };
                resourceFiles.AddRange(resources);
                resourceFiles.AddRange(inputs);

                var task = client.JobOperations.ListTasks(jobId).SingleOrDefault(t => t.Id == computeTask.Id);

                if (task == null)
                {
                    task = new CloudTask(computeTask.Id, computeTask.CommandLine)
                    {
                        ResourceFiles = resourceFiles
                    };

                    cloudJob.AddTask(task);
                    cloudJob.Commit();
                    cloudJob.Refresh();
                }
            }

            client.Utilities.CreateTaskStateMonitor().WaitAll(cloudJob.ListTasks(), TaskState.Completed, new TimeSpan(0, 30, 0));
        }