Beispiel #1
0
        public AzureHDInsightJob WaitJob()
        {
            _clusterName = GetClusterConnection(ResourceGroupName, ClusterName);

            TimeSpan?duration = null;

            if (TimeoutInSeconds > 0)
            {
                duration = TimeSpan.FromSeconds(TimeoutInSeconds);
            }

            TimeSpan?waitInterval = null;

            if (WaitIntervalInSeconds > 0)
            {
                waitInterval = TimeSpan.FromSeconds(WaitIntervalInSeconds);
            }

            JobDetailRootJsonObject jobDetail = null;

            try
            {
                jobDetail = HDInsightJobClient.WaitForJobCompletion(JobId, duration, waitInterval).JobDetail;
            }
            catch (TimeoutException)
            {
                var exceptionMessage = string.Format(CultureInfo.InvariantCulture, "HDInsight job with ID {0} has not completed in {1} seconds. Increase the value of -TimeoutInSeconds or check the job runtime.", JobId, TimeoutInSeconds);
                throw new CloudException(exceptionMessage);
            }

            var jobDetails = new AzureHDInsightJob(jobDetail, HDInsightJobClient.ClusterName);

            return(jobDetails);
        }
Beispiel #2
0
        public void CanGetTaskLogsForCompletedJob()
        {
            ClusterDetails cluster = CmdletScenariosTestCaseBase.GetHttpAccessEnabledCluster();
            IGetAzureHDInsightJobCommand getJobsCommand = ServiceLocator.Instance.Locate <IAzureHDInsightCommandFactory>().CreateGetJobs();

            getJobsCommand.Credential = GetPSCredential(cluster.HttpUserName, cluster.HttpPassword);
            getJobsCommand.Cluster    = cluster.ConnectionUrl;
            getJobsCommand.EndProcessing();

            AzureHDInsightJob jobWithStatusDirectory = getJobsCommand.Output.First(j => !string.IsNullOrEmpty(j.StatusDirectory));
            string            logDirectoryPath       = Path.Combine(System.Environment.CurrentDirectory, Guid.NewGuid().ToString());
            IGetAzureHDInsightJobOutputCommand getJobOutputCommand =
                ServiceLocator.Instance.Locate <IAzureHDInsightCommandFactory>().CreateGetJobOutput();

            getJobOutputCommand.CurrentSubscription = GetCurrentSubscription();
            getJobOutputCommand.Cluster             = cluster.Name;
            getJobOutputCommand.OutputType          = JobOutputType.TaskLogs;
            getJobOutputCommand.TaskLogsDirectory   = logDirectoryPath;
            getJobOutputCommand.JobId = jobWithStatusDirectory.JobId;
            getJobOutputCommand.EndProcessing();

            IEnumerable <string> logFiles = Directory.EnumerateFiles(logDirectoryPath);

            Assert.IsTrue(logFiles.Any());
        }
        public override void ExecuteCmdlet()
        {
            if (ResourceGroupName == null)
            {
                ResourceGroupName = GetResourceGroupByAccountName(ClusterName);
            }

            _clusterName = GetClusterConnection(ResourceGroupName, ClusterName);

            if (NumOfJobs > 0)
            {
                var jobs = HDInsightJobClient.ListJobsAfterJobId(JobId, NumOfJobs).Select(job => new AzureHDInsightJob(job.Detail, HDInsightJobClient.ClusterName));
                WriteObject(jobs, true);
            }
            else if (JobId != null)
            {
                var job        = HDInsightJobClient.GetJob(JobId);
                var jobDetails = new AzureHDInsightJob(job.JobDetail, HDInsightJobClient.ClusterName);
                WriteObject(jobDetails);
            }
            else
            {
                var jobs = HDInsightJobClient.ListJobs().Select(job => job.Id);
                WriteObject(jobs, true);
            }
        }
Beispiel #4
0
        public void CanListJobsForAValidClusterWithJobId()
        {
            ClusterDetails cluster       = CmdletScenariosTestCaseBase.GetHttpAccessEnabledCluster();
            PSCredential   psCredentials = GetPSCredential(cluster.HttpUserName, cluster.HttpPassword);
            IGetAzureHDInsightJobCommand getJobsCommand = ServiceLocator.Instance.Locate <IAzureHDInsightCommandFactory>().CreateGetJobs();

            getJobsCommand.Credential = psCredentials;
            getJobsCommand.Cluster    = cluster.ConnectionUrl;
            getJobsCommand.EndProcessing();

            if (!getJobsCommand.Output.Any())
            {
                return;
            }

            AzureHDInsightJob jobDetail = getJobsCommand.Output.First();

            IGetAzureHDInsightJobCommand getJobWithIdCommand = ServiceLocator.Instance.Locate <IAzureHDInsightCommandFactory>().CreateGetJobs();

            getJobWithIdCommand.Credential = psCredentials;
            getJobWithIdCommand.Cluster    = cluster.ConnectionUrl;
            getJobWithIdCommand.JobId      = jobDetail.JobId;
            getJobWithIdCommand.EndProcessing();

            Assert.AreEqual(1, getJobWithIdCommand.Output.Count, "Should have only one jobDetails when called with jobId.");
            Assert.AreEqual(jobDetail.JobId, getJobsCommand.Output.First().JobId, "Should get jobDetails with the same jobId as the one requested.");
        }
Beispiel #5
0
        private static void TestJobLifecycle(AzureHDInsightJobDefinition mapReduceJobDefinition)
        {
            ClusterDetails    testCluster = CmdletScenariosTestCaseBase.GetHttpAccessEnabledCluster();
            AzureHDInsightJob startedJob  = TestJobStart(mapReduceJobDefinition, testCluster);

            TestJobStop(testCluster, startedJob.JobId);
        }
        public void CanCreateNewHiveJob_WithoutJobName_WithQuery()
        {
            var hiveJobDefinition = new AzureHDInsightHiveJobDefinition {
                Query = "show tables"
            };

            AzureHDInsightJob startedJob = TestJobStart(hiveJobDefinition);

            Assert.AreEqual(startedJob.Name, "Hive: show tables");
        }
        public void CanCreateNewPigJob_WithoutJobName_WithQuery()
        {
            var pigJobDefinition = new AzureHDInsightPigJobDefinition {
                Query = "show tables"
            };

            AzureHDInsightJob startedJob = TestJobStart(pigJobDefinition);

            Assert.AreEqual(string.Empty, startedJob.Name);
        }
        public void CanCreateNewPigJob_WithoutJobName_WithFile()
        {
            var pigJobDefinition = new AzureHDInsightPigJobDefinition
            {
                File = TestConstants.WabsProtocolSchemeName + "container@hostname/Container1/myqueryfile.hql"
            };

            AzureHDInsightJob startedJob = TestJobStart(pigJobDefinition);

            Assert.AreEqual(string.Empty, startedJob.Name);
        }
        public AzureHDInsightJob Execute()
        {
            _clusterName = GetClusterConnection(ResourceGroupName, ClusterName);

            var jobCreationResults = SubmitJob();

            var startedJob = HDInsightJobClient.GetJob(jobCreationResults.JobSubmissionJsonResponse.Id);

            var jobDetail = new AzureHDInsightJob(startedJob.JobDetail, HDInsightJobClient.ClusterName);

            return(jobDetail);
        }
Beispiel #10
0
        public AzureHDInsightJob WaitJob()
        {
            _clusterName = GetClusterConnection(ResourceGroupName, ClusterName);
            var jobDetail = HDInsightJobClient.GetJob(JobId).JobDetail;

            while (!jobDetail.Status.JobComplete)
            {
                jobDetail = HDInsightJobClient.GetJob(JobId).JobDetail;
            }
            var jobDetails = new AzureHDInsightJob(jobDetail, HDInsightJobClient.ClusterName);

            return(jobDetails);
        }
        public void CanAutoGenerateStatusDirectoryForMapReduceJob()
        {
            var mapReduceJobDefinition = new AzureHDInsightMapReduceJobDefinition
            {
                JobName   = "pi estimation jobDetails",
                ClassName = "pi",
                JarFile   = TestConstants.WabsProtocolSchemeName + "container@hostname/examples.jar"
            };

            AzureHDInsightJob startedJob = TestJobStart(mapReduceJobDefinition);

            Assert.IsFalse(string.IsNullOrEmpty(startedJob.StatusDirectory));
        }
        public void CanCreateNewMapReduceJob_WithoutJobName()
        {
            var mapReduceJobDefinition = new AzureHDInsightMapReduceJobDefinition
            {
                ClassName    = "pi",
                JarFile      = TestConstants.WabsProtocolSchemeName + "container@hostname/examples.jar",
                StatusFolder = "/myoutputfolder"
            };

            AzureHDInsightJob startedJob = TestJobStart(mapReduceJobDefinition);

            Assert.AreEqual("pi", startedJob.Name);
        }
        public void UserCanSupplyStatusDirectoryForMapReduceJob()
        {
            var mapReduceJobDefinition = new AzureHDInsightMapReduceJobDefinition
            {
                JobName      = "pi estimation jobDetails",
                ClassName    = "pi",
                JarFile      = TestConstants.WabsProtocolSchemeName + "container@hostname/examples.jar",
                StatusFolder = "/myoutputfolder"
            };

            AzureHDInsightJob startedJob = TestJobStart(mapReduceJobDefinition);

            Assert.AreEqual(startedJob.StatusDirectory, mapReduceJobDefinition.StatusFolder);
        }
Beispiel #14
0
        private static AzureHDInsightJob TestJobStart(AzureHDInsightJobDefinition mapReduceJobDefinition, ClusterDetails testCluster)
        {
            IStartAzureHDInsightJobCommand startJobCommand = ServiceLocator.Instance.Locate <IAzureHDInsightCommandFactory>().CreateStartJob();

            startJobCommand.Cluster       = testCluster.ConnectionUrl;
            startJobCommand.Credential    = GetPSCredential(testCluster.HttpUserName, testCluster.HttpPassword);
            startJobCommand.JobDefinition = mapReduceJobDefinition;
            startJobCommand.EndProcessing();
            AzureHDInsightJob jobCreationResults = startJobCommand.Output.ElementAt(0);

            Assert.IsNotNull(jobCreationResults.JobId, "Should get a non-null jobDetails id");
            Assert.IsNotNull(jobCreationResults.StatusDirectory, "StatusDirectory should be set on jobDetails");
            return(jobCreationResults);
        }
        public void CanCreateNewStreamingMapReduceJob_WithoutJobName()
        {
            var streamingMapReduceJobDefinition = new AzureHDInsightStreamingMapReduceJobDefinition
            {
                Input        = TestConstants.WabsProtocolSchemeName + "container@hostname/input",
                Output       = TestConstants.WabsProtocolSchemeName + "container@hostname/input",
                Mapper       = TestConstants.WabsProtocolSchemeName + "container@hostname/mapper",
                Reducer      = TestConstants.WabsProtocolSchemeName + "container@hostname/combiner",
                StatusFolder = TestConstants.WabsProtocolSchemeName + "container@hostname/someotherlocation"
            };

            AzureHDInsightJob startedJob = TestJobStart(streamingMapReduceJobDefinition);

            Assert.AreEqual(startedJob.Name, "mapper");
        }
        public void CanCreateNewStreamingMapReduceJob_WithoutJobName_FilesRelative()
        {
            var streamingMapReduceJobDefinition = new AzureHDInsightStreamingMapReduceJobDefinition
            {
                Input        = "input",
                Output       = "output",
                Mapper       = "/examples/mapper.exe",
                Reducer      = "/examples/combiner.exe",
                StatusFolder = "/someotherlocation"
            };

            AzureHDInsightJob startedJob = TestJobStart(streamingMapReduceJobDefinition);

            Assert.AreEqual("mapper.exe", startedJob.Name);
        }
Beispiel #17
0
 public override void ExecuteCmdlet()
 {
     _clusterName = GetClusterConnection(ResourceGroupName, ClusterName);
     if (JobId != null)
     {
         var job        = HDInsightJobClient.GetJob(JobId);
         var jobDetails = new AzureHDInsightJob(job.JobDetail, HDInsightJobClient.ClusterName);
         WriteObject(jobDetails);
     }
     else
     {
         var jobs = HDInsightJobClient.ListJobs().Select(job => job.Id);
         WriteObject(jobs, true);
     }
 }
        public virtual void WaitForInvalidJobDoesNotThrow()
        {
            var jobDetails = new JobDetails { JobId = Guid.NewGuid().ToString() };
            var invalidJob = new AzureHDInsightJob(jobDetails, TestCredentials.WellKnownCluster.DnsName);

            // IHadoopClientExtensions.GetPollingInterval = () => 0;
            ClusterDetails cluster = CmdletScenariosTestCaseBase.GetHttpAccessEnabledCluster();
            using (IRunspace runspace = this.GetPowerShellRunspace())
            {
                runspace.NewPipeline()
                                      .AddCommand(CmdletConstants.WaitAzureHDInsightJob)
                        .WithParameter(CmdletConstants.Credential, GetPSCredential(cluster.HttpUserName, cluster.HttpPassword))
                        .WithParameter(CmdletConstants.Job, invalidJob)
                                      .Invoke();
            }
        }
Beispiel #19
0
        private static AzureHDInsightJob TestJobStop(ClusterDetails testCluster, string jobId)
        {
            IStopAzureHDInsightJobCommand stopJobCommand = ServiceLocator.Instance.Locate <IAzureHDInsightCommandFactory>().CreateStopJob();

            stopJobCommand.Cluster    = testCluster.ConnectionUrl;
            stopJobCommand.Credential = GetPSCredential(testCluster.HttpUserName, testCluster.HttpPassword);
            stopJobCommand.JobId      = jobId;
            stopJobCommand.EndProcessing();

            if (stopJobCommand.Output.Count != 0)
            {
                AzureHDInsightJob jobCancellationResults = stopJobCommand.Output.ElementAt(0);
                Assert.IsNotNull(jobCancellationResults.JobId, "Should get a non-null jobDetails id");
                Assert.IsNotNull(jobCancellationResults.StatusDirectory, "StatusDirectory should be set on jobDetails");
                return(jobCancellationResults);
            }
            return(null);
        }
Beispiel #20
0
        protected virtual async Task <AzureHDInsightJob> WaitForCompletion(
            AzureHDInsightJob startedJob, AzureHDInsightClusterConnection currentConnection)
        {
            startedJob.ArgumentNotNull("startedJob");
            currentConnection.ArgumentNotNull("currentConnection");
            this.ValidateNotCanceled();
            var waitJobCommand = ServiceLocator.Instance.Locate <IAzureHDInsightCommandFactory>().CreateWaitJobs();

            waitJobCommand.JobStatusEvent += this.ClientOnJobStatus;
            this.SetClient(waitJobCommand);
            waitJobCommand.Job    = startedJob;
            waitJobCommand.Logger = this.Logger;
            waitJobCommand.WaitTimeoutInSeconds = WaitAnHourInSeconds;
            waitJobCommand.CurrentSubscription  = this.CurrentSubscription;
            this.ValidateNotCanceled();
            await waitJobCommand.ProcessRecord();

            return(waitJobCommand.Output.Last());
        }
Beispiel #21
0
        internal static AzureHDInsightJob RunJobInPowershell(
            IRunspace runspace, AzureHDInsightJobDefinition mapReduceJobDefinition, ClusterDetails cluster)
        {
            IPipelineResult results =
                runspace.NewPipeline()
                .AddCommand(CmdletConstants.StartAzureHDInsightJob)
                .WithParameter(CmdletConstants.Cluster, cluster.ConnectionUrl)
                .WithParameter(CmdletConstants.Credential, GetPSCredential(cluster.HttpUserName, cluster.HttpPassword))
                .WithParameter(CmdletConstants.JobDefinition, mapReduceJobDefinition)
                .Invoke();

            Assert.AreEqual(1, results.Results.Count);
            IEnumerable <AzureHDInsightJob> jobCreationCmdletResults = results.Results.ToEnumerable <AzureHDInsightJob>();
            AzureHDInsightJob jobCreationResults = jobCreationCmdletResults.First();

            Assert.IsNotNull(jobCreationResults.JobId, "Should get a non-null jobDetails id");

            return(jobCreationResults);
        }
        public void ICanCallThe_Get_HDInsightJobsCmdletWithJobId()
        {
            using (IRunspace runspace = this.GetPowerShellRunspace())
            {
                ClusterDetails  testCluster = CmdletScenariosTestCaseBase.GetHttpAccessEnabledCluster();
                IPipelineResult results     =
                    runspace.NewPipeline()
                    .AddCommand(CmdletConstants.GetAzureHDInsightJob)
                    .WithParameter(CmdletConstants.Cluster, testCluster.ConnectionUrl)
                    .WithParameter(CmdletConstants.Credential, GetPSCredential(testCluster.HttpUserName, testCluster.HttpPassword))
                    .Invoke();
                var jobDetail = results.Results.ElementAt(0).ImmediateBaseObject as AzureHDInsightJobBase;
                Assert.IsNotNull(jobDetail);

                AzureHDInsightJob getJobDetailObj = GetJobWithID(runspace, jobDetail.JobId, testCluster);
                Assert.IsNotNull(getJobDetailObj);
                Assert.AreEqual(jobDetail.JobId, getJobDetailObj.JobId);
            }
        }
Beispiel #23
0
 protected override void ProcessRecord()
 {
     if (ResourceGroupName == null)
     {
         ResourceGroupName = GetResourceGroupByAccountName(ClusterName);
     }
     _clusterName = GetClusterConnection(ResourceGroupName, ClusterName);
     if (JobId != null)
     {
         var job        = HDInsightJobClient.GetJob(JobId);
         var jobDetails = new AzureHDInsightJob(job.JobDetail, HDInsightJobClient.ClusterName);
         WriteObject(jobDetails);
     }
     else
     {
         var jobs = HDInsightJobClient.ListJobs().Select(job => job.Id);
         WriteObject(jobs, true);
     }
 }
        public virtual void WaitForInvalidJobDoesNotThrow()
        {
            var jobDetails = new JobDetails {
                JobId = Guid.NewGuid().ToString()
            };
            var invalidJob = new AzureHDInsightJob(jobDetails, TestCredentials.WellKnownCluster.DnsName);

            // IHadoopClientExtensions.GetPollingInterval = () => 0;
            ClusterDetails cluster = CmdletScenariosTestCaseBase.GetHttpAccessEnabledCluster();

            using (IRunspace runspace = this.GetPowerShellRunspace())
            {
                runspace.NewPipeline()
                .AddCommand(CmdletConstants.WaitAzureHDInsightJob)
                .WithParameter(CmdletConstants.Credential, GetPSCredential(cluster.HttpUserName, cluster.HttpPassword))
                .WithParameter(CmdletConstants.Job, invalidJob)
                .Invoke();
            }
        }
Beispiel #25
0
        public virtual void NewPigJob_StartJob_GetJob()
        {
            var pigJobDefinition = new PigJobCreateParameters {
                Query = "load 'passwd' using PigStorage(':'); B = foreach A generate $0 as id;"
            };

            using (IRunspace runspace = this.GetPowerShellRunspace())
            {
                IPipelineResult results =
                    runspace.NewPipeline()
                    .AddCommand(CmdletConstants.NewAzureHDInsightPigJobDefinition)
                    .WithParameter(CmdletConstants.Query, pigJobDefinition.Query)
                    .Invoke();
                Assert.AreEqual(1, results.Results.Count);
                AzureHDInsightPigJobDefinition pigJobFromPowershell = results.Results.ToEnumerable <AzureHDInsightPigJobDefinition>().First();
                AzureHDInsightJob pigJobfromHistory = RunJobAndGetWithId(runspace, pigJobFromPowershell);
                Assert.AreEqual(pigJobfromHistory.Query, pigJobDefinition.Query, "Failed to retrieve query for executed pig jobDetails");
            }
        }
Beispiel #26
0
        private static AzureHDInsightJob RunJobAndGetWithId <TJobType>(IRunspace runspace, TJobType jobDefinition)
            where TJobType : AzureHDInsightJobDefinition
        {
            ClusterDetails  testCluster            = GetHttpAccessEnabledCluster();
            PSCredential    testClusterCredentials = GetPSCredential(testCluster.HttpUserName, testCluster.HttpPassword);
            IPipelineResult jobWithIdResults       =
                runspace.NewPipeline()
                .AddCommand(CmdletConstants.StartAzureHDInsightJob)
                .WithParameter(CmdletConstants.Cluster, testCluster.ConnectionUrl)
                .WithParameter(CmdletConstants.Credential, testClusterCredentials)
                .WithParameter(CmdletConstants.JobDefinition, jobDefinition)
                .AddCommand(CmdletConstants.WaitAzureHDInsightJob)
                .WithParameter(CmdletConstants.Credential, testClusterCredentials)
                .Invoke();

            AzureHDInsightJob jobWithId = jobWithIdResults.Results.ToEnumerable <AzureHDInsightJob>().First();

            Assert.AreEqual(jobWithId.State, JobStatusCode.Completed.ToString(), "jobDetails failed.");
            return(jobWithId);
        }
Beispiel #27
0
        public virtual void ICanCallThe_NewHiveJob_Then_Start_HDInsightJobsCmdlet()
        {
            var hiveJobDefinition = new HiveJobCreateParameters {
                JobName = "show tables jobDetails", Query = "show tables"
            };

            using (IRunspace runspace = this.GetPowerShellRunspace())
            {
                IPipelineResult results =
                    runspace.NewPipeline()
                    .AddCommand(CmdletConstants.NewAzureHDInsightHiveJobDefinition)
                    .WithParameter(CmdletConstants.JobName, hiveJobDefinition.JobName)
                    .WithParameter(CmdletConstants.Query, hiveJobDefinition.Query)
                    .Invoke();
                Assert.AreEqual(1, results.Results.Count);
                AzureHDInsightHiveJobDefinition hiveJobFromPowershell = results.Results.ToEnumerable <AzureHDInsightHiveJobDefinition>().First();
                ClusterDetails    testCluster        = CmdletScenariosTestCaseBase.GetHttpAccessEnabledCluster();
                AzureHDInsightJob jobCreationDetails = RunJobInPowershell(runspace, hiveJobFromPowershell, testCluster);
                AzureHDInsightJob jobHistoryResult   = GetJobsCmdletTests.GetJobWithID(runspace, jobCreationDetails.JobId, testCluster);
            }
        }
Beispiel #28
0
        protected virtual async Task WriteJobSuccess(AzureHDInsightJob completedJob, AzureHDInsightClusterConnection currentConnection)
        {
            completedJob.ArgumentNotNull("completedJob");
            currentConnection.ArgumentNotNull("currentConnection");
            this.WriteOutput("Hive query completed Successfully");
            this.WriteOutput(Environment.NewLine);
            this.ValidateNotCanceled();
            var getJobOutputCommand = ServiceLocator.Instance.Locate <IAzureHDInsightCommandFactory>().CreateGetJobOutput();

            this.SetClient(getJobOutputCommand);
            getJobOutputCommand.JobId  = completedJob.JobId;
            getJobOutputCommand.Logger = this.Logger;
            getJobOutputCommand.CurrentSubscription = this.CurrentSubscription;
            getJobOutputCommand.Cluster             = currentConnection.Cluster.Name;
            this.ValidateNotCanceled();
            await getJobOutputCommand.EndProcessing();

            var    outputStream = getJobOutputCommand.Output.First();
            string content      = new StreamReader(outputStream).ReadToEnd();

            this.WriteOutput(content);
        }
Beispiel #29
0
        internal static AzureHDInsightJob RunJobInPowershell(
            IRunspace runspace, AzureHDInsightJobDefinition mapReduceJobDefinition, ClusterDetails cluster, bool debug, string expectedLogMessage)
        {
            IPipelineResult result = null;

            if (debug)
            {
                var logWriter = new PowershellLogWriter();
                BufferingLogWriterFactory.Instance = logWriter;
                result =
                    runspace.NewPipeline()
                    .AddCommand(CmdletConstants.StartAzureHDInsightJob)
                    .WithParameter(CmdletConstants.Cluster, cluster.ConnectionUrl)
                    .WithParameter(CmdletConstants.Credential, GetPSCredential(cluster.HttpUserName, cluster.HttpPassword))
                    .WithParameter(CmdletConstants.JobDefinition, mapReduceJobDefinition)
                    .WithParameter(CmdletConstants.Debug, null)
                    .Invoke();

                Assert.IsTrue(logWriter.Buffer.Any(message => message.Contains(expectedLogMessage)));
                BufferingLogWriterFactory.Reset();
            }
            else
            {
                result =
                    runspace.NewPipeline()
                    .AddCommand(CmdletConstants.StartAzureHDInsightJob)
                    .WithParameter(CmdletConstants.Cluster, cluster.ConnectionUrl)
                    .WithParameter(CmdletConstants.Credential, GetPSCredential(cluster.HttpUserName, cluster.HttpPassword))
                    .WithParameter(CmdletConstants.JobDefinition, mapReduceJobDefinition)
                    .Invoke();
            }
            Assert.AreEqual(1, result.Results.Count);
            IEnumerable <AzureHDInsightJob> jobCreationCmdletResults = result.Results.ToEnumerable <AzureHDInsightJob>();
            AzureHDInsightJob jobCreationResults = jobCreationCmdletResults.First();

            Assert.IsNotNull(jobCreationResults.JobId, "Should get a non-null jobDetails id");

            return(jobCreationResults);
        }
Beispiel #30
0
        public void CanGetJobOutputForCompletedJob()
        {
            ClusterDetails cluster = CmdletScenariosTestCaseBase.GetHttpAccessEnabledCluster();
            IGetAzureHDInsightJobCommand getJobsCommand = ServiceLocator.Instance.Locate <IAzureHDInsightCommandFactory>().CreateGetJobs();

            getJobsCommand.Credential = GetPSCredential(cluster.HttpUserName, cluster.HttpPassword);
            getJobsCommand.Cluster    = cluster.ConnectionUrl;
            getJobsCommand.EndProcessing();

            AzureHDInsightJob jobWithStatusDirectory = getJobsCommand.Output.First(j => !string.IsNullOrEmpty(j.StatusDirectory));

            IGetAzureHDInsightJobOutputCommand getJobOutputCommand =
                ServiceLocator.Instance.Locate <IAzureHDInsightCommandFactory>().CreateGetJobOutput();

            getJobOutputCommand.CurrentSubscription = GetCurrentSubscription();
            getJobOutputCommand.Cluster             = cluster.Name;
            getJobOutputCommand.JobId = jobWithStatusDirectory.JobId;
            getJobOutputCommand.EndProcessing();

            Stream outputStream = getJobOutputCommand.Output.First();

            Assert.IsTrue(outputStream.Length > 0);
        }
 protected virtual async Task<AzureHDInsightJob> WaitForCompletion(
     AzureHDInsightJob startedJob, AzureHDInsightClusterConnection currentConnection)
 {
     startedJob.ArgumentNotNull("startedJob");
     currentConnection.ArgumentNotNull("currentConnection");
     this.ValidateNotCanceled();
     var waitJobCommand = ServiceLocator.Instance.Locate<IAzureHDInsightCommandFactory>().CreateWaitJobs();
     waitJobCommand.JobStatusEvent += this.ClientOnJobStatus;
     this.SetClient(waitJobCommand);
     waitJobCommand.Job = startedJob;
     waitJobCommand.Logger = this.Logger;
     waitJobCommand.WaitTimeoutInSeconds = WaitAnHourInSeconds;
     waitJobCommand.CurrentSubscription = this.CurrentSubscription;
     this.ValidateNotCanceled();
     await waitJobCommand.ProcessRecord();
     return waitJobCommand.Output.Last();
 }
 protected virtual async Task WriteJobSuccess(AzureHDInsightJob completedJob, AzureHDInsightClusterConnection currentConnection)
 {
     completedJob.ArgumentNotNull("completedJob");
     currentConnection.ArgumentNotNull("currentConnection");
     this.WriteOutput("Hive query completed Successfully");
     this.WriteOutput(Environment.NewLine);
     this.ValidateNotCanceled();
     var getJobOutputCommand = ServiceLocator.Instance.Locate<IAzureHDInsightCommandFactory>().CreateGetJobOutput();
     this.SetClient(getJobOutputCommand);
     getJobOutputCommand.JobId = completedJob.JobId;
     getJobOutputCommand.Logger = this.Logger;
     getJobOutputCommand.CurrentSubscription = this.CurrentSubscription;
     getJobOutputCommand.Cluster = currentConnection.Cluster.Name;
     this.ValidateNotCanceled();
     await getJobOutputCommand.EndProcessing();
     var outputStream = getJobOutputCommand.Output.First();
     string content = new StreamReader(outputStream).ReadToEnd();
     this.WriteOutput(content);
 }