Beispiel #1
0
        public virtual void NewHiveJob_StartJob_GetJob()
        {
            var hiveJobDefinition = new HiveJobCreateParameters {
                JobName = "show tables", Query = "show tables"
            };

            using (IRunspace runspace = this.GetPowerShellRunspace())
            {
                ClusterDetails  testCluster = GetHttpAccessEnabledCluster();
                IPipelineResult results     =
                    runspace.NewPipeline()
                    .AddCommand(CmdletConstants.NewAzureHDInsightHiveJobDefinition)
                    .WithParameter(CmdletConstants.JobName, hiveJobDefinition.JobName)
                    .WithParameter(CmdletConstants.Query, hiveJobDefinition.Query)
                    .AddCommand(CmdletConstants.StartAzureHDInsightJob)
                    .WithParameter(CmdletConstants.Cluster, testCluster.Name)
                    .AddCommand(CmdletConstants.WaitAzureHDInsightJob)
                    .WithParameter(CmdletConstants.WaitTimeoutInSeconds, 10)
                    .AddCommand(CmdletConstants.GetAzureHDInsightJobOutput)
                    .WithParameter(CmdletConstants.Cluster, testCluster.Name)
                    .Invoke();

                Assert.IsNotNull(results.Results.ToEnumerable <string>());
                Assert.IsTrue(results.Results.ToEnumerable <string>().Any(str => str == "hivesampletable"));
            }
        }
Beispiel #2
0
        //Run Hive Job
        public static void DoHiveOperations()
        {
            HiveJobCreateParameters hiveJobDefinition = new HiveJobCreateParameters()
            {
                JobName      = "Show tables job",
                StatusFolder = "/TableListFolder",
                Query        = "show tables;"
            };

            var store = new X509Store();

            store.Open(OpenFlags.ReadOnly);
            var cert      = store.Certificates.Cast <X509Certificate2>().First(item => item.Thumbprint == Constants.thumbprint);
            var creds     = new JobSubmissionCertificateCredential(Constants.subscriptionId, cert, Constants.clusterName);
            var jobClient = JobSubmissionClientFactory.Connect(creds);
            JobCreationResults jobResults = jobClient.CreateHiveJob(hiveJobDefinition);

            Console.Write("Executing Hive Job.");
            // Wait for the job to complete
            WaitForJobCompletion(jobResults, jobClient);
            // Print the Hive job output
            System.IO.Stream       stream = jobClient.GetJobOutput(jobResults.JobId);
            System.IO.StreamReader reader = new System.IO.StreamReader(stream);
            Console.Write("Done..List of Tables are:\n");
            Console.WriteLine(reader.ReadToEnd());
        }
        public void ICanCallThe_New_HDInsightHiveJobDefinitionCmdlet_WithParameters()
        {
            var HiveJobDefinition = new HiveJobCreateParameters {
                JobName = "show tables jobDetails", Query = "show tables"
            };

            HiveJobDefinition.Defines.Add("map.input.tasks", "1000");
            HiveJobDefinition.Defines.Add("map.input.reducers", "1000");

            using (IRunspace runspace = this.GetPowerShellRunspace())
            {
                IPipelineResult results =
                    runspace.NewPipeline()
                    .AddCommand(CmdletConstants.NewAzureHDInsightHiveJobDefinition)
                    .WithParameter(CmdletConstants.JobName, HiveJobDefinition.JobName)
                    .WithParameter(CmdletConstants.Query, HiveJobDefinition.Query)
                    .WithParameter(CmdletConstants.Parameters, HiveJobDefinition.Defines)
                    .Invoke();
                Assert.AreEqual(1, results.Results.Count);
                AzureHDInsightHiveJobDefinition HiveJobFromPowershell = results.Results.ToEnumerable <AzureHDInsightHiveJobDefinition>().First();

                Assert.AreEqual(HiveJobDefinition.JobName, HiveJobFromPowershell.JobName);
                Assert.AreEqual(HiveJobDefinition.Query, HiveJobFromPowershell.Query);

                foreach (var parameter in HiveJobDefinition.Defines)
                {
                    Assert.IsTrue(
                        HiveJobFromPowershell.Defines.Any(arg => string.Equals(parameter.Key, arg.Key) && string.Equals(parameter.Value, arg.Value)),
                        "Unable to find parameter '{0}' in value returned from powershell",
                        parameter.Key);
                }
            }
        }
        public void ICanCallThe_New_HDInsightHiveJobDefinitionCmdlet_WithArguments()
        {
            var HiveJobDefinition = new HiveJobCreateParameters { JobName = "show tables jobDetails", Query = "show tables" };
            HiveJobDefinition.Arguments.Add("arg 1");
            HiveJobDefinition.Arguments.Add("arg 2");

            using (IRunspace runspace = this.GetPowerShellRunspace())
            {
                IPipelineResult results =
                    runspace.NewPipeline()
                            .AddCommand(CmdletConstants.NewAzureHDInsightHiveJobDefinition)
                            .WithParameter(CmdletConstants.JobName, HiveJobDefinition.JobName)
                            .WithParameter(CmdletConstants.Query, HiveJobDefinition.Query)
                            .WithParameter(CmdletConstants.HiveArgs, HiveJobDefinition.Arguments)
                            .Invoke();
                Assert.AreEqual(1, results.Results.Count);
                AzureHDInsightHiveJobDefinition HiveJobFromPowershell = results.Results.ToEnumerable<AzureHDInsightHiveJobDefinition>().First();

                Assert.AreEqual(HiveJobDefinition.JobName, HiveJobFromPowershell.JobName);
                Assert.AreEqual(HiveJobDefinition.Query, HiveJobFromPowershell.Query);

                foreach (string args in HiveJobDefinition.Arguments)
                {
                    Assert.IsTrue(
                        HiveJobFromPowershell.Arguments.Any(arg => string.Equals(args, arg)),
                        "Unable to find argument '{0}' in value returned from powershell",
                        args);
                }
            }
        }
        public virtual void WaitForJobWithId()
        {
            var hiveJobDefinition = new HiveJobCreateParameters()
            {
                JobName = "show tables jobDetails",
                Query = "show tables"
            };

            var cluster = CmdletScenariosTestCaseBase.GetHttpAccessEnabledCluster();
            using (var runspace = this.GetPowerShellRunspace())
            {
                var results = runspace.NewPipeline()
                                      .AddCommand(CmdletConstants.NewAzureHDInsightHiveJobDefinition)
                                      .WithParameter(CmdletConstants.JobName, hiveJobDefinition.JobName)
                                      .WithParameter(CmdletConstants.Query, hiveJobDefinition.Query)
                                      .AddCommand(CmdletConstants.StartAzureHDInsightJob)
                                      .WithParameter(CmdletConstants.Cluster, cluster.ConnectionUrl)
                                      .WithParameter(CmdletConstants.Credential, IntegrationTestBase.GetPSCredential(cluster.HttpUserName, cluster.HttpPassword))
                                      .Invoke();
                Assert.AreEqual(1, results.Results.Count);
                var job = results.Results.First().BaseObject as Microsoft.WindowsAzure.Management.HDInsight.Cmdlet.DataObjects.AzureHDInsightJob;

                results = runspace.NewPipeline()
                                      .AddCommand(CmdletConstants.WaitAzureHDInsightJob)
                                      .WithParameter(CmdletConstants.Credential, IntegrationTestBase.GetPSCredential(cluster.HttpUserName, cluster.HttpPassword))
                                      .WithParameter(CmdletConstants.JobId, job.JobId)
                                      .WithParameter(CmdletConstants.Cluster, job.Cluster)
                                      .Invoke();
                var completedJob = results.Results.ToEnumerable<AzureHDInsightJob>().FirstOrDefault();
                Assert.IsNotNull(completedJob);
                Assert.AreEqual(job.JobId, completedJob.JobId);
                Assert.AreEqual("Completed", completedJob.State);
            }
        }
Beispiel #6
0
        public Task <JobCreationResults> CreateHiveJobAsync(HiveJobCreateParameters hiveJobCreateParameters)
        {
            if (hiveJobCreateParameters.Query.IsNullOrEmpty())
            {
                hiveJobCreateParameters.File.ArgumentNotNullOrEmpty("File");
                if (hiveJobCreateParameters.File.Contains("://") &&
                    !hiveJobCreateParameters.File.StartsWith("wasb", StringComparison.OrdinalIgnoreCase))
                {
                    throw new InvalidOperationException("Invalid file protocol : " + hiveJobCreateParameters.File);
                }
            }
            else
            {
                this.PrepareQueryJob(hiveJobCreateParameters);
            }

            JobCreationResults retval =
                this.CreateJobSuccessResult(
                    new JobDetails
            {
                Name            = hiveJobCreateParameters.JobName,
                Query           = hiveJobCreateParameters.Query,
                StatusDirectory = hiveJobCreateParameters.StatusFolder
            },
                    hiveJobCreateParameters.JobName);

            return(TaskEx2.FromResult(retval));
        }
        public void CannotCreateNewHiveJob_WithRestrictedCharacters_StartJob()
        {
            var hiveJobDefinition = new HiveJobCreateParameters {
                JobName = "show tables jobDetails", Query = "show tables %"
            };

            INewAzureHDInsightHiveJobDefinitionCommand newHiveJobDefinitionCommand =
                ServiceLocator.Instance.Locate <IAzureHDInsightCommandFactory>().CreateNewHiveDefinition();

            newHiveJobDefinitionCommand.JobName = hiveJobDefinition.JobName;
            newHiveJobDefinitionCommand.Query   = hiveJobDefinition.Query;
            newHiveJobDefinitionCommand.EndProcessing();

            AzureHDInsightHiveJobDefinition hiveJobFromCommand = newHiveJobDefinitionCommand.Output.ElementAt(0);

            try
            {
                TestJobStart(hiveJobFromCommand);
                Assert.Fail();
            }
            catch (AggregateException aggregateException)
            {
                var invalidOperationException = aggregateException.GetBaseException() as InvalidOperationException;
                Assert.IsNotNull(invalidOperationException);
                Assert.IsTrue(invalidOperationException.Message.Contains("Query contains restricted character :'%'"), "Exception not thrown for special character");
            }
        }
        public void ICannotCallThe_New_HDInsightHiveJobDefinitionCmdlet_WithoutFileOrQueryParameter()
        {
            var HiveJobDefinition = new HiveJobCreateParameters
            {
                JobName = "show tables jobDetails",
                File    = TestConstants.WabsProtocolSchemeName + "filepath.hql"
            };

            try
            {
                using (IRunspace runspace = this.GetPowerShellRunspace())
                {
                    runspace.NewPipeline()
                    .AddCommand(CmdletConstants.NewAzureHDInsightHiveJobDefinition)
                    .WithParameter(CmdletConstants.JobName, HiveJobDefinition.JobName)
                    .Invoke();
                    Assert.Fail("test failed.");
                }
            }
            catch (CmdletInvocationException invokeException)
            {
                var psArgumentException = invokeException.GetBaseException() as PSArgumentException;
                Assert.IsNotNull(psArgumentException);
                Assert.AreEqual("Either File or Query should be specified for Hive jobs.", psArgumentException.Message);
            }
        }
        public void ICanCallThe_New_HDInsightHiveJobDefinitionCmdlet_WithArguments()
        {
            var HiveJobDefinition = new HiveJobCreateParameters {
                JobName = "show tables jobDetails", Query = "show tables"
            };

            HiveJobDefinition.Arguments.Add("arg 1");
            HiveJobDefinition.Arguments.Add("arg 2");

            using (IRunspace runspace = this.GetPowerShellRunspace())
            {
                IPipelineResult results =
                    runspace.NewPipeline()
                    .AddCommand(CmdletConstants.NewAzureHDInsightHiveJobDefinition)
                    .WithParameter(CmdletConstants.JobName, HiveJobDefinition.JobName)
                    .WithParameter(CmdletConstants.Query, HiveJobDefinition.Query)
                    .WithParameter(CmdletConstants.HiveArgs, HiveJobDefinition.Arguments)
                    .Invoke();
                Assert.AreEqual(1, results.Results.Count);
                AzureHDInsightHiveJobDefinition HiveJobFromPowershell = results.Results.ToEnumerable <AzureHDInsightHiveJobDefinition>().First();

                Assert.AreEqual(HiveJobDefinition.JobName, HiveJobFromPowershell.JobName);
                Assert.AreEqual(HiveJobDefinition.Query, HiveJobFromPowershell.Query);

                foreach (string args in HiveJobDefinition.Arguments)
                {
                    Assert.IsTrue(
                        HiveJobFromPowershell.Arguments.Any(arg => string.Equals(args, arg)),
                        "Unable to find argument '{0}' in value returned from powershell",
                        args);
                }
            }
        }
Beispiel #10
0
        public void schedule_hive_job()
        {
            var client = new WebHCatalogJobClient(Connect.WithTestUser(to: WebHcatBase));

            var job = new HiveJobCreateParameters
            {
                StatusFolder = "test.output",
                JobName      = "test-hive-query",
                Query        = @"
                    SELECT s07.description, s07.total_emp, s08.total_emp, s07.salary
                    FROM
                      sample_07 s07 JOIN 
                      sample_08 s08
                    ON ( s07.code = s08.code )
                    WHERE
                    ( s07.total_emp > s08.total_emp
                     AND s07.salary > 100000 )
                    SORT BY s07.salary DESC"
            };

            var result = client.SubmitHiveJob(job).Result;

            result.JobId.Should().NotBeNullOrEmpty();
            Console.WriteLine(result.JobId);
        }
        public virtual void WaitForJobWithId()
        {
            var hiveJobDefinition = new HiveJobCreateParameters()
            {
                JobName = "show tables jobDetails",
                Query   = "show tables"
            };

            var cluster = CmdletScenariosTestCaseBase.GetHttpAccessEnabledCluster();

            using (var runspace = this.GetPowerShellRunspace())
            {
                var results = runspace.NewPipeline()
                              .AddCommand(CmdletConstants.NewAzureHDInsightHiveJobDefinition)
                              .WithParameter(CmdletConstants.JobName, hiveJobDefinition.JobName)
                              .WithParameter(CmdletConstants.Query, hiveJobDefinition.Query)
                              .AddCommand(CmdletConstants.StartAzureHDInsightJob)
                              .WithParameter(CmdletConstants.Cluster, cluster.ConnectionUrl)
                              .WithParameter(CmdletConstants.Credential, IntegrationTestBase.GetPSCredential(cluster.HttpUserName, cluster.HttpPassword))
                              .Invoke();
                Assert.AreEqual(1, results.Results.Count);
                var job = results.Results.First().BaseObject as Microsoft.WindowsAzure.Management.HDInsight.Cmdlet.DataObjects.AzureHDInsightJob;

                results = runspace.NewPipeline()
                          .AddCommand(CmdletConstants.WaitAzureHDInsightJob)
                          .WithParameter(CmdletConstants.Credential, IntegrationTestBase.GetPSCredential(cluster.HttpUserName, cluster.HttpPassword))
                          .WithParameter(CmdletConstants.JobId, job.JobId)
                          .WithParameter(CmdletConstants.Cluster, job.Cluster)
                          .Invoke();
                var completedJob = results.Results.ToEnumerable <AzureHDInsightJob>().FirstOrDefault();
                Assert.IsNotNull(completedJob);
                Assert.AreEqual(job.JobId, completedJob.JobId);
                Assert.AreEqual("Completed", completedJob.State);
            }
        }
        public virtual void WaitForJob()
        {
            var hiveJobDefinition = new HiveJobCreateParameters {
                JobName = "show tables jobDetails", Query = "show tables"
            };

            // IHadoopClientExtensions.GetPollingInterval = () => 0;
            ClusterDetails cluster = CmdletScenariosTestCaseBase.GetHttpAccessEnabledCluster();

            using (IRunspace runspace = this.GetPowerShellRunspace())
            {
                IPipelineResult results =
                    runspace.NewPipeline()
                    .AddCommand(CmdletConstants.NewAzureHDInsightHiveJobDefinition)
                    .WithParameter(CmdletConstants.JobName, hiveJobDefinition.JobName)
                    .WithParameter(CmdletConstants.Query, hiveJobDefinition.Query)
                    .AddCommand(CmdletConstants.StartAzureHDInsightJob)
                    .WithParameter(CmdletConstants.Cluster, cluster.ConnectionUrl)
                    .WithParameter(CmdletConstants.Credential, GetPSCredential(cluster.HttpUserName, cluster.HttpPassword))
                    .AddCommand(CmdletConstants.WaitAzureHDInsightJob)
                    .WithParameter(CmdletConstants.Credential, GetPSCredential(cluster.HttpUserName, cluster.HttpPassword))
                    .Invoke();
                Assert.AreEqual(1, results.Results.Count);
                Assert.AreEqual("Completed", results.Results.ToEnumerable <AzureHDInsightJob>().First().State);
            }
        }
        public void CannotCreateNewHiveJob_WithRestrictedCharacters_StartJob()
        {
            var hiveJobDefinition = new HiveJobCreateParameters {
                JobName = "show tables jobDetails", Query = "show tables %"
            };

            INewAzureHDInsightHiveJobDefinitionCommand newHiveJobDefinitionCommand =
                ServiceLocator.Instance.Locate <IAzureHDInsightCommandFactory>().CreateNewHiveDefinition();

            newHiveJobDefinitionCommand.JobName = hiveJobDefinition.JobName;
            newHiveJobDefinitionCommand.Query   = hiveJobDefinition.Query;
            newHiveJobDefinitionCommand.EndProcessing();

            AzureHDInsightHiveJobDefinition hiveJobFromCommand = newHiveJobDefinitionCommand.Output.ElementAt(0);

            try
            {
                TestJobStart(hiveJobFromCommand);
                Assert.Fail();
            }
            catch (AggregateException aggregateException)
            {
                var invalidOperationException = aggregateException.GetBaseException() as InvalidOperationException;
                Assert.IsNotNull(invalidOperationException);
                Assert.AreEqual("Query text contains restricted character '%', please upload the query to a file in storage and re-submit the job using the -File parameter",
                                invalidOperationException.Message);
            }
        }
Beispiel #14
0
        /// <summary>
        /// Performs HQL query and returns the query results.
        /// </summary>
        /// <param name="jobParams">The query parameters.</param>
        /// <returns>The query result.</returns>
        public string Query(HiveJobCreateParameters jobParams)
        {
            // Assign status folder
            jobParams.StatusFolder = RootDirectory + "/status";

            JobCreationResults jobDetails = null;

            try
            {
                // Create Hive job
                jobDetails = this.job.CreateHiveJob(jobParams);
            }
            catch (Exception e)
            {
                AvroHdiSample.ReportError("Error while creating a Hive job\n" + e);
            }

            JobDetails jobInProgress = null;

            try
            {
                // Get job status
                jobInProgress = this.job.GetJob(jobDetails.JobId);
            }
            catch (Exception e)
            {
                AvroHdiSample.ReportError("Error while getting Hive job status\n" + e);
            }


            // If job is not finished then sleep until the next client polling interval
            while (jobInProgress.StatusCode != JobStatusCode.Completed &&
                   jobInProgress.StatusCode != JobStatusCode.Failed)
            {
                try
                {
                    // Get job status
                    jobInProgress = this.job.GetJob(jobDetails.JobId);
                }
                catch (Exception e)
                {
                    AvroHdiSample.ReportError("Error while getting Hive job status\n" + e);
                }

                Thread.Sleep(this.client.PollingInterval);
            }

            try
            {
                // Job is finished; get its output stream, read it, and return the value
                return(new StreamReader(this.job.GetJobOutput(jobDetails.JobId)).ReadToEnd());
            }
            catch (Exception e)
            {
                AvroHdiSample.ReportError("Error while reading Hibe job result\n" + e);
            }

            return(string.Empty);
        }
        public void GivenIHaveAHiveJobRequestObject()
        {
            var request = new HiveJobCreateParameters();

            request.JobName      = string.Empty;
            request.Query        = string.Empty;
            request.StatusFolder = string.Empty;
            this.transferObject  = request;
        }
            public Task <JobCreationResults> SubmitHiveJob(HiveJobCreateParameters details)
            {
                this.SubmitHiveJobCalled = true;
                var job = new JobCreationResults()
                {
                    JobId = JobId
                };

                return(Task.Run(() => job));
            }
Beispiel #17
0
        public void CanSerializeValidHiveJobRequest()
        {
            var hiveJob = new HiveJobCreateParameters()
            {
                Query = "show tables"
            };

            var payloadConverter = new PayloadConverterBase();
            var payload          = payloadConverter.SerializeHiveRequest("hadoop", hiveJob);

            Assert.IsTrue(payload.Contains(string.Format("{0}={1}", WebHCatConstants.Execute, Uri.EscapeDataString(hiveJob.Query))));
        }
Beispiel #18
0
        public void DoesNotAddDefineIfJobNameAbsent()
        {
            var hiveJob = new HiveJobCreateParameters()
            {
                Query = "show tables"
            };

            var payloadConverter = new PayloadConverterBase();
            var payload          = payloadConverter.SerializeHiveRequest("hadoop", hiveJob);

            Assert.IsFalse(payload.Contains(Uri.EscapeDataString(string.Format("{0}={1}", WebHCatConstants.DefineJobName, hiveJob.JobName))));
        }
Beispiel #19
0
        public void PayloadHasEnableLogsFalse()
        {
            var hiveJob = new HiveJobCreateParameters()
            {
                Query = "show tables"
            };

            var payloadConverter = new PayloadConverterBase();
            var payload          = payloadConverter.SerializeHiveRequest("hadoop", hiveJob);

            Assert.IsTrue(payload.Contains(string.Format("{0}={1}", HadoopRemoteRestConstants.EnableLogging, "false")));
        }
Beispiel #20
0
        public void CanSerializeValidHiveJobRequest_WithFile()
        {
            var hiveJob = new HiveJobCreateParameters()
            {
                File = Constants.WabsProtocolSchemeName + "filepath.hql"
            };

            var payloadConverter = new PayloadConverterBase();
            var payload          = payloadConverter.SerializeHiveRequest("hadoop", hiveJob);

            Assert.IsTrue(payload.Contains(string.Format("{0}={1}", WebHCatConstants.File, Uri.EscapeDataString(hiveJob.File))));
        }
        /// <inheritdoc />
        public async Task <JobCreationResults> SubmitHiveJob(HiveJobCreateParameters details)
        {
            //NEIN: Any code modification here should add unit tests for this class
            var converter = new PayloadConverter();
            var payload   = converter.SerializeHiveRequest(this.credentials.UserName, details);
            var client    = ServiceLocator.Instance.Locate <IHadoopRemoteJobSubmissionRestClientFactory>().Create(this.credentials, this.context, this.ignoreSslErrors, this.GetUserAgentString());
            var result    = await client.SubmitHiveJob(payload);

            return(new JobCreationResults()
            {
                JobId = converter.DeserializeJobSubmissionResponse(result.Content)
            });
        }
        public void CanCreateNewHiveJob_StartJob()
        {
            var hiveJobDefinition = new HiveJobCreateParameters { JobName = "show tables jobDetails", Query = "show tables" };

            INewAzureHDInsightHiveJobDefinitionCommand newHiveJobDefinitionCommand =
                ServiceLocator.Instance.Locate<IAzureHDInsightCommandFactory>().CreateNewHiveDefinition();
            newHiveJobDefinitionCommand.JobName = hiveJobDefinition.JobName;
            newHiveJobDefinitionCommand.Query = hiveJobDefinition.Query;
            newHiveJobDefinitionCommand.EndProcessing();

            AzureHDInsightHiveJobDefinition hiveJobFromCommand = newHiveJobDefinitionCommand.Output.ElementAt(0);
            TestJobStart(hiveJobFromCommand);
        }
Beispiel #23
0
        private async Task <JobCreationResults> ScheduleNewJob(string hiveQuery, string path, Guid jobIdentifier)
        {
            var jobParams = new HiveJobCreateParameters
            {
                StatusFolder = path,
                JobName      = jobIdentifier.ToString(),
                Query        = hiveQuery
            };

            var creationResult = await _jobClient.SubmitHiveJob(jobParams);

            return(creationResult);
        }
        public async Task<JobCreationResults> SubmitHiveJob(HiveJobCreateParameters details)
        {
            var relative = new Uri(
                HadoopRemoteRestConstants.Hive + "?" +
                HadoopRemoteRestConstants.UserName + "=" +
                _connectionConfig.UserName.EscapeDataString(),
                UriKind.Relative);

            var requestContent = _converter.SerializeHiveRequest(_connectionConfig.UserName, details);
            var result = await MakeAsyncJobSubmissionRequest(relative, requestContent);
            
            return new JobCreationResults { JobId = _converter.DeserializeJobSubmissionResponse(result) };
        }
        private static JobCreationResults RunHiveJob(HiveJobCreateParameters job)
        {
            JobCreationResults jobDetails    = jobSubmissionClient.CreateHiveJob(job);
            JobDetails         jobInProgress = jobSubmissionClient.GetJob(jobDetails.JobId);

            while (jobInProgress.StatusCode != JobStatusCode.Completed && jobInProgress.StatusCode != JobStatusCode.Failed)
            {
                jobInProgress = jobSubmissionClient.GetJob(jobInProgress.JobId);
                Thread.Sleep(TimeSpan.FromMilliseconds(IHadoopClientExtensions.GetPollingInterval()));
            }
            Assert.IsNull(jobDetails.ErrorCode, "Should not fail hive jobDetails submission");
            Assert.IsNotNull(jobDetails.JobId, "Should have a non-null jobDetails id");
            return(jobDetails);
        }
Beispiel #26
0
        public void JobCanSetEnableLogsTrue()
        {
            var hiveJob = new HiveJobCreateParameters()
            {
                Query          = "show tables",
                EnableTaskLogs = true,
                StatusFolder   = "/showtableslocation"
            };

            var payloadConverter = new PayloadConverterBase();
            var payload          = payloadConverter.SerializeHiveRequest("hadoop", hiveJob);

            Assert.IsTrue(payload.Contains(string.Format("{0}={1}", HadoopRemoteRestConstants.EnableLogging, "true")));
        }
Beispiel #27
0
        /// <inheritdoc/>
        public async Task <JobCreationResults> CreateHiveJobAsync(HiveJobCreateParameters hiveJobCreateParameters)
        {
            hiveJobCreateParameters = this.PrepareQueryJob(hiveJobCreateParameters);
            var pocoClient = this.GetPocoClient();

            try
            {
                return(await pocoClient.SubmitHiveJob(hiveJobCreateParameters));
            }
            catch (UnauthorizedAccessException)
            {
                pocoClient = this.GetPocoClient(true);
            }
            return(await pocoClient.SubmitHiveJob(hiveJobCreateParameters));
        }
        public async Task <JobCreationResults> SubmitHiveJob(HiveJobCreateParameters details)
        {
            var relative = new Uri(
                HadoopRemoteRestConstants.Hive + "?" +
                HadoopRemoteRestConstants.UserName + "=" +
                _connectionConfig.UserName.EscapeDataString(),
                UriKind.Relative);

            var requestContent = _converter.SerializeHiveRequest(_connectionConfig.UserName, details);
            var result         = await MakeAsyncJobSubmissionRequest(relative, requestContent);

            return(new JobCreationResults {
                JobId = _converter.DeserializeJobSubmissionResponse(result)
            });
        }
        public void CanCreateNewHiveJob_StartJob()
        {
            var hiveJobDefinition = new HiveJobCreateParameters {
                JobName = "show tables jobDetails", Query = "show tables"
            };

            INewAzureHDInsightHiveJobDefinitionCommand newHiveJobDefinitionCommand =
                ServiceLocator.Instance.Locate <IAzureHDInsightCommandFactory>().CreateNewHiveDefinition();

            newHiveJobDefinitionCommand.JobName = hiveJobDefinition.JobName;
            newHiveJobDefinitionCommand.Query   = hiveJobDefinition.Query;
            newHiveJobDefinitionCommand.EndProcessing();

            AzureHDInsightHiveJobDefinition hiveJobFromCommand = newHiveJobDefinitionCommand.Output.ElementAt(0);

            TestJobStart(hiveJobFromCommand);
        }
        public HiveJobCreateParameters DeserializeHiveJobCreationDetails(string content)
        {
            var request = this.DeserializePayload(content);
            var result  = new HiveJobCreateParameters()
            {
                JobName      = request.JobName,
                StatusFolder = request.OutputStorageLocation,
                Query        = request.Query
            };

            foreach (var jobRequestParameter in request.Parameters)
            {
                result.Defines.Add(jobRequestParameter.Key, jobRequestParameter.Value.ToString());
            }

            this.SetStandardProperties(request, result);
            return(result);
        }
        public void ICanCallThe_New_HDInsightHiveJobDefinitionCmdlet_WithoutJobName()
        {
            var HiveJobDefinition = new HiveJobCreateParameters {
                Query = "show tables"
            };

            using (IRunspace runspace = this.GetPowerShellRunspace())
            {
                IPipelineResult results =
                    runspace.NewPipeline()
                    .AddCommand(CmdletConstants.NewAzureHDInsightHiveJobDefinition)
                    .WithParameter(CmdletConstants.Query, HiveJobDefinition.Query)
                    .Invoke();
                Assert.AreEqual(1, results.Results.Count);
                AzureHDInsightHiveJobDefinition HiveJobFromPowershell = results.Results.ToEnumerable <AzureHDInsightHiveJobDefinition>().First();

                Assert.AreEqual(HiveJobDefinition.Query, HiveJobFromPowershell.Query);
            }
        }
        public virtual void ICanCallThe_NewHiveJob_Then_Start_HDInsightJobsCmdlet()
        {
            var hiveJobDefinition = new HiveJobCreateParameters { JobName = "show tables jobDetails", Query = "show tables" };

            using (IRunspace runspace = this.GetPowerShellRunspace())
            {
                IPipelineResult results =
                    runspace.NewPipeline()
                            .AddCommand(CmdletConstants.NewAzureHDInsightHiveJobDefinition)
                            .WithParameter(CmdletConstants.JobName, hiveJobDefinition.JobName)
                            .WithParameter(CmdletConstants.Query, hiveJobDefinition.Query)
                            .Invoke();
                Assert.AreEqual(1, results.Results.Count);
                AzureHDInsightHiveJobDefinition hiveJobFromPowershell = results.Results.ToEnumerable<AzureHDInsightHiveJobDefinition>().First();
                ClusterDetails testCluster = CmdletScenariosTestCaseBase.GetHttpAccessEnabledCluster();
                AzureHDInsightJob jobCreationDetails = RunJobInPowershell(runspace, hiveJobFromPowershell, testCluster);
                AzureHDInsightJob jobHistoryResult = GetJobsCmdletTests.GetJobWithID(runspace, jobCreationDetails.JobId, testCluster);
            }
        }
        public void ICanCallThe_New_HDInsightHiveJobDefinitionCmdlet()
        {
            var HiveJobDefinition = new HiveJobCreateParameters { JobName = "show tables jobDetails", Query = "show tables" };

            using (IRunspace runspace = this.GetPowerShellRunspace())
            {
                IPipelineResult results =
                    runspace.NewPipeline()
                            .AddCommand(CmdletConstants.NewAzureHDInsightHiveJobDefinition)
                            .WithParameter(CmdletConstants.JobName, HiveJobDefinition.JobName)
                            .WithParameter(CmdletConstants.Query, HiveJobDefinition.Query)
                            .Invoke();
                Assert.AreEqual(1, results.Results.Count);
                AzureHDInsightHiveJobDefinition HiveJobFromPowershell = results.Results.ToEnumerable<AzureHDInsightHiveJobDefinition>().First();

                Assert.AreEqual(HiveJobDefinition.JobName, HiveJobFromPowershell.JobName);
                Assert.AreEqual(HiveJobDefinition.Query, HiveJobFromPowershell.Query);
            }
        }
        private static async Task <JobCreationResults> SubmitHiveJob(
            AzureHDInsightHiveJobDefinition azureHiveJobDefinition, IJobSubmissionClient client)
        {
            AssertQueryDoesNotContainRestrictedCharacters(azureHiveJobDefinition.Query, "Query");
            var hiveJobDefinition = new HiveJobCreateParameters
            {
                JobName = azureHiveJobDefinition.JobName,
                Query   = azureHiveJobDefinition.Query,
                File    = azureHiveJobDefinition.File
            };

            hiveJobDefinition.StatusFolder = azureHiveJobDefinition.StatusFolder;
            hiveJobDefinition.Arguments.AddRange(azureHiveJobDefinition.Arguments);
            hiveJobDefinition.Defines.AddRange(azureHiveJobDefinition.Defines);
            hiveJobDefinition.Files.AddRange(azureHiveJobDefinition.Files);

            var jobCreationResults = await client.CreateHiveJobAsync(hiveJobDefinition);

            return(jobCreationResults);
        }
Beispiel #35
0
        private static async Task <JobCreationResults> SubmitHiveJob(
            AzureHDInsightHiveJobDefinition azureHiveJobDefinition, IJobSubmissionClient client)
        {
            var hiveJobDefinition = new HiveJobCreateParameters
            {
                JobName      = azureHiveJobDefinition.JobName,
                Query        = azureHiveJobDefinition.Query,
                File         = azureHiveJobDefinition.File,
                RunAsFileJob = azureHiveJobDefinition.RunAsFileJob
            };

            hiveJobDefinition.StatusFolder = azureHiveJobDefinition.StatusFolder;
            hiveJobDefinition.Arguments.AddRange(azureHiveJobDefinition.Arguments);
            hiveJobDefinition.Defines.AddRange(azureHiveJobDefinition.Defines);
            hiveJobDefinition.Files.AddRange(azureHiveJobDefinition.Files);

            var jobCreationResults = await client.CreateHiveJobAsync(hiveJobDefinition);

            return(jobCreationResults);
        }
Beispiel #36
0
        public virtual void ICanCallThe_NewHiveJob_Then_Start_HDInsightJobsCmdlet()
        {
            var hiveJobDefinition = new HiveJobCreateParameters {
                JobName = "show tables jobDetails", Query = "show tables"
            };

            using (IRunspace runspace = this.GetPowerShellRunspace())
            {
                IPipelineResult results =
                    runspace.NewPipeline()
                    .AddCommand(CmdletConstants.NewAzureHDInsightHiveJobDefinition)
                    .WithParameter(CmdletConstants.JobName, hiveJobDefinition.JobName)
                    .WithParameter(CmdletConstants.Query, hiveJobDefinition.Query)
                    .Invoke();
                Assert.AreEqual(1, results.Results.Count);
                AzureHDInsightHiveJobDefinition hiveJobFromPowershell = results.Results.ToEnumerable <AzureHDInsightHiveJobDefinition>().First();
                ClusterDetails    testCluster        = CmdletScenariosTestCaseBase.GetHttpAccessEnabledCluster();
                AzureHDInsightJob jobCreationDetails = RunJobInPowershell(runspace, hiveJobFromPowershell, testCluster);
                AzureHDInsightJob jobHistoryResult   = GetJobsCmdletTests.GetJobWithID(runspace, jobCreationDetails.JobId, testCluster);
            }
        }
        public Task<JobCreationResults> CreateHiveJobAsync(HiveJobCreateParameters hiveJobCreateParameters)
        {
            if (hiveJobCreateParameters.Query.IsNullOrEmpty())
            {
                hiveJobCreateParameters.File.ArgumentNotNullOrEmpty("File");
                if (hiveJobCreateParameters.File.Contains("://") &&
                    !hiveJobCreateParameters.File.StartsWith("wasb", StringComparison.OrdinalIgnoreCase))
                {
                    throw new InvalidOperationException("Invalid file protocol : " + hiveJobCreateParameters.File);
                }
            }

            JobCreationResults retval =
                this.CreateJobSuccessResult(
                    new JobDetails
                    {
                        Name = hiveJobCreateParameters.JobName,
                        Query = hiveJobCreateParameters.Query,
                        StatusDirectory = hiveJobCreateParameters.StatusFolder
                    },
                    hiveJobCreateParameters.JobName);
            return TaskEx2.FromResult(retval);
        }
        public void CannotCreateNewHiveJob_WithRestrictedCharacters_StartJob()
        {
            var hiveJobDefinition = new HiveJobCreateParameters { JobName = "show tables jobDetails", Query = "show tables %" };

            INewAzureHDInsightHiveJobDefinitionCommand newHiveJobDefinitionCommand =
                ServiceLocator.Instance.Locate<IAzureHDInsightCommandFactory>().CreateNewHiveDefinition();
            newHiveJobDefinitionCommand.JobName = hiveJobDefinition.JobName;
            newHiveJobDefinitionCommand.Query = hiveJobDefinition.Query;
            newHiveJobDefinitionCommand.EndProcessing();

            AzureHDInsightHiveJobDefinition hiveJobFromCommand = newHiveJobDefinitionCommand.Output.ElementAt(0);
            try
            {
                TestJobStart(hiveJobFromCommand);
                Assert.Fail();
            }
            catch (AggregateException aggregateException)
            {
                var invalidOperationException = aggregateException.GetBaseException() as InvalidOperationException;
                Assert.IsNotNull(invalidOperationException);
                Assert.AreEqual("Query text contains restricted character '%', please upload the query to a file in storage and re-submit the job using the -File parameter",
                    invalidOperationException.Message);
            }
        }
 /// <inheritdoc />
 public async Task<JobCreationResults> SubmitHiveJob(HiveJobCreateParameters details)
 {
     //NEIN: Any code modification here should add unit tests for this class
     var converter = new PayloadConverter();
     var payload = converter.SerializeHiveRequest(this.credentials.UserName, details);
     var client = ServiceLocator.Instance.Locate<IHadoopRemoteJobSubmissionRestClientFactory>().Create(this.credentials, this.context, this.ignoreSslErrors, this.GetUserAgentString());
     var result = await client.SubmitHiveJob(payload);
     return new JobCreationResults() { JobId = converter.DeserializeJobSubmissionResponse(result.Content) };
 }
 /// <summary>
 /// Creates the payload for a Hive request.
 /// </summary>
 /// <param name="userName"> The user name.</param>
 /// <param name="details"> The details.</param>
 /// <returns>A string that represents the payload for the request.</returns>
 public string SerializeHiveRequest(string userName, HiveJobCreateParameters details)
 {
     details.ArgumentNotNull("details");
     return this.SerializeQueryRequest(userName, details, details.JobName, details.File, details.Query, WebHCatConstants.Execute, details.Arguments, details.Defines);
 }
        public virtual void WaitForJob()
        {
            var hiveJobDefinition = new HiveJobCreateParameters { JobName = "show tables jobDetails", Query = "show tables" };

            // IHadoopClientExtensions.GetPollingInterval = () => 0;
            ClusterDetails cluster = CmdletScenariosTestCaseBase.GetHttpAccessEnabledCluster();
            using (IRunspace runspace = this.GetPowerShellRunspace())
            {
                IPipelineResult results =
                runspace.NewPipeline()
                            .AddCommand(CmdletConstants.NewAzureHDInsightHiveJobDefinition)
                            .WithParameter(CmdletConstants.JobName, hiveJobDefinition.JobName)
                            .WithParameter(CmdletConstants.Query, hiveJobDefinition.Query)
                            .AddCommand(CmdletConstants.StartAzureHDInsightJob)
                            .WithParameter(CmdletConstants.Cluster, cluster.ConnectionUrl)
                            .WithParameter(CmdletConstants.Credential, GetPSCredential(cluster.HttpUserName, cluster.HttpPassword))
                .AddCommand(CmdletConstants.WaitAzureHDInsightJob)
                            .WithParameter(CmdletConstants.Credential, GetPSCredential(cluster.HttpUserName, cluster.HttpPassword))
                .Invoke();
                Assert.AreEqual(1, results.Results.Count);
                Assert.AreEqual("Completed", results.Results.ToEnumerable<AzureHDInsightJob>().First().State);
            }
        }
 public JobCreationResults CreateHiveJob(HiveJobCreateParameters hiveJobCreateParameters)
 {
     return this.CreateHiveJobAsync(hiveJobCreateParameters).WaitForResult();
 }
        public void ICanCallThe_New_HDInsightHiveJobDefinitionCmdlet_WithParameters()
        {
            var HiveJobDefinition = new HiveJobCreateParameters { JobName = "show tables jobDetails", Query = "show tables" };

            HiveJobDefinition.Defines.Add("map.input.tasks", "1000");
            HiveJobDefinition.Defines.Add("map.input.reducers", "1000");

            using (IRunspace runspace = this.GetPowerShellRunspace())
            {
                IPipelineResult results =
                    runspace.NewPipeline()
                            .AddCommand(CmdletConstants.NewAzureHDInsightHiveJobDefinition)
                            .WithParameter(CmdletConstants.JobName, HiveJobDefinition.JobName)
                            .WithParameter(CmdletConstants.Query, HiveJobDefinition.Query)
                            .WithParameter(CmdletConstants.Parameters, HiveJobDefinition.Defines)
                            .Invoke();
                Assert.AreEqual(1, results.Results.Count);
                AzureHDInsightHiveJobDefinition HiveJobFromPowershell = results.Results.ToEnumerable<AzureHDInsightHiveJobDefinition>().First();

                Assert.AreEqual(HiveJobDefinition.JobName, HiveJobFromPowershell.JobName);
                Assert.AreEqual(HiveJobDefinition.Query, HiveJobFromPowershell.Query);

                foreach (var parameter in HiveJobDefinition.Defines)
                {
                    Assert.IsTrue(
                        HiveJobFromPowershell.Defines.Any(arg => string.Equals(parameter.Key, arg.Key) && string.Equals(parameter.Value, arg.Value)),
                        "Unable to find parameter '{0}' in value returned from powershell",
                        parameter.Key);
                }
            }
        }
        public void ICannotCallThe_New_HDInsightHiveJobDefinitionCmdlet_WithoutFileOrQueryParameter()
        {
            var HiveJobDefinition = new HiveJobCreateParameters
            {
                JobName = "show tables jobDetails",
                File = TestConstants.WabsProtocolSchemeName + "filepath.hql"
            };

            try
            {
                using (IRunspace runspace = this.GetPowerShellRunspace())
                {
                    runspace.NewPipeline()
                            .AddCommand(CmdletConstants.NewAzureHDInsightHiveJobDefinition)
                            .WithParameter(CmdletConstants.JobName, HiveJobDefinition.JobName)
                            .Invoke();
                    Assert.Fail("test failed.");
                }
            }
            catch (CmdletInvocationException invokeException)
            {
                var psArgumentException = invokeException.GetBaseException() as PSArgumentException;
                Assert.IsNotNull(psArgumentException);
                Assert.AreEqual("Either File or Query should be specified for Hive jobs.", psArgumentException.Message);
            }
        }