public void GetHiveJob() { using (var context = UndoContext.Current) { context.Start(); var username = TestUtils.UserName; var password = TestUtils.Password; var clustername = TestUtils.ClusterName; var credentials = new BasicAuthenticationCloudCredentials { Username = username, Password = password }; var client = TestUtils.GetHDInsightJobManagementClient(clustername, credentials); var parameters = new HiveJobSubmissionParameters { UserName = username, Query = "SHOW TABLES;", Arguments = "arg1", Defines = "def1" }; var jobid = client.JobManagement.SubmitHiveJob(parameters).JobSubmissionJsonResponse.Id; var response = client.JobManagement.GetJob(jobid); Assert.NotNull(response); Assert.Equal(response.StatusCode, HttpStatusCode.OK); } }
public virtual JobSubmissionResponse SubmitHiveJob(AzureHDInsightHiveJobDefinition hiveJobDef) { var hiveJobParams = new HiveJobSubmissionParameters { Arguments = hiveJobDef.Arguments, Defines = hiveJobDef.Defines, File = hiveJobDef.File, Files = hiveJobDef.Files, Query = hiveJobDef.Query, StatusDir = hiveJobDef.StatusFolder }; return HdInsightJobManagementClient.JobManagement.SubmitHiveJob(hiveJobParams); }
public virtual JobSubmissionResponse SubmitHiveJob(AzureHDInsightHiveJobDefinition hiveJobDef) { var hiveJobParams = new HiveJobSubmissionParameters { Arguments = ConvertListToString(hiveJobDef.Arguments, "arg"), Defines = ConvertDefinesToString(hiveJobDef.Defines), File = hiveJobDef.File, Files = ConvertListToString(hiveJobDef.Files, "file"), Query = hiveJobDef.Query, StatusDir = hiveJobDef.StatusFolder, UserName = HdInsightJobManagementClient.Credentials.Username }; return HdInsightJobManagementClient.JobManagement.SubmitHiveJob(hiveJobParams); }
// [Fact] public void GetJobOutput() { using (var context = UndoContext.Current) { context.Start(); var username = TestUtils.UserName; var password = TestUtils.Password; var clustername = TestUtils.ClusterName; var credentials = new BasicAuthenticationCloudCredentials { Username = username, Password = password }; var client = TestUtils.GetHDInsightJobManagementClient(clustername, credentials); var parameters = new HiveJobSubmissionParameters { UserName = username, Query = "SHOW TABLES;", StatusDir = "jobstatus" }; var response = client.JobManagement.SubmitHiveJob(parameters); Assert.NotNull(response); Assert.Equal(response.StatusCode, HttpStatusCode.OK); var job = client.JobManagement.GetJob(response.JobSubmissionJsonResponse.Id); var output = client.JobManagement.GetJobOutput(job.JobDetail.Id, TestUtils.StorageAccountName, TestUtils.StorageAccountKey, TestUtils.DefaultContainer); Assert.NotNull(output); var outputStr = Convert(output); Assert.NotNull(outputStr); } }
public void SubmitHiveJob() { var parameters = new HiveJobSubmissionParameters { Query = @"select querydwelltime+2 from hivesampletable where clientid = 8" }; SubmitHiveJobAndValidateOutput(parameters, "15.92"); }
public void SubmitHiveJob_Defines() { var defines = new Dictionary<string, string> { { "hive.execution.engine", "ravi" }, { "hive.exec.reducers.max", "1" }, { "time", "10" }, { "rows", "20" } }; var parameters = new HiveJobSubmissionParameters { Query = @"select * from hivesampletable where querydwelltime > ${hiveconf:time} limit ${hiveconf:rows}", Defines = defines }; SubmitHiveJobAndValidateOutput(parameters, "Massachusetts United States"); }
public void SubmitHiveJobError() { using (var context = UndoContext.Current) { context.Start(); var username = TestUtils.UserName; var password = TestUtils.Password; var clustername = TestUtils.ClusterName; var credentials = new BasicAuthenticationCloudCredentials { Username = username, Password = password }; var client = TestUtils.GetHDInsightJobManagementClient(clustername, credentials); var parameters = new HiveJobSubmissionParameters { Query = "FAKEQUERY;", StatusDir = "jobstatus" }; var response = client.JobManagement.SubmitHiveJob(parameters); Assert.NotNull(response); Assert.Equal(response.StatusCode, HttpStatusCode.OK); var jobId = response.JobSubmissionJsonResponse.Id; Assert.Contains("job_", jobId, StringComparison.InvariantCulture); var jobStatus = GetJobFinalStatus(client, jobId); Assert.True(jobStatus.JobDetail.ExitValue > 0); var storageAccess = GetStorageAccessObject(); if (HttpMockServer.Mode == HttpRecorderMode.Record) { var output = client.JobManagement.GetJobErrorLogs(jobId, storageAccess); Assert.NotNull(output); Assert.True(output.Length > 0); string errorTextOutput = Convert(output); Assert.True(!string.IsNullOrEmpty(errorTextOutput)); } } }
public void SubmitHiveJobAndValidateOutput(HiveJobSubmissionParameters parameters, string expectedOutputPart) { using (var context = UndoContext.Current) { context.Start(); var username = TestUtils.UserName; var password = TestUtils.Password; var clustername = TestUtils.ClusterName; var credentials = new BasicAuthenticationCloudCredentials { Username = username, Password = password }; var client = TestUtils.GetHDInsightJobManagementClient(clustername, credentials); var response = client.JobManagement.SubmitHiveJob(parameters); Assert.NotNull(response); Assert.Equal(response.StatusCode, HttpStatusCode.OK); var jobId = response.JobSubmissionJsonResponse.Id; Assert.Contains("job_", jobId, StringComparison.InvariantCulture); var jobStatus = GetJobFinalStatus(client, jobId); var storageAccess = GetStorageAccessObject(); if (jobStatus.JobDetail.ExitValue == 0) { if (HttpMockServer.Mode == HttpRecorderMode.Record) { // Retrieve Job Output var output = client.JobManagement.GetJobOutput(jobId, storageAccess); string textOutput = Convert(output); Assert.True(textOutput.Contains(expectedOutputPart)); } } else { if (HttpMockServer.Mode == HttpRecorderMode.Record) { var output = client.JobManagement.GetJobErrorLogs(jobId, storageAccess); string errorTextOutput = Convert(output); Assert.NotNull(errorTextOutput); } Assert.True(false); } } }
/// <summary> /// Submits an Hive job to an HDINSIGHT cluster. /// </summary> /// <param name='operations'> /// Reference to the /// Microsoft.Azure.Management.HDInsight.Job.IJobOperations. /// </param> /// <param name='parameters'> /// Required. Hive job parameters. /// </param> /// <returns> /// The Create Job operation response. /// </returns> public static Task<JobSubmissionResponse> SubmitHiveJobAsync(this IJobOperations operations, HiveJobSubmissionParameters parameters) { return operations.SubmitHiveJobAsync(parameters, CancellationToken.None); }
/// <summary> /// Submits an Hive job to an HDINSIGHT cluster. /// </summary> /// <param name='operations'> /// Reference to the /// Microsoft.Azure.Management.HDInsight.Job.IJobOperations. /// </param> /// <param name='parameters'> /// Required. Hive job parameters. /// </param> /// <returns> /// The Create Job operation response. /// </returns> public static JobSubmissionResponse SubmitHiveJob(this IJobOperations operations, HiveJobSubmissionParameters parameters) { return Task.Factory.StartNew((object s) => { return ((IJobOperations)s).SubmitHiveJobAsync(parameters); } , operations, CancellationToken.None, TaskCreationOptions.None, TaskScheduler.Default).Unwrap().GetAwaiter().GetResult(); }
/// <summary> /// Submits a Hive job to an HDInsight cluster. /// </summary> /// <param name='operations'> /// Reference to the /// Microsoft.Azure.Management.HDInsight.Job.IJobOperations. /// </param> /// <param name='parameters'> /// Required. Hive job parameters. /// </param> /// <returns> /// The Create Job operation response. /// </returns> public static JobSubmissionResponse SubmitHiveJob(this IJobOperations operations, HiveJobSubmissionParameters parameters) { return operations.SubmitHiveJob(new JobSubmissionParameters { Content = parameters.GetJobPostRequestContent() }); }
public void SubmitHiveJobError() { using (var context = UndoContext.Current) { context.Start(); var client = TestUtils.GetHDInsightJobManagementClient(); var parameters = new HiveJobSubmissionParameters { Query = "FAKEQUERY;", StatusDir = "jobstatus" }; var response = client.JobManagement.SubmitHiveJob(parameters); Assert.NotNull(response); Assert.Equal(response.StatusCode, HttpStatusCode.OK); var jobId = response.JobSubmissionJsonResponse.Id; Assert.Contains("job_", jobId, StringComparison.InvariantCulture); var jobStatus = client.JobManagement.WaitForJobCompletionAsync(jobId).Result; Assert.True(jobStatus.JobDetail.ExitValue > 0); var storageAccess = GetStorageAccessObject(); if (HttpMockServer.GetCurrentMode() == HttpRecorderMode.Record) { var output = client.JobManagement.GetJobErrorLogs(jobId, storageAccess); Assert.NotNull(output); Assert.True(output.Length > 0); string errorTextOutput = Convert(output); Assert.True(!string.IsNullOrEmpty(errorTextOutput)); } } }
public void SubmitHiveJobAndValidateOutput(HiveJobSubmissionParameters parameters, string expectedOutputPart, bool runAyncAPI = false, bool isWindowsCluster = false) { using (var context = UndoContext.Current) { context.Start(); var client = TestUtils.GetHDInsightJobManagementClient(isWindowsCluster); var response = runAyncAPI ? client.JobManagement.SubmitHiveJobAsync(parameters).Result : client.JobManagement.SubmitHiveJob(parameters); Assert.NotNull(response); Assert.Equal(response.StatusCode, HttpStatusCode.OK); var jobId = response.JobSubmissionJsonResponse.Id; Assert.Contains("job_", jobId, StringComparison.InvariantCulture); var jobStatus = client.JobManagement.WaitForJobCompletion(jobId, TestUtils.JobWaitInterval, TestUtils.JobPollInterval); var storageAccess = GetStorageAccessObject(isWindowsCluster); if (jobStatus.JobDetail.ExitValue == 0) { if (HttpMockServer.GetCurrentMode() == HttpRecorderMode.Record) { // Retrieve Job Output var output = client.JobManagement.GetJobOutput(jobId, storageAccess); string textOutput = Convert(output); Assert.True(textOutput.Contains(expectedOutputPart)); } } else { if (HttpMockServer.GetCurrentMode() == HttpRecorderMode.Record) { var output = client.JobManagement.GetJobErrorLogs(jobId, storageAccess); string errorTextOutput = Convert(output); Assert.NotNull(errorTextOutput); } Assert.True(false); } } }
public void SubmitHiveJobAsync_Windows() { var parameters = new HiveJobSubmissionParameters { Query = @"select querydwelltime+2 from hivesampletable where clientid = 8" }; SubmitHiveJobAndValidateOutput(parameters, "15.92", runAyncAPI: true, isWindowsCluster : true); }
public void SubmitHiveJobLargeQuery() { var query = string.Empty; // Maximum input string size Uri.EscapeDataString can accept is 65520. while(query.Length < 65520) { query += "select 1.0000000001 + 0.00001 limit 1;"; } var parameters = new HiveJobSubmissionParameters { Query = query }; SubmitHiveJobAndValidateOutput(parameters, "1.0000100001"); }
public void SubmitHiveJob() { using (var context = UndoContext.Current) { context.Start(); var username = TestUtils.UserName; var password = TestUtils.Password; var clustername = TestUtils.ClusterName; var credentials = new BasicAuthenticationCloudCredentials { Username = username, Password = password }; var client = TestUtils.GetHDInsightJobManagementClient(clustername, credentials); var defines = new Dictionary<string, string> { {"hive.execution.engine", "ravi"}, {"hive.exec.reducers.max", "1"} }; var args = new List<string> {"argA", "argB"}; var parameters = new HiveJobSubmissionParameters { UserName = username, Query = "SHOW TABLES;", Defines = ConvertDefinesToString(defines), Arguments = ConvertArgsToString(args) }; var response = client.JobManagement.SubmitHiveJob(parameters); Assert.NotNull(response); Assert.Equal(response.StatusCode, HttpStatusCode.OK); Assert.Contains("job_", response.JobSubmissionJsonResponse.Id, StringComparison.InvariantCulture); } }