public void SubmitSqoopJob() { using (var context = UndoContext.Current) { context.Start(); var username = TestUtils.UserName; var password = TestUtils.Password; var clustername = TestUtils.ClusterName; var credentials = new BasicAuthenticationCloudCredentials { Username = username, Password = password }; var client = TestUtils.GetHDInsightJobManagementClient(clustername, credentials); // Before we run this test in Record mode, we should run following commands on cluster // hdfs dfs -mkdir /user/hcat/lib // hadoop fs -copyFromLocal -f /usr/share/java/sqljdbc_4.1/enu/sqljdbc41.jar /user/hcat/lib // Generate sqoopcommand.txt using content // --connect // <Connection string to DB which has table dept.> // --table // dept // Keep these in separate lines otherwise, sqoop command will fail. Copy the sqoopcommand.txt // hdfs dfs -mkdir /example/data/sqoop/ // hadoop fs -copyFromLocal -f sqoopcommand.txt /example/data/sqoop/ var parameters = new SqoopJobSubmissionParameters { LibDir = "/user/hcat/lib", Files = new List<string>{"/example/data/sqoop/sqoopcommand.txt"}, Command = "import --options-file sqoopcommand.txt --hive-import -m 1", StatusDir = "sqoopstatus", }; var response = client.JobManagement.SubmitSqoopJob(parameters); Assert.NotNull(response); Assert.Equal(response.StatusCode, HttpStatusCode.OK); var jobId = response.JobSubmissionJsonResponse.Id; Assert.Contains("job_", jobId, StringComparison.InvariantCulture); var jobStatus = GetJobFinalStatus(client, jobId); var storageAccess = GetStorageAccessObject(); if (jobStatus.JobDetail.ExitValue == 0) { if (HttpMockServer.Mode == HttpRecorderMode.Record) { // Retrieve Job Output var output = client.JobManagement.GetJobOutput(jobId, storageAccess); string textOutput = Convert(output); Assert.True(textOutput.Length > 0); } } else { if (HttpMockServer.Mode == HttpRecorderMode.Record) { var output = client.JobManagement.GetJobErrorLogs(jobId, storageAccess); string errorTextOutput = Convert(output); Assert.NotNull(errorTextOutput); } Assert.True(false); } } }
/// <summary> /// Submits a Sqoop job to an HDInsight cluster. /// </summary> /// <param name='operations'> /// Reference to the /// Microsoft.Azure.Management.HDInsight.Job.IJobOperations. /// </param> /// <param name='parameters'> /// Required. Sqoop job parameters. /// </param> /// <returns> /// The Create Job operation response. /// </returns> public static JobSubmissionResponse SubmitSqoopJob(this IJobOperations operations, SqoopJobSubmissionParameters parameters) { return operations.SubmitSqoopJob(new JobSubmissionParameters { Content = parameters.GetJobPostRequestContent() }); }
/// <summary> /// Submits a Sqoop job to an HDInsight cluster. /// </summary> /// <param name='operations'> /// Reference to the /// Microsoft.Azure.Management.HDInsight.Job.IJobOperations. /// </param> /// <param name='parameters'> /// Required. Sqoop job parameters. /// </param> /// <returns> /// The Create Job operation response. /// </returns> public static Task<JobSubmissionResponse> SubmitSqoopJobAsync(this IJobOperations operations, SqoopJobSubmissionParameters parameters) { return operations.SubmitSqoopJobAsync(parameters); }
public void SubmitSqoopJobAndValidateOutput(SqoopJobSubmissionParameters parameters, bool runAyncAPI = false, bool isWindowsCluster = false) { using (var context = UndoContext.Current) { context.Start(); var client = TestUtils.GetHDInsightJobManagementClient(isWindowsCluster); var response = runAyncAPI ? client.JobManagement.SubmitSqoopJobAsync(parameters).Result : client.JobManagement.SubmitSqoopJob(parameters); Assert.NotNull(response); Assert.Equal(response.StatusCode, HttpStatusCode.OK); var jobId = response.JobSubmissionJsonResponse.Id; Assert.Contains("job_", jobId, StringComparison.InvariantCulture); var jobStatus = client.JobManagement.WaitForJobCompletion(jobId, TestUtils.JobWaitInterval, TestUtils.JobPollInterval); var storageAccess = GetStorageAccessObject(isWindowsCluster); if (jobStatus.JobDetail.ExitValue == 0) { if (HttpMockServer.GetCurrentMode() == HttpRecorderMode.Record) { // Retrieve Job Output var output = client.JobManagement.GetJobOutput(jobId, storageAccess); string textOutput = Convert(output); Assert.True(textOutput.Length > 0); } } else { if (HttpMockServer.GetCurrentMode() == HttpRecorderMode.Record) { var output = client.JobManagement.GetJobErrorLogs(jobId, storageAccess); string errorTextOutput = Convert(output); Assert.NotNull(errorTextOutput); } Assert.True(false); } } }
public void GetSqoopJob() { using (var context = UndoContext.Current) { context.Start(); var username = TestUtils.UserName; var password = TestUtils.Password; var clustername = TestUtils.ClusterName; var credentials = new BasicAuthenticationCloudCredentials { Username = username, Password = password }; var client = TestUtils.GetHDInsightJobManagementClient(clustername, credentials); var parameters = new SqoopJobSubmissionParameters { UserName = username, Command = "some command", StatusDir = "sqoopstatus", }; var jobid = client.JobManagement.SubmitSqoopJob(parameters).JobSubmissionJsonResponse.Id; var response = client.JobManagement.GetJob(jobid); Assert.NotNull(response); Assert.Equal(response.StatusCode, HttpStatusCode.OK); } }
public SqoopJobSubmissionParameters GetSqoopJobSubmissionParameters(bool isWindowsCluster = false) { var parameters = new SqoopJobSubmissionParameters { Command = "import --connect " + TestUtils.SQLServerConnectionString + " --table " + TestUtils.SQLServerTableName + " --warehouse-dir /user/admin/sqoop/" + Guid.NewGuid().ToString() + " --hive-import -m 1 --hive-table " + TestUtils.SQLServerTableName + Guid.NewGuid().ToString().Replace("-",""), StatusDir = "SqoopStatus", }; return parameters; }
public SqoopJobSubmissionParameters GetSqoopJobSubmissionParameters(bool isWindowsCluster = false) { var parameters = new SqoopJobSubmissionParameters { Command = "import --connect " + TestUtils.SQLServerConnectionString + " --table " + TestUtils.SQLServerTableName + " --warehouse-dir /user/admin/sqoop/" + Guid.NewGuid().ToString() + " --hive-import -m 1 --hive-table " + TestUtils.SQLServerTableName + Guid.NewGuid().ToString().Replace("-",""), StatusDir = "SqoopStatus", }; if (!isWindowsCluster) { // This line is required for Linux-based cluster. parameters.Files = new List<string> { "/user/oozie/share/lib/sqoop/sqljdbc41.jar" }; } return parameters; }
/// <summary> /// Submits an Sqoop job to an HDINSIGHT cluster. /// </summary> /// <param name='operations'> /// Reference to the /// Microsoft.Azure.Management.HDInsight.Job.IJobOperations. /// </param> /// <param name='parameters'> /// Required. Sqoop job parameters. /// </param> /// <returns> /// The Create Job operation response. /// </returns> public static JobSubmissionResponse SubmitSqoopJob(this IJobOperations operations, SqoopJobSubmissionParameters parameters) { return Task.Factory.StartNew((object s) => { return ((IJobOperations)s).SubmitSqoopJobAsync(parameters); } , operations, CancellationToken.None, TaskCreationOptions.None, TaskScheduler.Default).Unwrap().GetAwaiter().GetResult(); }
public virtual JobSubmissionResponse SubmitSqoopJob(AzureHDInsightSqoopJobDefinition sqoopJobDef) { var sqoopJobParams = new SqoopJobSubmissionParameters { Command = sqoopJobDef.Command, File = sqoopJobDef.File, Files = ConvertListToString(sqoopJobDef.Files, "file"), StatusDir = sqoopJobDef.StatusFolder, UserName = HdInsightJobManagementClient.Credentials.Username }; return HdInsightJobManagementClient.JobManagement.SubmitSqoopJob(sqoopJobParams); }
public virtual JobSubmissionResponse SubmitSqoopJob(AzureHDInsightSqoopJobDefinition sqoopJobDef) { var sqoopJobParams = new SqoopJobSubmissionParameters { Command = sqoopJobDef.Command, File = sqoopJobDef.File, Files = sqoopJobDef.Files, LibDir = sqoopJobDef.LibDir, StatusDir = sqoopJobDef.StatusFolder }; return HdInsightJobManagementClient.JobManagement.SubmitSqoopJob(sqoopJobParams); }