public void GetSqoopJob()
{
using (var context = UndoContext.Current)
{
context.Start();
var username = TestUtils.UserName;
var password = TestUtils.Password;
var clustername = TestUtils.ClusterName;
var credentials = new BasicAuthenticationCloudCredentials
{
Username = username,
Password = password
};
var client = TestUtils.GetHDInsightJobManagementClient(clustername, credentials);
var parameters = new SqoopJobSubmissionParameters
{
UserName = username,
Command = "some command",
StatusDir = "sqoopstatus",
};
var jobid = client.JobManagement.SubmitSqoopJob(parameters).JobSubmissionJsonResponse.Id;
var response = client.JobManagement.GetJob(jobid);
Assert.NotNull(response);
Assert.Equal(response.StatusCode, HttpStatusCode.OK);
}
}
public SqoopJobSubmissionParameters GetSqoopJobSubmissionParameters(bool isWindowsCluster = false)
{
var parameters = new SqoopJobSubmissionParameters
{
Command = "import --connect " + TestUtils.SQLServerConnectionString + " --table " + TestUtils.SQLServerTableName
+ " --warehouse-dir /user/admin/sqoop/" + Guid.NewGuid().ToString()
+ " --hive-import -m 1 --hive-table " + TestUtils.SQLServerTableName + Guid.NewGuid().ToString().Replace("-", ""),
StatusDir = "SqoopStatus",
};
return(parameters);
}
public SqoopJobSubmissionParameters GetSqoopJobSubmissionParameters()
{
var parameters = new SqoopJobSubmissionParameters
{
Command = "import --connect " + this.CommonData.SQLServerJdbcConnectionString + " --table " + this.CommonData.SQLServerTableName
+ " --warehouse-dir /user/admin/sqoop/" + Guid.NewGuid().ToString()
+ " --hive-import -m 1 --hive-table " + this.CommonData.SQLServerTableName + Guid.NewGuid().ToString().Replace("-", ""),
StatusDir = "SqoopStatus",
};
return(parameters);
}
public virtual JobSubmissionResponse SubmitSqoopJob(AzureHDInsightSqoopJobDefinition sqoopJobDef)
{
var sqoopJobParams = new SqoopJobSubmissionParameters
{
Command = sqoopJobDef.Command,
File = sqoopJobDef.File,
Files = ConvertListToString(sqoopJobDef.Files, "file"),
StatusDir = sqoopJobDef.StatusFolder,
UserName = HdInsightJobManagementClient.Credentials.Username
};
return(HdInsightJobManagementClient.JobManagement.SubmitSqoopJob(sqoopJobParams));
}
public virtual JobSubmissionResponse SubmitSqoopJob(AzureHDInsightSqoopJobDefinition sqoopJobDef)
{
var sqoopJobParams = new SqoopJobSubmissionParameters
{
Command = sqoopJobDef.Command,
File = sqoopJobDef.File,
Files = sqoopJobDef.Files,
LibDir = sqoopJobDef.LibDir,
StatusDir = sqoopJobDef.StatusFolder
};
return(HdInsightJobManagementClient.JobManagement.SubmitSqoopJob(sqoopJobParams));
}
public void SubmitSqoopJobAndValidateOutput(SqoopJobSubmissionParameters parameters, bool runAyncAPI = false, bool isWindowsCluster = false)
{
using (var context = UndoContext.Current)
{
context.Start();
var client = TestUtils.GetHDInsightJobManagementClient(isWindowsCluster);
var response = runAyncAPI ? client.JobManagement.SubmitSqoopJobAsync(parameters).Result
: client.JobManagement.SubmitSqoopJob(parameters);
Assert.NotNull(response);
Assert.Equal(response.StatusCode, HttpStatusCode.OK);
var jobId = response.JobSubmissionJsonResponse.Id;
Assert.Contains("job_", jobId, StringComparison.InvariantCulture);
WaitForJobCompletion(client, jobId, TestUtils.JobPollInterval, TestUtils.JobWaitInterval);
var jobStatus = client.JobManagement.GetJob(jobId);
var storageAccess = GetStorageAccessObject(isWindowsCluster);
if (jobStatus.JobDetail.ExitValue == 0)
{
if (HttpMockServer.Mode == HttpRecorderMode.Record)
{
// Retrieve Job Output
var output = client.JobManagement.GetJobOutput(jobId, storageAccess);
string textOutput = Convert(output);
Assert.True(textOutput.Length > 0);
}
}
else
{
if (HttpMockServer.Mode == HttpRecorderMode.Record)
{
var output = client.JobManagement.GetJobErrorLogs(jobId, storageAccess);
string errorTextOutput = Convert(output);
Assert.NotNull(errorTextOutput);
}
Assert.True(false);
}
}
}
public SqoopJobSubmissionParameters GetSqoopJobSubmissionParameters(bool isWindowsCluster = false)
{
var parameters = new SqoopJobSubmissionParameters
{
Command = "import --connect " + TestUtils.SQLServerConnectionString + " --table " + TestUtils.SQLServerTableName
+ " --warehouse-dir /user/admin/sqoop/" + Guid.NewGuid().ToString()
+ " --hive-import -m 1 --hive-table " + TestUtils.SQLServerTableName + Guid.NewGuid().ToString().Replace("-", ""),
StatusDir = "SqoopStatus",
};
if (!isWindowsCluster)
{
// This line is required for Linux-based cluster.
parameters.Files = new List <string> {
"/user/oozie/share/lib/sqoop/sqljdbc41.jar"
};
}
return(parameters);
}
private void SubmitSqoopJobAndValidateOutput(SqoopJobSubmissionParameters parameters, bool runAyncAPI = false)
{
using (var context = MockContext.Start(this.GetType()))
using (var client = this.CommonData.GetHDInsightJobClient(context))
{
var response = runAyncAPI ? client.Job.SubmitSqoopJobAsync(parameters).Result
: client.Job.SubmitSqoopJob(parameters);
Assert.NotNull(response);
var jobId = response.Id;
Assert.Contains("job_", jobId, StringComparison.InvariantCulture);
var jobStatus = client.Job.WaitForJobCompletion(jobId, this.CommonData.JobWaitInterval, this.CommonData.JobPollInterval);
var storageAccess = GetStorageAccessObject();
if (jobStatus.ExitValue == 0)
{
if (HttpMockServer.GetCurrentMode() == HttpRecorderMode.Record)
{
// Retrieve Job Output
var output = client.Job.GetJobOutput(jobId, storageAccess);
string textOutput = Convert(output);
Assert.True(textOutput.Length > 0);
}
}
else
{
if (HttpMockServer.GetCurrentMode() == HttpRecorderMode.Record)
{
var output = client.Job.GetJobErrorLogs(jobId, storageAccess);
string errorTextOutput = Convert(output);
Assert.NotNull(errorTextOutput);
}
Assert.True(false);
}
}
}
/// <summary>
/// Submits a Sqoop job to an HDInsight cluster.
/// </summary>
/// <param name='parameters'>
/// Required. Sqoop job parameters.
/// </param>
/// <returns>
/// The Create Job operation response.
/// </returns>
public async Task <JobSubmissionResponse> SubmitSqoopJobAsync(SqoopJobSubmissionParameters parameters)
{
return(await SubmitSqoopJobAsync(new JobSubmissionParameters { Content = parameters.GetJobPostRequestContent() }, CancellationToken.None));
}
/// <summary>
/// Submits a Sqoop job to an HDInsight cluster.
/// </summary>
/// <param name='operations'>
/// Reference to the
/// Microsoft.Azure.HDInsight.Job.IJobOperations.
/// </param>
/// <param name='parameters'>
/// Required. Sqoop job parameters.
/// </param>
/// <returns>
/// The Create Job operation response.
/// </returns>
public static async Task <JobSubmissionJsonResponse> SubmitSqoopJobAsync(this IJobOperations operations, SqoopJobSubmissionParameters parameters, CancellationToken cancellationToken = default(CancellationToken))
{
using (var _result = await operations.SubmitSqoopJobWithHttpMessagesAsync(parameters, null, cancellationToken).ConfigureAwait(false))
{
return(_result.Body);
}
}
/// <summary>
/// Submits a Sqoop job to an HDInsight cluster.
/// </summary>
/// <param name='operations'>
/// Reference to the
/// Microsoft.Azure.HDInsight.Job.IJobOperations.
/// </param>
/// <param name='parameters'>
/// Required. Sqoop job parameters.
/// </param>
/// <returns>
/// The Create Job operation response.
/// </returns>
public static JobSubmissionJsonResponse SubmitSqoopJob(this IJobOperations operations, SqoopJobSubmissionParameters parameters)
{
return(operations.SubmitSqoopJobAsync(parameters).GetAwaiter().GetResult());
}
/// <summary>
/// Submits a Sqoop job to an HDInsight cluster.
/// </summary>
/// <param name='parameters'>
/// Required. Sqoop job parameters.
/// </param>
/// <returns>
/// The Create Job operation response.
/// </returns>
public async Task <AzureOperationResponse <JobSubmissionJsonResponse> > SubmitSqoopJobWithHttpMessagesAsync(SqoopJobSubmissionParameters parameters, Dictionary <string, List <string> > customHeaders = null, CancellationToken cancellationToken = default(CancellationToken))
{
using (var requestContents = new MemoryStream(Encoding.UTF8.GetBytes(parameters.GetJobPostRequestContent())))
{
return(await SubmitSqoopJobWithHttpMessagesAsync(requestContents, customHeaders, cancellationToken).ConfigureAwait(false));
}
}
/// <summary>
/// Submits an Sqoop job to an HDINSIGHT cluster.
/// </summary>
/// <param name='operations'>
/// Reference to the
/// Microsoft.Azure.Management.HDInsight.Job.IJobOperations.
/// </param>
/// <param name='parameters'>
/// Required. Sqoop job parameters.
/// </param>
/// <returns>
/// The Create Job operation response.
/// </returns>
public static Task <JobSubmissionResponse> SubmitSqoopJobAsync(this IJobOperations operations, SqoopJobSubmissionParameters parameters)
{
return(operations.SubmitSqoopJobAsync(parameters, CancellationToken.None));
}
/// <summary>
/// Submits an Sqoop job to an HDINSIGHT cluster.
/// </summary>
/// <param name='operations'>
/// Reference to the
/// Microsoft.Azure.Management.HDInsight.Job.IJobOperations.
/// </param>
/// <param name='parameters'>
/// Required. Sqoop job parameters.
/// </param>
/// <returns>
/// The Create Job operation response.
/// </returns>
public static JobSubmissionResponse SubmitSqoopJob(this IJobOperations operations, SqoopJobSubmissionParameters parameters)
{
return(Task.Factory.StartNew((object s) =>
{
return ((IJobOperations)s).SubmitSqoopJobAsync(parameters);
}
, operations, CancellationToken.None, TaskCreationOptions.None, TaskScheduler.Default).Unwrap().GetAwaiter().GetResult());
}
public void SubmitSqoopJob()
{
using (var context = UndoContext.Current)
{
context.Start();
var username = TestUtils.UserName;
var password = TestUtils.Password;
var clustername = TestUtils.ClusterName;
var credentials = new BasicAuthenticationCloudCredentials
{
Username = username,
Password = password
};
var client = TestUtils.GetHDInsightJobManagementClient(clustername, credentials);
// Before we run this test in Record mode, we should run following commands on cluster
// hdfs dfs -mkdir /user/hcat/lib
// hadoop fs -copyFromLocal -f /usr/share/java/sqljdbc_4.1/enu/sqljdbc41.jar /user/hcat/lib
// Generate sqoopcommand.txt using content
// --connect
// <Connection string to DB which has table dept.>
// --table
// dept
// Keep these in separate lines otherwise, sqoop command will fail. Copy the sqoopcommand.txt
// hdfs dfs -mkdir /example/data/sqoop/
// hadoop fs -copyFromLocal -f sqoopcommand.txt /example/data/sqoop/
var parameters = new SqoopJobSubmissionParameters
{
LibDir = "/user/hcat/lib",
Files = new List <string> {
"/example/data/sqoop/sqoopcommand.txt"
},
Command = "import --options-file sqoopcommand.txt --hive-import -m 1",
StatusDir = "sqoopstatus",
};
var response = client.JobManagement.SubmitSqoopJob(parameters);
Assert.NotNull(response);
Assert.Equal(response.StatusCode, HttpStatusCode.OK);
var jobId = response.JobSubmissionJsonResponse.Id;
Assert.Contains("job_", jobId, StringComparison.InvariantCulture);
var jobStatus = GetJobFinalStatus(client, jobId);
var storageAccess = GetStorageAccessObject();
if (jobStatus.JobDetail.ExitValue == 0)
{
if (HttpMockServer.Mode == HttpRecorderMode.Record)
{
// Retrieve Job Output
var output = client.JobManagement.GetJobOutput(jobId, storageAccess);
string textOutput = Convert(output);
Assert.True(textOutput.Length > 0);
}
}
else
{
if (HttpMockServer.Mode == HttpRecorderMode.Record)
{
var output = client.JobManagement.GetJobErrorLogs(jobId, storageAccess);
string errorTextOutput = Convert(output);
Assert.NotNull(errorTextOutput);
}
Assert.True(false);
}
}
}
/// <summary>
/// Submits a Sqoop job to an HDInsight cluster.
/// </summary>
/// <param name='operations'>
/// Reference to the
/// Microsoft.Azure.Management.HDInsight.Job.IJobOperations.
/// </param>
/// <param name='parameters'>
/// Required. Sqoop job parameters.
/// </param>
/// <returns>
/// The Create Job operation response.
/// </returns>
public static JobSubmissionResponse SubmitSqoopJob(this IJobOperations operations, SqoopJobSubmissionParameters parameters)
{
return(operations.SubmitSqoopJob(new JobSubmissionParameters {
Content = parameters.GetJobPostRequestContent()
}));
}
