public void GetHiveJob()
{
using (var context = UndoContext.Current)
{
context.Start();
var username = TestUtils.UserName;
var password = TestUtils.Password;
var clustername = TestUtils.ClusterName;
var credentials = new BasicAuthenticationCloudCredentials
{
Username = username,
Password = password
};
var client = TestUtils.GetHDInsightJobManagementClient(clustername, credentials);
var parameters = new HiveJobSubmissionParameters
{
UserName = username,
Query = "SHOW TABLES;",
Arguments = "arg1",
Defines = "def1"
};
var jobid = client.JobManagement.SubmitHiveJob(parameters).JobSubmissionJsonResponse.Id;
var response = client.JobManagement.GetJob(jobid);
Assert.NotNull(response);
Assert.Equal(response.StatusCode, HttpStatusCode.OK);
}
}
public virtual JobSubmissionResponse SubmitHiveJob(AzureHDInsightHiveJobDefinition hiveJobDef)
{
var hiveJobParams = new HiveJobSubmissionParameters
{
Arguments = hiveJobDef.Arguments,
Defines = hiveJobDef.Defines,
File = hiveJobDef.File,
Files = hiveJobDef.Files,
Query = hiveJobDef.Query,
StatusDir = hiveJobDef.StatusFolder
};
return HdInsightJobManagementClient.JobManagement.SubmitHiveJob(hiveJobParams);
}
public virtual JobSubmissionResponse SubmitHiveJob(AzureHDInsightHiveJobDefinition hiveJobDef)
{
var hiveJobParams = new HiveJobSubmissionParameters
{
Arguments = ConvertListToString(hiveJobDef.Arguments, "arg"),
Defines = ConvertDefinesToString(hiveJobDef.Defines),
File = hiveJobDef.File,
Files = ConvertListToString(hiveJobDef.Files, "file"),
Query = hiveJobDef.Query,
StatusDir = hiveJobDef.StatusFolder,
UserName = HdInsightJobManagementClient.Credentials.Username
};
return HdInsightJobManagementClient.JobManagement.SubmitHiveJob(hiveJobParams);
}
// [Fact]
public void GetJobOutput()
{
using (var context = UndoContext.Current)
{
context.Start();
var username = TestUtils.UserName;
var password = TestUtils.Password;
var clustername = TestUtils.ClusterName;
var credentials = new BasicAuthenticationCloudCredentials
{
Username = username,
Password = password
};
var client = TestUtils.GetHDInsightJobManagementClient(clustername, credentials);
var parameters = new HiveJobSubmissionParameters
{
UserName = username,
Query = "SHOW TABLES;",
StatusDir = "jobstatus"
};
var response = client.JobManagement.SubmitHiveJob(parameters);
Assert.NotNull(response);
Assert.Equal(response.StatusCode, HttpStatusCode.OK);
var job = client.JobManagement.GetJob(response.JobSubmissionJsonResponse.Id);
var output = client.JobManagement.GetJobOutput(job.JobDetail.Id,
TestUtils.StorageAccountName, TestUtils.StorageAccountKey, TestUtils.DefaultContainer);
Assert.NotNull(output);
var outputStr = Convert(output);
Assert.NotNull(outputStr);
}
}
public void SubmitHiveJob()
{
var parameters = new HiveJobSubmissionParameters
{
Query = @"select querydwelltime+2 from hivesampletable where clientid = 8"
};
SubmitHiveJobAndValidateOutput(parameters, "15.92");
}
public void SubmitHiveJob_Defines()
{
var defines = new Dictionary<string, string>
{
{ "hive.execution.engine", "ravi" },
{ "hive.exec.reducers.max", "1" },
{ "time", "10" },
{ "rows", "20" }
};
var parameters = new HiveJobSubmissionParameters
{
Query = @"select * from hivesampletable where querydwelltime > ${hiveconf:time} limit ${hiveconf:rows}",
Defines = defines
};
SubmitHiveJobAndValidateOutput(parameters, "Massachusetts United States");
}
public void SubmitHiveJobError()
{
using (var context = UndoContext.Current)
{
context.Start();
var username = TestUtils.UserName;
var password = TestUtils.Password;
var clustername = TestUtils.ClusterName;
var credentials = new BasicAuthenticationCloudCredentials
{
Username = username,
Password = password
};
var client = TestUtils.GetHDInsightJobManagementClient(clustername, credentials);
var parameters = new HiveJobSubmissionParameters
{
Query = "FAKEQUERY;",
StatusDir = "jobstatus"
};
var response = client.JobManagement.SubmitHiveJob(parameters);
Assert.NotNull(response);
Assert.Equal(response.StatusCode, HttpStatusCode.OK);
var jobId = response.JobSubmissionJsonResponse.Id;
Assert.Contains("job_", jobId, StringComparison.InvariantCulture);
var jobStatus = GetJobFinalStatus(client, jobId);
Assert.True(jobStatus.JobDetail.ExitValue > 0);
var storageAccess = GetStorageAccessObject();
if (HttpMockServer.Mode == HttpRecorderMode.Record)
{
var output = client.JobManagement.GetJobErrorLogs(jobId, storageAccess);
Assert.NotNull(output);
Assert.True(output.Length > 0);
string errorTextOutput = Convert(output);
Assert.True(!string.IsNullOrEmpty(errorTextOutput));
}
}
}
public void SubmitHiveJobAndValidateOutput(HiveJobSubmissionParameters parameters, string expectedOutputPart)
{
using (var context = UndoContext.Current)
{
context.Start();
var username = TestUtils.UserName;
var password = TestUtils.Password;
var clustername = TestUtils.ClusterName;
var credentials = new BasicAuthenticationCloudCredentials
{
Username = username,
Password = password
};
var client = TestUtils.GetHDInsightJobManagementClient(clustername, credentials);
var response = client.JobManagement.SubmitHiveJob(parameters);
Assert.NotNull(response);
Assert.Equal(response.StatusCode, HttpStatusCode.OK);
var jobId = response.JobSubmissionJsonResponse.Id;
Assert.Contains("job_", jobId, StringComparison.InvariantCulture);
var jobStatus = GetJobFinalStatus(client, jobId);
var storageAccess = GetStorageAccessObject();
if (jobStatus.JobDetail.ExitValue == 0)
{
if (HttpMockServer.Mode == HttpRecorderMode.Record)
{
// Retrieve Job Output
var output = client.JobManagement.GetJobOutput(jobId, storageAccess);
string textOutput = Convert(output);
Assert.True(textOutput.Contains(expectedOutputPart));
}
}
else
{
if (HttpMockServer.Mode == HttpRecorderMode.Record)
{
var output = client.JobManagement.GetJobErrorLogs(jobId, storageAccess);
string errorTextOutput = Convert(output);
Assert.NotNull(errorTextOutput);
}
Assert.True(false);
}
}
}
/// <summary>
/// Submits an Hive job to an HDINSIGHT cluster.
/// </summary>
/// <param name='operations'>
/// Reference to the
/// Microsoft.Azure.Management.HDInsight.Job.IJobOperations.
/// </param>
/// <param name='parameters'>
/// Required. Hive job parameters.
/// </param>
/// <returns>
/// The Create Job operation response.
/// </returns>
public static Task<JobSubmissionResponse> SubmitHiveJobAsync(this IJobOperations operations, HiveJobSubmissionParameters parameters)
{
return operations.SubmitHiveJobAsync(parameters, CancellationToken.None);
}
/// <summary>
/// Submits an Hive job to an HDINSIGHT cluster.
/// </summary>
/// <param name='operations'>
/// Reference to the
/// Microsoft.Azure.Management.HDInsight.Job.IJobOperations.
/// </param>
/// <param name='parameters'>
/// Required. Hive job parameters.
/// </param>
/// <returns>
/// The Create Job operation response.
/// </returns>
public static JobSubmissionResponse SubmitHiveJob(this IJobOperations operations, HiveJobSubmissionParameters parameters)
{
return Task.Factory.StartNew((object s) =>
{
return ((IJobOperations)s).SubmitHiveJobAsync(parameters);
}
, operations, CancellationToken.None, TaskCreationOptions.None, TaskScheduler.Default).Unwrap().GetAwaiter().GetResult();
}
/// <summary>
/// Submits a Hive job to an HDInsight cluster.
/// </summary>
/// <param name='operations'>
/// Reference to the
/// Microsoft.Azure.Management.HDInsight.Job.IJobOperations.
/// </param>
/// <param name='parameters'>
/// Required. Hive job parameters.
/// </param>
/// <returns>
/// The Create Job operation response.
/// </returns>
public static JobSubmissionResponse SubmitHiveJob(this IJobOperations operations, HiveJobSubmissionParameters parameters)
{
return operations.SubmitHiveJob(new JobSubmissionParameters { Content = parameters.GetJobPostRequestContent() });
}
public void SubmitHiveJobError()
{
using (var context = UndoContext.Current)
{
context.Start();
var client = TestUtils.GetHDInsightJobManagementClient();
var parameters = new HiveJobSubmissionParameters
{
Query = "FAKEQUERY;",
StatusDir = "jobstatus"
};
var response = client.JobManagement.SubmitHiveJob(parameters);
Assert.NotNull(response);
Assert.Equal(response.StatusCode, HttpStatusCode.OK);
var jobId = response.JobSubmissionJsonResponse.Id;
Assert.Contains("job_", jobId, StringComparison.InvariantCulture);
var jobStatus = client.JobManagement.WaitForJobCompletionAsync(jobId).Result;
Assert.True(jobStatus.JobDetail.ExitValue > 0);
var storageAccess = GetStorageAccessObject();
if (HttpMockServer.GetCurrentMode() == HttpRecorderMode.Record)
{
var output = client.JobManagement.GetJobErrorLogs(jobId, storageAccess);
Assert.NotNull(output);
Assert.True(output.Length > 0);
string errorTextOutput = Convert(output);
Assert.True(!string.IsNullOrEmpty(errorTextOutput));
}
}
}
public void SubmitHiveJobAndValidateOutput(HiveJobSubmissionParameters parameters, string expectedOutputPart, bool runAyncAPI = false, bool isWindowsCluster = false)
{
using (var context = UndoContext.Current)
{
context.Start();
var client = TestUtils.GetHDInsightJobManagementClient(isWindowsCluster);
var response = runAyncAPI ? client.JobManagement.SubmitHiveJobAsync(parameters).Result
: client.JobManagement.SubmitHiveJob(parameters);
Assert.NotNull(response);
Assert.Equal(response.StatusCode, HttpStatusCode.OK);
var jobId = response.JobSubmissionJsonResponse.Id;
Assert.Contains("job_", jobId, StringComparison.InvariantCulture);
var jobStatus = client.JobManagement.WaitForJobCompletion(jobId, TestUtils.JobWaitInterval, TestUtils.JobPollInterval);
var storageAccess = GetStorageAccessObject(isWindowsCluster);
if (jobStatus.JobDetail.ExitValue == 0)
{
if (HttpMockServer.GetCurrentMode() == HttpRecorderMode.Record)
{
// Retrieve Job Output
var output = client.JobManagement.GetJobOutput(jobId, storageAccess);
string textOutput = Convert(output);
Assert.True(textOutput.Contains(expectedOutputPart));
}
}
else
{
if (HttpMockServer.GetCurrentMode() == HttpRecorderMode.Record)
{
var output = client.JobManagement.GetJobErrorLogs(jobId, storageAccess);
string errorTextOutput = Convert(output);
Assert.NotNull(errorTextOutput);
}
Assert.True(false);
}
}
}
public void SubmitHiveJobAsync_Windows()
{
var parameters = new HiveJobSubmissionParameters
{
Query = @"select querydwelltime+2 from hivesampletable where clientid = 8"
};
SubmitHiveJobAndValidateOutput(parameters, "15.92", runAyncAPI: true, isWindowsCluster : true);
}
public void SubmitHiveJobLargeQuery()
{
var query = string.Empty;
// Maximum input string size Uri.EscapeDataString can accept is 65520.
while(query.Length < 65520)
{
query += "select 1.0000000001 + 0.00001 limit 1;";
}
var parameters = new HiveJobSubmissionParameters
{
Query = query
};
SubmitHiveJobAndValidateOutput(parameters, "1.0000100001");
}
public void SubmitHiveJob()
{
using (var context = UndoContext.Current)
{
context.Start();
var username = TestUtils.UserName;
var password = TestUtils.Password;
var clustername = TestUtils.ClusterName;
var credentials = new BasicAuthenticationCloudCredentials
{
Username = username,
Password = password
};
var client = TestUtils.GetHDInsightJobManagementClient(clustername, credentials);
var defines = new Dictionary<string, string>
{
{"hive.execution.engine", "ravi"},
{"hive.exec.reducers.max", "1"}
};
var args = new List<string> {"argA", "argB"};
var parameters = new HiveJobSubmissionParameters
{
UserName = username,
Query = "SHOW TABLES;",
Defines = ConvertDefinesToString(defines),
Arguments = ConvertArgsToString(args)
};
var response = client.JobManagement.SubmitHiveJob(parameters);
Assert.NotNull(response);
Assert.Equal(response.StatusCode, HttpStatusCode.OK);
Assert.Contains("job_", response.JobSubmissionJsonResponse.Id, StringComparison.InvariantCulture);
}
}