public virtual void NewHiveJob_StartJob_GetJob()
{
var hiveJobDefinition = new HiveJobCreateParameters {
JobName = "show tables", Query = "show tables"
};
using (IRunspace runspace = this.GetPowerShellRunspace())
{
ClusterDetails testCluster = GetHttpAccessEnabledCluster();
IPipelineResult results =
runspace.NewPipeline()
.AddCommand(CmdletConstants.NewAzureHDInsightHiveJobDefinition)
.WithParameter(CmdletConstants.JobName, hiveJobDefinition.JobName)
.WithParameter(CmdletConstants.Query, hiveJobDefinition.Query)
.AddCommand(CmdletConstants.StartAzureHDInsightJob)
.WithParameter(CmdletConstants.Cluster, testCluster.Name)
.AddCommand(CmdletConstants.WaitAzureHDInsightJob)
.WithParameter(CmdletConstants.WaitTimeoutInSeconds, 10)
.AddCommand(CmdletConstants.GetAzureHDInsightJobOutput)
.WithParameter(CmdletConstants.Cluster, testCluster.Name)
.Invoke();
Assert.IsNotNull(results.Results.ToEnumerable <string>());
Assert.IsTrue(results.Results.ToEnumerable <string>().Any(str => str == "hivesampletable"));
}
}
//Run Hive Job
public static void DoHiveOperations()
{
HiveJobCreateParameters hiveJobDefinition = new HiveJobCreateParameters()
{
JobName = "Show tables job",
StatusFolder = "/TableListFolder",
Query = "show tables;"
};
var store = new X509Store();
store.Open(OpenFlags.ReadOnly);
var cert = store.Certificates.Cast <X509Certificate2>().First(item => item.Thumbprint == Constants.thumbprint);
var creds = new JobSubmissionCertificateCredential(Constants.subscriptionId, cert, Constants.clusterName);
var jobClient = JobSubmissionClientFactory.Connect(creds);
JobCreationResults jobResults = jobClient.CreateHiveJob(hiveJobDefinition);
Console.Write("Executing Hive Job.");
// Wait for the job to complete
WaitForJobCompletion(jobResults, jobClient);
// Print the Hive job output
System.IO.Stream stream = jobClient.GetJobOutput(jobResults.JobId);
System.IO.StreamReader reader = new System.IO.StreamReader(stream);
Console.Write("Done..List of Tables are:\n");
Console.WriteLine(reader.ReadToEnd());
}
public void ICanCallThe_New_HDInsightHiveJobDefinitionCmdlet_WithParameters()
{
var HiveJobDefinition = new HiveJobCreateParameters {
JobName = "show tables jobDetails", Query = "show tables"
};
HiveJobDefinition.Defines.Add("map.input.tasks", "1000");
HiveJobDefinition.Defines.Add("map.input.reducers", "1000");
using (IRunspace runspace = this.GetPowerShellRunspace())
{
IPipelineResult results =
runspace.NewPipeline()
.AddCommand(CmdletConstants.NewAzureHDInsightHiveJobDefinition)
.WithParameter(CmdletConstants.JobName, HiveJobDefinition.JobName)
.WithParameter(CmdletConstants.Query, HiveJobDefinition.Query)
.WithParameter(CmdletConstants.Parameters, HiveJobDefinition.Defines)
.Invoke();
Assert.AreEqual(1, results.Results.Count);
AzureHDInsightHiveJobDefinition HiveJobFromPowershell = results.Results.ToEnumerable <AzureHDInsightHiveJobDefinition>().First();
Assert.AreEqual(HiveJobDefinition.JobName, HiveJobFromPowershell.JobName);
Assert.AreEqual(HiveJobDefinition.Query, HiveJobFromPowershell.Query);
foreach (var parameter in HiveJobDefinition.Defines)
{
Assert.IsTrue(
HiveJobFromPowershell.Defines.Any(arg => string.Equals(parameter.Key, arg.Key) && string.Equals(parameter.Value, arg.Value)),
"Unable to find parameter '{0}' in value returned from powershell",
parameter.Key);
}
}
}
public void ICanCallThe_New_HDInsightHiveJobDefinitionCmdlet_WithArguments()
{
var HiveJobDefinition = new HiveJobCreateParameters { JobName = "show tables jobDetails", Query = "show tables" };
HiveJobDefinition.Arguments.Add("arg 1");
HiveJobDefinition.Arguments.Add("arg 2");
using (IRunspace runspace = this.GetPowerShellRunspace())
{
IPipelineResult results =
runspace.NewPipeline()
.AddCommand(CmdletConstants.NewAzureHDInsightHiveJobDefinition)
.WithParameter(CmdletConstants.JobName, HiveJobDefinition.JobName)
.WithParameter(CmdletConstants.Query, HiveJobDefinition.Query)
.WithParameter(CmdletConstants.HiveArgs, HiveJobDefinition.Arguments)
.Invoke();
Assert.AreEqual(1, results.Results.Count);
AzureHDInsightHiveJobDefinition HiveJobFromPowershell = results.Results.ToEnumerable<AzureHDInsightHiveJobDefinition>().First();
Assert.AreEqual(HiveJobDefinition.JobName, HiveJobFromPowershell.JobName);
Assert.AreEqual(HiveJobDefinition.Query, HiveJobFromPowershell.Query);
foreach (string args in HiveJobDefinition.Arguments)
{
Assert.IsTrue(
HiveJobFromPowershell.Arguments.Any(arg => string.Equals(args, arg)),
"Unable to find argument '{0}' in value returned from powershell",
args);
}
}
}
public virtual void WaitForJobWithId()
{
var hiveJobDefinition = new HiveJobCreateParameters()
{
JobName = "show tables jobDetails",
Query = "show tables"
};
var cluster = CmdletScenariosTestCaseBase.GetHttpAccessEnabledCluster();
using (var runspace = this.GetPowerShellRunspace())
{
var results = runspace.NewPipeline()
.AddCommand(CmdletConstants.NewAzureHDInsightHiveJobDefinition)
.WithParameter(CmdletConstants.JobName, hiveJobDefinition.JobName)
.WithParameter(CmdletConstants.Query, hiveJobDefinition.Query)
.AddCommand(CmdletConstants.StartAzureHDInsightJob)
.WithParameter(CmdletConstants.Cluster, cluster.ConnectionUrl)
.WithParameter(CmdletConstants.Credential, IntegrationTestBase.GetPSCredential(cluster.HttpUserName, cluster.HttpPassword))
.Invoke();
Assert.AreEqual(1, results.Results.Count);
var job = results.Results.First().BaseObject as Microsoft.WindowsAzure.Management.HDInsight.Cmdlet.DataObjects.AzureHDInsightJob;
results = runspace.NewPipeline()
.AddCommand(CmdletConstants.WaitAzureHDInsightJob)
.WithParameter(CmdletConstants.Credential, IntegrationTestBase.GetPSCredential(cluster.HttpUserName, cluster.HttpPassword))
.WithParameter(CmdletConstants.JobId, job.JobId)
.WithParameter(CmdletConstants.Cluster, job.Cluster)
.Invoke();
var completedJob = results.Results.ToEnumerable<AzureHDInsightJob>().FirstOrDefault();
Assert.IsNotNull(completedJob);
Assert.AreEqual(job.JobId, completedJob.JobId);
Assert.AreEqual("Completed", completedJob.State);
}
}
public Task <JobCreationResults> CreateHiveJobAsync(HiveJobCreateParameters hiveJobCreateParameters)
{
if (hiveJobCreateParameters.Query.IsNullOrEmpty())
{
hiveJobCreateParameters.File.ArgumentNotNullOrEmpty("File");
if (hiveJobCreateParameters.File.Contains("://") &&
!hiveJobCreateParameters.File.StartsWith("wasb", StringComparison.OrdinalIgnoreCase))
{
throw new InvalidOperationException("Invalid file protocol : " + hiveJobCreateParameters.File);
}
}
else
{
this.PrepareQueryJob(hiveJobCreateParameters);
}
JobCreationResults retval =
this.CreateJobSuccessResult(
new JobDetails
{
Name = hiveJobCreateParameters.JobName,
Query = hiveJobCreateParameters.Query,
StatusDirectory = hiveJobCreateParameters.StatusFolder
},
hiveJobCreateParameters.JobName);
return(TaskEx2.FromResult(retval));
}
public void CannotCreateNewHiveJob_WithRestrictedCharacters_StartJob()
{
var hiveJobDefinition = new HiveJobCreateParameters {
JobName = "show tables jobDetails", Query = "show tables %"
};
INewAzureHDInsightHiveJobDefinitionCommand newHiveJobDefinitionCommand =
ServiceLocator.Instance.Locate <IAzureHDInsightCommandFactory>().CreateNewHiveDefinition();
newHiveJobDefinitionCommand.JobName = hiveJobDefinition.JobName;
newHiveJobDefinitionCommand.Query = hiveJobDefinition.Query;
newHiveJobDefinitionCommand.EndProcessing();
AzureHDInsightHiveJobDefinition hiveJobFromCommand = newHiveJobDefinitionCommand.Output.ElementAt(0);
try
{
TestJobStart(hiveJobFromCommand);
Assert.Fail();
}
catch (AggregateException aggregateException)
{
var invalidOperationException = aggregateException.GetBaseException() as InvalidOperationException;
Assert.IsNotNull(invalidOperationException);
Assert.IsTrue(invalidOperationException.Message.Contains("Query contains restricted character :'%'"), "Exception not thrown for special character");
}
}
public void ICannotCallThe_New_HDInsightHiveJobDefinitionCmdlet_WithoutFileOrQueryParameter()
{
var HiveJobDefinition = new HiveJobCreateParameters
{
JobName = "show tables jobDetails",
File = TestConstants.WabsProtocolSchemeName + "filepath.hql"
};
try
{
using (IRunspace runspace = this.GetPowerShellRunspace())
{
runspace.NewPipeline()
.AddCommand(CmdletConstants.NewAzureHDInsightHiveJobDefinition)
.WithParameter(CmdletConstants.JobName, HiveJobDefinition.JobName)
.Invoke();
Assert.Fail("test failed.");
}
}
catch (CmdletInvocationException invokeException)
{
var psArgumentException = invokeException.GetBaseException() as PSArgumentException;
Assert.IsNotNull(psArgumentException);
Assert.AreEqual("Either File or Query should be specified for Hive jobs.", psArgumentException.Message);
}
}
public void ICanCallThe_New_HDInsightHiveJobDefinitionCmdlet_WithArguments()
{
var HiveJobDefinition = new HiveJobCreateParameters {
JobName = "show tables jobDetails", Query = "show tables"
};
HiveJobDefinition.Arguments.Add("arg 1");
HiveJobDefinition.Arguments.Add("arg 2");
using (IRunspace runspace = this.GetPowerShellRunspace())
{
IPipelineResult results =
runspace.NewPipeline()
.AddCommand(CmdletConstants.NewAzureHDInsightHiveJobDefinition)
.WithParameter(CmdletConstants.JobName, HiveJobDefinition.JobName)
.WithParameter(CmdletConstants.Query, HiveJobDefinition.Query)
.WithParameter(CmdletConstants.HiveArgs, HiveJobDefinition.Arguments)
.Invoke();
Assert.AreEqual(1, results.Results.Count);
AzureHDInsightHiveJobDefinition HiveJobFromPowershell = results.Results.ToEnumerable <AzureHDInsightHiveJobDefinition>().First();
Assert.AreEqual(HiveJobDefinition.JobName, HiveJobFromPowershell.JobName);
Assert.AreEqual(HiveJobDefinition.Query, HiveJobFromPowershell.Query);
foreach (string args in HiveJobDefinition.Arguments)
{
Assert.IsTrue(
HiveJobFromPowershell.Arguments.Any(arg => string.Equals(args, arg)),
"Unable to find argument '{0}' in value returned from powershell",
args);
}
}
}
public void schedule_hive_job()
{
var client = new WebHCatalogJobClient(Connect.WithTestUser(to: WebHcatBase));
var job = new HiveJobCreateParameters
{
StatusFolder = "test.output",
JobName = "test-hive-query",
Query = @"
SELECT s07.description, s07.total_emp, s08.total_emp, s07.salary
FROM
sample_07 s07 JOIN
sample_08 s08
ON ( s07.code = s08.code )
WHERE
( s07.total_emp > s08.total_emp
AND s07.salary > 100000 )
SORT BY s07.salary DESC"
};
var result = client.SubmitHiveJob(job).Result;
result.JobId.Should().NotBeNullOrEmpty();
Console.WriteLine(result.JobId);
}
public virtual void WaitForJobWithId()
{
var hiveJobDefinition = new HiveJobCreateParameters()
{
JobName = "show tables jobDetails",
Query = "show tables"
};
var cluster = CmdletScenariosTestCaseBase.GetHttpAccessEnabledCluster();
using (var runspace = this.GetPowerShellRunspace())
{
var results = runspace.NewPipeline()
.AddCommand(CmdletConstants.NewAzureHDInsightHiveJobDefinition)
.WithParameter(CmdletConstants.JobName, hiveJobDefinition.JobName)
.WithParameter(CmdletConstants.Query, hiveJobDefinition.Query)
.AddCommand(CmdletConstants.StartAzureHDInsightJob)
.WithParameter(CmdletConstants.Cluster, cluster.ConnectionUrl)
.WithParameter(CmdletConstants.Credential, IntegrationTestBase.GetPSCredential(cluster.HttpUserName, cluster.HttpPassword))
.Invoke();
Assert.AreEqual(1, results.Results.Count);
var job = results.Results.First().BaseObject as Microsoft.WindowsAzure.Management.HDInsight.Cmdlet.DataObjects.AzureHDInsightJob;
results = runspace.NewPipeline()
.AddCommand(CmdletConstants.WaitAzureHDInsightJob)
.WithParameter(CmdletConstants.Credential, IntegrationTestBase.GetPSCredential(cluster.HttpUserName, cluster.HttpPassword))
.WithParameter(CmdletConstants.JobId, job.JobId)
.WithParameter(CmdletConstants.Cluster, job.Cluster)
.Invoke();
var completedJob = results.Results.ToEnumerable <AzureHDInsightJob>().FirstOrDefault();
Assert.IsNotNull(completedJob);
Assert.AreEqual(job.JobId, completedJob.JobId);
Assert.AreEqual("Completed", completedJob.State);
}
}
public virtual void WaitForJob()
{
var hiveJobDefinition = new HiveJobCreateParameters {
JobName = "show tables jobDetails", Query = "show tables"
};
// IHadoopClientExtensions.GetPollingInterval = () => 0;
ClusterDetails cluster = CmdletScenariosTestCaseBase.GetHttpAccessEnabledCluster();
using (IRunspace runspace = this.GetPowerShellRunspace())
{
IPipelineResult results =
runspace.NewPipeline()
.AddCommand(CmdletConstants.NewAzureHDInsightHiveJobDefinition)
.WithParameter(CmdletConstants.JobName, hiveJobDefinition.JobName)
.WithParameter(CmdletConstants.Query, hiveJobDefinition.Query)
.AddCommand(CmdletConstants.StartAzureHDInsightJob)
.WithParameter(CmdletConstants.Cluster, cluster.ConnectionUrl)
.WithParameter(CmdletConstants.Credential, GetPSCredential(cluster.HttpUserName, cluster.HttpPassword))
.AddCommand(CmdletConstants.WaitAzureHDInsightJob)
.WithParameter(CmdletConstants.Credential, GetPSCredential(cluster.HttpUserName, cluster.HttpPassword))
.Invoke();
Assert.AreEqual(1, results.Results.Count);
Assert.AreEqual("Completed", results.Results.ToEnumerable <AzureHDInsightJob>().First().State);
}
}
public void CannotCreateNewHiveJob_WithRestrictedCharacters_StartJob()
{
var hiveJobDefinition = new HiveJobCreateParameters {
JobName = "show tables jobDetails", Query = "show tables %"
};
INewAzureHDInsightHiveJobDefinitionCommand newHiveJobDefinitionCommand =
ServiceLocator.Instance.Locate <IAzureHDInsightCommandFactory>().CreateNewHiveDefinition();
newHiveJobDefinitionCommand.JobName = hiveJobDefinition.JobName;
newHiveJobDefinitionCommand.Query = hiveJobDefinition.Query;
newHiveJobDefinitionCommand.EndProcessing();
AzureHDInsightHiveJobDefinition hiveJobFromCommand = newHiveJobDefinitionCommand.Output.ElementAt(0);
try
{
TestJobStart(hiveJobFromCommand);
Assert.Fail();
}
catch (AggregateException aggregateException)
{
var invalidOperationException = aggregateException.GetBaseException() as InvalidOperationException;
Assert.IsNotNull(invalidOperationException);
Assert.AreEqual("Query text contains restricted character '%', please upload the query to a file in storage and re-submit the job using the -File parameter",
invalidOperationException.Message);
}
}
/// <summary>
/// Performs HQL query and returns the query results.
/// </summary>
/// <param name="jobParams">The query parameters.</param>
/// <returns>The query result.</returns>
public string Query(HiveJobCreateParameters jobParams)
{
// Assign status folder
jobParams.StatusFolder = RootDirectory + "/status";
JobCreationResults jobDetails = null;
try
{
// Create Hive job
jobDetails = this.job.CreateHiveJob(jobParams);
}
catch (Exception e)
{
AvroHdiSample.ReportError("Error while creating a Hive job\n" + e);
}
JobDetails jobInProgress = null;
try
{
// Get job status
jobInProgress = this.job.GetJob(jobDetails.JobId);
}
catch (Exception e)
{
AvroHdiSample.ReportError("Error while getting Hive job status\n" + e);
}
// If job is not finished then sleep until the next client polling interval
while (jobInProgress.StatusCode != JobStatusCode.Completed &&
jobInProgress.StatusCode != JobStatusCode.Failed)
{
try
{
// Get job status
jobInProgress = this.job.GetJob(jobDetails.JobId);
}
catch (Exception e)
{
AvroHdiSample.ReportError("Error while getting Hive job status\n" + e);
}
Thread.Sleep(this.client.PollingInterval);
}
try
{
// Job is finished; get its output stream, read it, and return the value
return(new StreamReader(this.job.GetJobOutput(jobDetails.JobId)).ReadToEnd());
}
catch (Exception e)
{
AvroHdiSample.ReportError("Error while reading Hibe job result\n" + e);
}
return(string.Empty);
}
public void GivenIHaveAHiveJobRequestObject()
{
var request = new HiveJobCreateParameters();
request.JobName = string.Empty;
request.Query = string.Empty;
request.StatusFolder = string.Empty;
this.transferObject = request;
}
public Task <JobCreationResults> SubmitHiveJob(HiveJobCreateParameters details)
{
this.SubmitHiveJobCalled = true;
var job = new JobCreationResults()
{
JobId = JobId
};
return(Task.Run(() => job));
}
public void CanSerializeValidHiveJobRequest()
{
var hiveJob = new HiveJobCreateParameters()
{
Query = "show tables"
};
var payloadConverter = new PayloadConverterBase();
var payload = payloadConverter.SerializeHiveRequest("hadoop", hiveJob);
Assert.IsTrue(payload.Contains(string.Format("{0}={1}", WebHCatConstants.Execute, Uri.EscapeDataString(hiveJob.Query))));
}
public void DoesNotAddDefineIfJobNameAbsent()
{
var hiveJob = new HiveJobCreateParameters()
{
Query = "show tables"
};
var payloadConverter = new PayloadConverterBase();
var payload = payloadConverter.SerializeHiveRequest("hadoop", hiveJob);
Assert.IsFalse(payload.Contains(Uri.EscapeDataString(string.Format("{0}={1}", WebHCatConstants.DefineJobName, hiveJob.JobName))));
}
public void PayloadHasEnableLogsFalse()
{
var hiveJob = new HiveJobCreateParameters()
{
Query = "show tables"
};
var payloadConverter = new PayloadConverterBase();
var payload = payloadConverter.SerializeHiveRequest("hadoop", hiveJob);
Assert.IsTrue(payload.Contains(string.Format("{0}={1}", HadoopRemoteRestConstants.EnableLogging, "false")));
}
public void CanSerializeValidHiveJobRequest_WithFile()
{
var hiveJob = new HiveJobCreateParameters()
{
File = Constants.WabsProtocolSchemeName + "filepath.hql"
};
var payloadConverter = new PayloadConverterBase();
var payload = payloadConverter.SerializeHiveRequest("hadoop", hiveJob);
Assert.IsTrue(payload.Contains(string.Format("{0}={1}", WebHCatConstants.File, Uri.EscapeDataString(hiveJob.File))));
}
/// <inheritdoc />
public async Task <JobCreationResults> SubmitHiveJob(HiveJobCreateParameters details)
{
//NEIN: Any code modification here should add unit tests for this class
var converter = new PayloadConverter();
var payload = converter.SerializeHiveRequest(this.credentials.UserName, details);
var client = ServiceLocator.Instance.Locate <IHadoopRemoteJobSubmissionRestClientFactory>().Create(this.credentials, this.context, this.ignoreSslErrors, this.GetUserAgentString());
var result = await client.SubmitHiveJob(payload);
return(new JobCreationResults()
{
JobId = converter.DeserializeJobSubmissionResponse(result.Content)
});
}
public void CanCreateNewHiveJob_StartJob()
{
var hiveJobDefinition = new HiveJobCreateParameters { JobName = "show tables jobDetails", Query = "show tables" };
INewAzureHDInsightHiveJobDefinitionCommand newHiveJobDefinitionCommand =
ServiceLocator.Instance.Locate<IAzureHDInsightCommandFactory>().CreateNewHiveDefinition();
newHiveJobDefinitionCommand.JobName = hiveJobDefinition.JobName;
newHiveJobDefinitionCommand.Query = hiveJobDefinition.Query;
newHiveJobDefinitionCommand.EndProcessing();
AzureHDInsightHiveJobDefinition hiveJobFromCommand = newHiveJobDefinitionCommand.Output.ElementAt(0);
TestJobStart(hiveJobFromCommand);
}
private async Task <JobCreationResults> ScheduleNewJob(string hiveQuery, string path, Guid jobIdentifier)
{
var jobParams = new HiveJobCreateParameters
{
StatusFolder = path,
JobName = jobIdentifier.ToString(),
Query = hiveQuery
};
var creationResult = await _jobClient.SubmitHiveJob(jobParams);
return(creationResult);
}
public async Task<JobCreationResults> SubmitHiveJob(HiveJobCreateParameters details)
{
var relative = new Uri(
HadoopRemoteRestConstants.Hive + "?" +
HadoopRemoteRestConstants.UserName + "=" +
_connectionConfig.UserName.EscapeDataString(),
UriKind.Relative);
var requestContent = _converter.SerializeHiveRequest(_connectionConfig.UserName, details);
var result = await MakeAsyncJobSubmissionRequest(relative, requestContent);
return new JobCreationResults { JobId = _converter.DeserializeJobSubmissionResponse(result) };
}
private static JobCreationResults RunHiveJob(HiveJobCreateParameters job)
{
JobCreationResults jobDetails = jobSubmissionClient.CreateHiveJob(job);
JobDetails jobInProgress = jobSubmissionClient.GetJob(jobDetails.JobId);
while (jobInProgress.StatusCode != JobStatusCode.Completed && jobInProgress.StatusCode != JobStatusCode.Failed)
{
jobInProgress = jobSubmissionClient.GetJob(jobInProgress.JobId);
Thread.Sleep(TimeSpan.FromMilliseconds(IHadoopClientExtensions.GetPollingInterval()));
}
Assert.IsNull(jobDetails.ErrorCode, "Should not fail hive jobDetails submission");
Assert.IsNotNull(jobDetails.JobId, "Should have a non-null jobDetails id");
return(jobDetails);
}
public void JobCanSetEnableLogsTrue()
{
var hiveJob = new HiveJobCreateParameters()
{
Query = "show tables",
EnableTaskLogs = true,
StatusFolder = "/showtableslocation"
};
var payloadConverter = new PayloadConverterBase();
var payload = payloadConverter.SerializeHiveRequest("hadoop", hiveJob);
Assert.IsTrue(payload.Contains(string.Format("{0}={1}", HadoopRemoteRestConstants.EnableLogging, "true")));
}
/// <inheritdoc/>
public async Task <JobCreationResults> CreateHiveJobAsync(HiveJobCreateParameters hiveJobCreateParameters)
{
hiveJobCreateParameters = this.PrepareQueryJob(hiveJobCreateParameters);
var pocoClient = this.GetPocoClient();
try
{
return(await pocoClient.SubmitHiveJob(hiveJobCreateParameters));
}
catch (UnauthorizedAccessException)
{
pocoClient = this.GetPocoClient(true);
}
return(await pocoClient.SubmitHiveJob(hiveJobCreateParameters));
}
public async Task <JobCreationResults> SubmitHiveJob(HiveJobCreateParameters details)
{
var relative = new Uri(
HadoopRemoteRestConstants.Hive + "?" +
HadoopRemoteRestConstants.UserName + "=" +
_connectionConfig.UserName.EscapeDataString(),
UriKind.Relative);
var requestContent = _converter.SerializeHiveRequest(_connectionConfig.UserName, details);
var result = await MakeAsyncJobSubmissionRequest(relative, requestContent);
return(new JobCreationResults {
JobId = _converter.DeserializeJobSubmissionResponse(result)
});
}
public void CanCreateNewHiveJob_StartJob()
{
var hiveJobDefinition = new HiveJobCreateParameters {
JobName = "show tables jobDetails", Query = "show tables"
};
INewAzureHDInsightHiveJobDefinitionCommand newHiveJobDefinitionCommand =
ServiceLocator.Instance.Locate <IAzureHDInsightCommandFactory>().CreateNewHiveDefinition();
newHiveJobDefinitionCommand.JobName = hiveJobDefinition.JobName;
newHiveJobDefinitionCommand.Query = hiveJobDefinition.Query;
newHiveJobDefinitionCommand.EndProcessing();
AzureHDInsightHiveJobDefinition hiveJobFromCommand = newHiveJobDefinitionCommand.Output.ElementAt(0);
TestJobStart(hiveJobFromCommand);
}
public HiveJobCreateParameters DeserializeHiveJobCreationDetails(string content)
{
var request = this.DeserializePayload(content);
var result = new HiveJobCreateParameters()
{
JobName = request.JobName,
StatusFolder = request.OutputStorageLocation,
Query = request.Query
};
foreach (var jobRequestParameter in request.Parameters)
{
result.Defines.Add(jobRequestParameter.Key, jobRequestParameter.Value.ToString());
}
this.SetStandardProperties(request, result);
return(result);
}
public void ICanCallThe_New_HDInsightHiveJobDefinitionCmdlet_WithoutJobName()
{
var HiveJobDefinition = new HiveJobCreateParameters {
Query = "show tables"
};
using (IRunspace runspace = this.GetPowerShellRunspace())
{
IPipelineResult results =
runspace.NewPipeline()
.AddCommand(CmdletConstants.NewAzureHDInsightHiveJobDefinition)
.WithParameter(CmdletConstants.Query, HiveJobDefinition.Query)
.Invoke();
Assert.AreEqual(1, results.Results.Count);
AzureHDInsightHiveJobDefinition HiveJobFromPowershell = results.Results.ToEnumerable <AzureHDInsightHiveJobDefinition>().First();
Assert.AreEqual(HiveJobDefinition.Query, HiveJobFromPowershell.Query);
}
}
public virtual void ICanCallThe_NewHiveJob_Then_Start_HDInsightJobsCmdlet()
{
var hiveJobDefinition = new HiveJobCreateParameters { JobName = "show tables jobDetails", Query = "show tables" };
using (IRunspace runspace = this.GetPowerShellRunspace())
{
IPipelineResult results =
runspace.NewPipeline()
.AddCommand(CmdletConstants.NewAzureHDInsightHiveJobDefinition)
.WithParameter(CmdletConstants.JobName, hiveJobDefinition.JobName)
.WithParameter(CmdletConstants.Query, hiveJobDefinition.Query)
.Invoke();
Assert.AreEqual(1, results.Results.Count);
AzureHDInsightHiveJobDefinition hiveJobFromPowershell = results.Results.ToEnumerable<AzureHDInsightHiveJobDefinition>().First();
ClusterDetails testCluster = CmdletScenariosTestCaseBase.GetHttpAccessEnabledCluster();
AzureHDInsightJob jobCreationDetails = RunJobInPowershell(runspace, hiveJobFromPowershell, testCluster);
AzureHDInsightJob jobHistoryResult = GetJobsCmdletTests.GetJobWithID(runspace, jobCreationDetails.JobId, testCluster);
}
}
public void ICanCallThe_New_HDInsightHiveJobDefinitionCmdlet()
{
var HiveJobDefinition = new HiveJobCreateParameters { JobName = "show tables jobDetails", Query = "show tables" };
using (IRunspace runspace = this.GetPowerShellRunspace())
{
IPipelineResult results =
runspace.NewPipeline()
.AddCommand(CmdletConstants.NewAzureHDInsightHiveJobDefinition)
.WithParameter(CmdletConstants.JobName, HiveJobDefinition.JobName)
.WithParameter(CmdletConstants.Query, HiveJobDefinition.Query)
.Invoke();
Assert.AreEqual(1, results.Results.Count);
AzureHDInsightHiveJobDefinition HiveJobFromPowershell = results.Results.ToEnumerable<AzureHDInsightHiveJobDefinition>().First();
Assert.AreEqual(HiveJobDefinition.JobName, HiveJobFromPowershell.JobName);
Assert.AreEqual(HiveJobDefinition.Query, HiveJobFromPowershell.Query);
}
}
private static async Task <JobCreationResults> SubmitHiveJob(
AzureHDInsightHiveJobDefinition azureHiveJobDefinition, IJobSubmissionClient client)
{
AssertQueryDoesNotContainRestrictedCharacters(azureHiveJobDefinition.Query, "Query");
var hiveJobDefinition = new HiveJobCreateParameters
{
JobName = azureHiveJobDefinition.JobName,
Query = azureHiveJobDefinition.Query,
File = azureHiveJobDefinition.File
};
hiveJobDefinition.StatusFolder = azureHiveJobDefinition.StatusFolder;
hiveJobDefinition.Arguments.AddRange(azureHiveJobDefinition.Arguments);
hiveJobDefinition.Defines.AddRange(azureHiveJobDefinition.Defines);
hiveJobDefinition.Files.AddRange(azureHiveJobDefinition.Files);
var jobCreationResults = await client.CreateHiveJobAsync(hiveJobDefinition);
return(jobCreationResults);
}
private static async Task <JobCreationResults> SubmitHiveJob(
AzureHDInsightHiveJobDefinition azureHiveJobDefinition, IJobSubmissionClient client)
{
var hiveJobDefinition = new HiveJobCreateParameters
{
JobName = azureHiveJobDefinition.JobName,
Query = azureHiveJobDefinition.Query,
File = azureHiveJobDefinition.File,
RunAsFileJob = azureHiveJobDefinition.RunAsFileJob
};
hiveJobDefinition.StatusFolder = azureHiveJobDefinition.StatusFolder;
hiveJobDefinition.Arguments.AddRange(azureHiveJobDefinition.Arguments);
hiveJobDefinition.Defines.AddRange(azureHiveJobDefinition.Defines);
hiveJobDefinition.Files.AddRange(azureHiveJobDefinition.Files);
var jobCreationResults = await client.CreateHiveJobAsync(hiveJobDefinition);
return(jobCreationResults);
}
public virtual void ICanCallThe_NewHiveJob_Then_Start_HDInsightJobsCmdlet()
{
var hiveJobDefinition = new HiveJobCreateParameters {
JobName = "show tables jobDetails", Query = "show tables"
};
using (IRunspace runspace = this.GetPowerShellRunspace())
{
IPipelineResult results =
runspace.NewPipeline()
.AddCommand(CmdletConstants.NewAzureHDInsightHiveJobDefinition)
.WithParameter(CmdletConstants.JobName, hiveJobDefinition.JobName)
.WithParameter(CmdletConstants.Query, hiveJobDefinition.Query)
.Invoke();
Assert.AreEqual(1, results.Results.Count);
AzureHDInsightHiveJobDefinition hiveJobFromPowershell = results.Results.ToEnumerable <AzureHDInsightHiveJobDefinition>().First();
ClusterDetails testCluster = CmdletScenariosTestCaseBase.GetHttpAccessEnabledCluster();
AzureHDInsightJob jobCreationDetails = RunJobInPowershell(runspace, hiveJobFromPowershell, testCluster);
AzureHDInsightJob jobHistoryResult = GetJobsCmdletTests.GetJobWithID(runspace, jobCreationDetails.JobId, testCluster);
}
}
public Task<JobCreationResults> CreateHiveJobAsync(HiveJobCreateParameters hiveJobCreateParameters)
{
if (hiveJobCreateParameters.Query.IsNullOrEmpty())
{
hiveJobCreateParameters.File.ArgumentNotNullOrEmpty("File");
if (hiveJobCreateParameters.File.Contains("://") &&
!hiveJobCreateParameters.File.StartsWith("wasb", StringComparison.OrdinalIgnoreCase))
{
throw new InvalidOperationException("Invalid file protocol : " + hiveJobCreateParameters.File);
}
}
JobCreationResults retval =
this.CreateJobSuccessResult(
new JobDetails
{
Name = hiveJobCreateParameters.JobName,
Query = hiveJobCreateParameters.Query,
StatusDirectory = hiveJobCreateParameters.StatusFolder
},
hiveJobCreateParameters.JobName);
return TaskEx2.FromResult(retval);
}
public void CannotCreateNewHiveJob_WithRestrictedCharacters_StartJob()
{
var hiveJobDefinition = new HiveJobCreateParameters { JobName = "show tables jobDetails", Query = "show tables %" };
INewAzureHDInsightHiveJobDefinitionCommand newHiveJobDefinitionCommand =
ServiceLocator.Instance.Locate<IAzureHDInsightCommandFactory>().CreateNewHiveDefinition();
newHiveJobDefinitionCommand.JobName = hiveJobDefinition.JobName;
newHiveJobDefinitionCommand.Query = hiveJobDefinition.Query;
newHiveJobDefinitionCommand.EndProcessing();
AzureHDInsightHiveJobDefinition hiveJobFromCommand = newHiveJobDefinitionCommand.Output.ElementAt(0);
try
{
TestJobStart(hiveJobFromCommand);
Assert.Fail();
}
catch (AggregateException aggregateException)
{
var invalidOperationException = aggregateException.GetBaseException() as InvalidOperationException;
Assert.IsNotNull(invalidOperationException);
Assert.AreEqual("Query text contains restricted character '%', please upload the query to a file in storage and re-submit the job using the -File parameter",
invalidOperationException.Message);
}
}
/// <inheritdoc />
public async Task<JobCreationResults> SubmitHiveJob(HiveJobCreateParameters details)
{
//NEIN: Any code modification here should add unit tests for this class
var converter = new PayloadConverter();
var payload = converter.SerializeHiveRequest(this.credentials.UserName, details);
var client = ServiceLocator.Instance.Locate<IHadoopRemoteJobSubmissionRestClientFactory>().Create(this.credentials, this.context, this.ignoreSslErrors, this.GetUserAgentString());
var result = await client.SubmitHiveJob(payload);
return new JobCreationResults() { JobId = converter.DeserializeJobSubmissionResponse(result.Content) };
}
/// <summary>
/// Creates the payload for a Hive request.
/// </summary>
/// <param name="userName"> The user name.</param>
/// <param name="details"> The details.</param>
/// <returns>A string that represents the payload for the request.</returns>
public string SerializeHiveRequest(string userName, HiveJobCreateParameters details)
{
details.ArgumentNotNull("details");
return this.SerializeQueryRequest(userName, details, details.JobName, details.File, details.Query, WebHCatConstants.Execute, details.Arguments, details.Defines);
}
public virtual void WaitForJob()
{
var hiveJobDefinition = new HiveJobCreateParameters { JobName = "show tables jobDetails", Query = "show tables" };
// IHadoopClientExtensions.GetPollingInterval = () => 0;
ClusterDetails cluster = CmdletScenariosTestCaseBase.GetHttpAccessEnabledCluster();
using (IRunspace runspace = this.GetPowerShellRunspace())
{
IPipelineResult results =
runspace.NewPipeline()
.AddCommand(CmdletConstants.NewAzureHDInsightHiveJobDefinition)
.WithParameter(CmdletConstants.JobName, hiveJobDefinition.JobName)
.WithParameter(CmdletConstants.Query, hiveJobDefinition.Query)
.AddCommand(CmdletConstants.StartAzureHDInsightJob)
.WithParameter(CmdletConstants.Cluster, cluster.ConnectionUrl)
.WithParameter(CmdletConstants.Credential, GetPSCredential(cluster.HttpUserName, cluster.HttpPassword))
.AddCommand(CmdletConstants.WaitAzureHDInsightJob)
.WithParameter(CmdletConstants.Credential, GetPSCredential(cluster.HttpUserName, cluster.HttpPassword))
.Invoke();
Assert.AreEqual(1, results.Results.Count);
Assert.AreEqual("Completed", results.Results.ToEnumerable<AzureHDInsightJob>().First().State);
}
}
public JobCreationResults CreateHiveJob(HiveJobCreateParameters hiveJobCreateParameters)
{
return this.CreateHiveJobAsync(hiveJobCreateParameters).WaitForResult();
}
public void ICanCallThe_New_HDInsightHiveJobDefinitionCmdlet_WithParameters()
{
var HiveJobDefinition = new HiveJobCreateParameters { JobName = "show tables jobDetails", Query = "show tables" };
HiveJobDefinition.Defines.Add("map.input.tasks", "1000");
HiveJobDefinition.Defines.Add("map.input.reducers", "1000");
using (IRunspace runspace = this.GetPowerShellRunspace())
{
IPipelineResult results =
runspace.NewPipeline()
.AddCommand(CmdletConstants.NewAzureHDInsightHiveJobDefinition)
.WithParameter(CmdletConstants.JobName, HiveJobDefinition.JobName)
.WithParameter(CmdletConstants.Query, HiveJobDefinition.Query)
.WithParameter(CmdletConstants.Parameters, HiveJobDefinition.Defines)
.Invoke();
Assert.AreEqual(1, results.Results.Count);
AzureHDInsightHiveJobDefinition HiveJobFromPowershell = results.Results.ToEnumerable<AzureHDInsightHiveJobDefinition>().First();
Assert.AreEqual(HiveJobDefinition.JobName, HiveJobFromPowershell.JobName);
Assert.AreEqual(HiveJobDefinition.Query, HiveJobFromPowershell.Query);
foreach (var parameter in HiveJobDefinition.Defines)
{
Assert.IsTrue(
HiveJobFromPowershell.Defines.Any(arg => string.Equals(parameter.Key, arg.Key) && string.Equals(parameter.Value, arg.Value)),
"Unable to find parameter '{0}' in value returned from powershell",
parameter.Key);
}
}
}
public void ICannotCallThe_New_HDInsightHiveJobDefinitionCmdlet_WithoutFileOrQueryParameter()
{
var HiveJobDefinition = new HiveJobCreateParameters
{
JobName = "show tables jobDetails",
File = TestConstants.WabsProtocolSchemeName + "filepath.hql"
};
try
{
using (IRunspace runspace = this.GetPowerShellRunspace())
{
runspace.NewPipeline()
.AddCommand(CmdletConstants.NewAzureHDInsightHiveJobDefinition)
.WithParameter(CmdletConstants.JobName, HiveJobDefinition.JobName)
.Invoke();
Assert.Fail("test failed.");
}
}
catch (CmdletInvocationException invokeException)
{
var psArgumentException = invokeException.GetBaseException() as PSArgumentException;
Assert.IsNotNull(psArgumentException);
Assert.AreEqual("Either File or Query should be specified for Hive jobs.", psArgumentException.Message);
}
}
