public void CanCreateNewPigDefinition_WithArguments()
{
var pigJobDefinition = new PigJobCreateParameters { Query = "load 'passwd' using PigStorage(':'); B = foreach A generate $0 as id;" };
pigJobDefinition.Arguments.Add("map.input.tasks=1000");
pigJobDefinition.Arguments.Add("map.input.reducers=1000");
INewAzureHDInsightPigJobDefinitionCommand newPigJobDefinitionCommand =
ServiceLocator.Instance.Locate<IAzureHDInsightCommandFactory>().CreateNewPigJobDefinition();
newPigJobDefinitionCommand.Query = pigJobDefinition.Query;
newPigJobDefinitionCommand.Arguments = pigJobDefinition.Arguments.ToArray();
newPigJobDefinitionCommand.EndProcessing();
AzureHDInsightPigJobDefinition pigJobFromCommand = newPigJobDefinitionCommand.Output.ElementAt(0);
Assert.AreEqual(pigJobDefinition.Query, pigJobFromCommand.Query);
foreach (string parameter in pigJobDefinition.Arguments)
{
Assert.IsTrue(
pigJobFromCommand.Arguments.Any(arg => string.Equals(parameter, arg)),
"Unable to find parameter '{0}' in value returned from command",
parameter);
}
}
public void ICanCallThe_New_HDInsightPigJobDefinitionCmdlet_WithArguments()
{
var pigJobDefinition = new PigJobCreateParameters { File = TestConstants.WabsProtocolSchemeName + "container@accountname/pigquery.q" };
pigJobDefinition.Arguments.Add("map.input.tasks=1000");
pigJobDefinition.Arguments.Add("map.input.reducers=1000");
using (IRunspace runspace = this.GetPowerShellRunspace())
{
IPipelineResult results =
runspace.NewPipeline()
.AddCommand(CmdletConstants.NewAzureHDInsightPigJobDefinition)
.WithParameter(CmdletConstants.File, pigJobDefinition.File)
.WithParameter(CmdletConstants.Arguments, pigJobDefinition.Arguments)
.Invoke();
Assert.AreEqual(1, results.Results.Count);
AzureHDInsightPigJobDefinition pigJobFromPowershell = results.Results.ToEnumerable<AzureHDInsightPigJobDefinition>().First();
Assert.AreEqual(pigJobDefinition.File, pigJobFromPowershell.File);
foreach (string argument in pigJobDefinition.Arguments)
{
Assert.IsTrue(
pigJobFromPowershell.Arguments.Any(arg => string.Equals(argument, arg)),
string.Format("Unable to find parameter '{0}' in value returned from powershell", argument));
}
}
}
public async Task<JobCreationResults> SubmitPigJob(PigJobCreateParameters pigJobCreateParameters)
{
var relative = new Uri(
HadoopRemoteRestConstants.MapReduceStreaming + "?" +
HadoopRemoteRestConstants.UserName + "=" +
_connectionConfig.UserName.EscapeDataString(),
UriKind.Relative);
var requestContent = _converter.SerializePigRequest(_connectionConfig.UserName, pigJobCreateParameters);
var result = await MakeAsyncJobSubmissionRequest(relative, requestContent);
return new JobCreationResults {JobId = _converter.DeserializeJobSubmissionResponse(result)};
}
public void CanCreateNewPigDefinition()
{
var pigJobDefinition = new PigJobCreateParameters { Query = "load 'passwd' using PigStorage(':'); B = foreach A generate $0 as id;" };
INewAzureHDInsightPigJobDefinitionCommand newPigJobDefinitionCommand =
ServiceLocator.Instance.Locate<IAzureHDInsightCommandFactory>().CreateNewPigJobDefinition();
newPigJobDefinitionCommand.Query = pigJobDefinition.Query;
newPigJobDefinitionCommand.EndProcessing();
AzureHDInsightPigJobDefinition pigJobFromCommand = newPigJobDefinitionCommand.Output.ElementAt(0);
Assert.AreEqual(pigJobDefinition.Query, pigJobFromCommand.Query);
}
public virtual void ICanCallThe_NewPigJob_Then_Start_HDInsightJobsCmdlet()
{
var pigJobDefinition = new PigJobCreateParameters { Query = "load table from 'A'" };
using (IRunspace runspace = this.GetPowerShellRunspace())
{
IPipelineResult results =
runspace.NewPipeline()
.AddCommand(CmdletConstants.NewAzureHDInsightPigJobDefinition)
.WithParameter(CmdletConstants.Query, pigJobDefinition.Query)
.Invoke();
Assert.AreEqual(1, results.Results.Count);
AzureHDInsightPigJobDefinition pigJobFromPowershell = results.Results.ToEnumerable<AzureHDInsightPigJobDefinition>().First();
RunJobInPowershell(runspace, pigJobFromPowershell);
}
}
public void ICanCallThe_New_HDInsightPigJobDefinitionCmdlet()
{
var pigJobDefinition = new PigJobCreateParameters { Query = "load 'passwd' using PigStorage(':'); B = foreach A generate $0 as id;" };
using (IRunspace runspace = this.GetPowerShellRunspace())
{
IPipelineResult results =
runspace.NewPipeline()
.AddCommand(CmdletConstants.NewAzureHDInsightPigJobDefinition)
.WithParameter(CmdletConstants.Query, pigJobDefinition.Query)
.Invoke();
Assert.AreEqual(1, results.Results.Count);
AzureHDInsightPigJobDefinition pigJobFromPowershell = results.Results.ToEnumerable<AzureHDInsightPigJobDefinition>().First();
Assert.AreEqual(pigJobDefinition.Query, pigJobFromPowershell.Query);
}
}
public void ICanCallThe_New_HDInsightPigJobDefinitionCmdlet_WithResources()
{
var pigJobDefinition = new PigJobCreateParameters { Query = "load 'passwd' using PigStorage(':'); B = foreach A generate $0 as id;" };
pigJobDefinition.Files.Add("pidata.txt");
pigJobDefinition.Files.Add("pidate2.txt");
using (IRunspace runspace = this.GetPowerShellRunspace())
{
IPipelineResult results =
runspace.NewPipeline()
.AddCommand(CmdletConstants.NewAzureHDInsightPigJobDefinition)
.WithParameter(CmdletConstants.Query, pigJobDefinition.Query)
.WithParameter(CmdletConstants.Files, pigJobDefinition.Files)
.Invoke();
Assert.AreEqual(1, results.Results.Count);
AzureHDInsightPigJobDefinition pigJobFromPowershell = results.Results.ToEnumerable<AzureHDInsightPigJobDefinition>().First();
Assert.AreEqual(pigJobDefinition.Query, pigJobFromPowershell.Query);
foreach (string file in pigJobDefinition.Files)
{
Assert.IsTrue(
pigJobFromPowershell.Files.Any(arg => string.Equals(file, arg)),
"Unable to find File '{0}' in value returned from powershell",
file);
}
}
}
public void ICanCallThe_New_HDInsightPigJobDefinitionCmdlet_WithQueryFile()
{
var pigJobDefinition = new PigJobCreateParameters { File = TestConstants.WabsProtocolSchemeName + "container@accountname/pigquery.q" };
pigJobDefinition.Arguments.Add("map.input.tasks=1000");
pigJobDefinition.Arguments.Add("map.input.reducers=1000");
using (IRunspace runspace = this.GetPowerShellRunspace())
{
IPipelineResult results =
runspace.NewPipeline()
.AddCommand(CmdletConstants.NewAzureHDInsightPigJobDefinition)
.WithParameter(CmdletConstants.File, pigJobDefinition.File)
.WithParameter(CmdletConstants.Arguments, pigJobDefinition.Arguments)
.Invoke();
Assert.AreEqual(1, results.Results.Count);
AzureHDInsightPigJobDefinition pigJobFromPowershell = results.Results.ToEnumerable<AzureHDInsightPigJobDefinition>().First();
Assert.AreEqual(pigJobDefinition.File, pigJobFromPowershell.File);
}
}
public void CannotCreateNewPigJob_WithRestrictedCharacters_StartJob()
{
var pigJobDefinition = new PigJobCreateParameters { Query = "load table from 'A' %" };
INewAzureHDInsightPigJobDefinitionCommand newMapReduceJobDefinitionCommand =
ServiceLocator.Instance.Locate<IAzureHDInsightCommandFactory>().CreateNewPigJobDefinition();
newMapReduceJobDefinitionCommand.Query = pigJobDefinition.Query;
newMapReduceJobDefinitionCommand.EndProcessing();
AzureHDInsightPigJobDefinition pigJobFromCommand = newMapReduceJobDefinitionCommand.Output.ElementAt(0);
try
{
TestJobStart(pigJobFromCommand);
Assert.Fail();
}
catch (AggregateException aggregateException)
{
var invalidOperationException = aggregateException.GetBaseException() as InvalidOperationException;
Assert.IsNotNull(invalidOperationException);
Assert.AreEqual("Query text contains restricted character '%', please upload the query to a file in storage and re-submit the job using the -File parameter",
invalidOperationException.Message);
}
}
public void CanCreateNewPigJob_StartJob()
{
var pigJobDefinition = new PigJobCreateParameters { Query = "load table from 'A'" };
INewAzureHDInsightPigJobDefinitionCommand newMapReduceJobDefinitionCommand =
ServiceLocator.Instance.Locate<IAzureHDInsightCommandFactory>().CreateNewPigJobDefinition();
newMapReduceJobDefinitionCommand.Query = pigJobDefinition.Query;
newMapReduceJobDefinitionCommand.EndProcessing();
AzureHDInsightPigJobDefinition pigJobFromCommand = newMapReduceJobDefinitionCommand.Output.ElementAt(0);
TestJobStart(pigJobFromCommand);
}
public JobCreationResults CreatePigJob(PigJobCreateParameters pigJobCreateParameters)
{
return(this.CreatePigJobAsync(pigJobCreateParameters).WaitForResult());
}
public JobCreationResults CreatePigJob(PigJobCreateParameters pigJobCreateParameters)
{
return this.CreatePigJobAsync(pigJobCreateParameters).WaitForResult();
}
public void CanCreateNewPigDefinition_WithResources()
{
var pigJobDefinition = new PigJobCreateParameters { Query = "load 'passwd' using PigStorage(':'); B = foreach A generate $0 as id;" };
pigJobDefinition.Files.Add("pidata.txt");
pigJobDefinition.Files.Add("pidate2.txt");
INewAzureHDInsightPigJobDefinitionCommand newPigJobDefinitionCommand =
ServiceLocator.Instance.Locate<IAzureHDInsightCommandFactory>().CreateNewPigJobDefinition();
newPigJobDefinitionCommand.Query = pigJobDefinition.Query;
newPigJobDefinitionCommand.Files = pigJobDefinition.Files.ToArray();
newPigJobDefinitionCommand.EndProcessing();
AzureHDInsightPigJobDefinition pigJobFromCommand = newPigJobDefinitionCommand.Output.ElementAt(0);
Assert.AreEqual(pigJobDefinition.Query, pigJobFromCommand.Query);
foreach (string resource in pigJobDefinition.Files)
{
Assert.IsTrue(
pigJobFromCommand.Files.Any(arg => string.Equals(resource, arg)),
"Unable to find File '{0}' in value returned from command",
resource);
}
}
/// <summary>
/// Creates the payload for a Pig request.
/// </summary>
/// <param name="userName"> The user name.</param>
/// <param name="details"> The details.</param>
/// <returns>A string that represents the payload for the request.</returns>
public string SerializePigRequest(string userName, PigJobCreateParameters details)
{
details.ArgumentNotNull("details");
return this.SerializeQueryRequest(userName, details, string.Empty, details.File, details.Query, WebHCatConstants.Execute, details.Arguments, null);
}
/// <inheritdoc />
public async Task<JobCreationResults> SubmitPigJob(PigJobCreateParameters pigJobCreateParameters)
{
//NEIN: Any code modification here should add unit tests for this class
var converter = new PayloadConverter();
var payload = converter.SerializePigRequest(this.credentials.UserName, pigJobCreateParameters);
var client = ServiceLocator.Instance.Locate<IHadoopRemoteJobSubmissionRestClientFactory>().Create(this.credentials, this.context, this.ignoreSslErrors, this.GetUserAgentString());
var result = await client.SubmitPigJob(payload);
return new JobCreationResults() { JobId = converter.DeserializeJobSubmissionResponse(result.Content) };
}
public Task<JobCreationResults> CreatePigJobAsync(PigJobCreateParameters pigJobCreateParameters)
{
if (pigJobCreateParameters == null)
{
throw new ArgumentNullException("pigJobCreateParameters");
}
JobCreationResults retval =
this.CreateJobSuccessResult(
new JobDetails { Query = pigJobCreateParameters.Query, StatusDirectory = pigJobCreateParameters.StatusFolder }, string.Empty);
return TaskEx2.FromResult(retval);
}
/// <inheritdoc />
public async Task <JobCreationResults> SubmitPigJob(PigJobCreateParameters pigJobCreateParameters)
{
var remoteClient = ServiceLocator.Instance.Locate <IRemoteHadoopJobSubmissionPocoClientFactory>().Create(this.remoteCreds, this.context, this.ignoreSslErrors, this.GetUserAgentString());
return(await remoteClient.SubmitPigJob(pigJobCreateParameters));
}
