public void ICanCallThe_New_HDInsightPigJobDefinitionCmdlet_WithResources()
{
var pigJobDefinition = new PigJobCreateParameters {
Query = "load 'passwd' using PigStorage(':'); B = foreach A generate $0 as id;"
};
pigJobDefinition.Files.Add("pidata.txt");
pigJobDefinition.Files.Add("pidate2.txt");
using (IRunspace runspace = this.GetPowerShellRunspace())
{
IPipelineResult results =
runspace.NewPipeline()
.AddCommand(CmdletConstants.NewAzureHDInsightPigJobDefinition)
.WithParameter(CmdletConstants.Query, pigJobDefinition.Query)
.WithParameter(CmdletConstants.Files, pigJobDefinition.Files)
.Invoke();
Assert.AreEqual(1, results.Results.Count);
AzureHDInsightPigJobDefinition pigJobFromPowershell = results.Results.ToEnumerable <AzureHDInsightPigJobDefinition>().First();
Assert.AreEqual(pigJobDefinition.Query, pigJobFromPowershell.Query);
foreach (string file in pigJobDefinition.Files)
{
Assert.IsTrue(
pigJobFromPowershell.Files.Any(arg => string.Equals(file, arg)),
"Unable to find File '{0}' in value returned from powershell",
file);
}
}
}
public void CannotCreateNewPigJob_WithRestrictedCharacters_StartJob()
{
var pigJobDefinition = new PigJobCreateParameters {
Query = "load table from 'A' %"
};
INewAzureHDInsightPigJobDefinitionCommand newMapReduceJobDefinitionCommand =
ServiceLocator.Instance.Locate <IAzureHDInsightCommandFactory>().CreateNewPigJobDefinition();
newMapReduceJobDefinitionCommand.Query = pigJobDefinition.Query;
newMapReduceJobDefinitionCommand.EndProcessing();
AzureHDInsightPigJobDefinition pigJobFromCommand = newMapReduceJobDefinitionCommand.Output.ElementAt(0);
try
{
TestJobStart(pigJobFromCommand);
Assert.Fail();
}
catch (AggregateException aggregateException)
{
var invalidOperationException = aggregateException.GetBaseException() as InvalidOperationException;
Assert.IsNotNull(invalidOperationException);
Assert.IsTrue(invalidOperationException.Message.Contains("Query contains restricted character :'%'"), "Exception not thrown for special character");
}
}
public void ICanCallThe_New_HDInsightPigJobDefinitionCmdlet_WithArguments()
{
var pigJobDefinition = new PigJobCreateParameters {
File = TestConstants.WabsProtocolSchemeName + "container@accountname/pigquery.q"
};
pigJobDefinition.Arguments.Add("map.input.tasks=1000");
pigJobDefinition.Arguments.Add("map.input.reducers=1000");
using (IRunspace runspace = this.GetPowerShellRunspace())
{
IPipelineResult results =
runspace.NewPipeline()
.AddCommand(CmdletConstants.NewAzureHDInsightPigJobDefinition)
.WithParameter(CmdletConstants.File, pigJobDefinition.File)
.WithParameter(CmdletConstants.Arguments, pigJobDefinition.Arguments)
.Invoke();
Assert.AreEqual(1, results.Results.Count);
AzureHDInsightPigJobDefinition pigJobFromPowershell = results.Results.ToEnumerable <AzureHDInsightPigJobDefinition>().First();
Assert.AreEqual(pigJobDefinition.File, pigJobFromPowershell.File);
foreach (string argument in pigJobDefinition.Arguments)
{
Assert.IsTrue(
pigJobFromPowershell.Arguments.Any(arg => string.Equals(argument, arg)),
string.Format("Unable to find parameter '{0}' in value returned from powershell", argument));
}
}
}
public void CanCreateNewPigDefinition_WithResources()
{
var pigJobDefinition = new PigJobCreateParameters {
Query = "load 'passwd' using PigStorage(':'); B = foreach A generate $0 as id;"
};
pigJobDefinition.Files.Add("pidata.txt");
pigJobDefinition.Files.Add("pidate2.txt");
INewAzureHDInsightPigJobDefinitionCommand newPigJobDefinitionCommand =
ServiceLocator.Instance.Locate <IAzureHDInsightCommandFactory>().CreateNewPigJobDefinition();
newPigJobDefinitionCommand.Query = pigJobDefinition.Query;
newPigJobDefinitionCommand.Files = pigJobDefinition.Files.ToArray();
newPigJobDefinitionCommand.EndProcessing();
AzureHDInsightPigJobDefinition pigJobFromCommand = newPigJobDefinitionCommand.Output.ElementAt(0);
Assert.AreEqual(pigJobDefinition.Query, pigJobFromCommand.Query);
foreach (string resource in pigJobDefinition.Files)
{
Assert.IsTrue(
pigJobFromCommand.Files.Any(arg => string.Equals(resource, arg)),
"Unable to find File '{0}' in value returned from command",
resource);
}
}
public void CannotCreateNewPigJob_WithRestrictedCharacters_StartJob()
{
var pigJobDefinition = new PigJobCreateParameters {
Query = "load table from 'A' %"
};
INewAzureHDInsightPigJobDefinitionCommand newMapReduceJobDefinitionCommand =
ServiceLocator.Instance.Locate <IAzureHDInsightCommandFactory>().CreateNewPigJobDefinition();
newMapReduceJobDefinitionCommand.Query = pigJobDefinition.Query;
newMapReduceJobDefinitionCommand.EndProcessing();
AzureHDInsightPigJobDefinition pigJobFromCommand = newMapReduceJobDefinitionCommand.Output.ElementAt(0);
try
{
TestJobStart(pigJobFromCommand);
Assert.Fail();
}
catch (AggregateException aggregateException)
{
var invalidOperationException = aggregateException.GetBaseException() as InvalidOperationException;
Assert.IsNotNull(invalidOperationException);
Assert.AreEqual("Query text contains restricted character '%', please upload the query to a file in storage and re-submit the job using the -File parameter",
invalidOperationException.Message);
}
}
public void CanCreateNewPigDefinition_WithArguments()
{
var pigJobDefinition = new PigJobCreateParameters {
Query = "load 'passwd' using PigStorage(':'); B = foreach A generate $0 as id;"
};
pigJobDefinition.Arguments.Add("map.input.tasks=1000");
pigJobDefinition.Arguments.Add("map.input.reducers=1000");
INewAzureHDInsightPigJobDefinitionCommand newPigJobDefinitionCommand =
ServiceLocator.Instance.Locate <IAzureHDInsightCommandFactory>().CreateNewPigJobDefinition();
newPigJobDefinitionCommand.Query = pigJobDefinition.Query;
newPigJobDefinitionCommand.Arguments = pigJobDefinition.Arguments.ToArray();
newPigJobDefinitionCommand.EndProcessing();
AzureHDInsightPigJobDefinition pigJobFromCommand = newPigJobDefinitionCommand.Output.ElementAt(0);
Assert.AreEqual(pigJobDefinition.Query, pigJobFromCommand.Query);
foreach (string parameter in pigJobDefinition.Arguments)
{
Assert.IsTrue(
pigJobFromCommand.Arguments.Any(arg => string.Equals(parameter, arg)),
"Unable to find parameter '{0}' in value returned from command",
parameter);
}
}
public void CanCreateNewPigJob_WithoutJobName_WithQuery()
{
var pigJobDefinition = new AzureHDInsightPigJobDefinition {
Query = "show tables"
};
AzureHDInsightJob startedJob = TestJobStart(pigJobDefinition);
Assert.AreEqual(string.Empty, startedJob.Name);
}
public void CanCreateNewPigJob_WithoutJobName_WithFile()
{
var pigJobDefinition = new AzureHDInsightPigJobDefinition
{
File = TestConstants.WabsProtocolSchemeName + "container@hostname/Container1/myqueryfile.hql"
};
AzureHDInsightJob startedJob = TestJobStart(pigJobDefinition);
Assert.AreEqual(string.Empty, startedJob.Name);
}
private static async Task <JobCreationResults> SubmitPigJob(AzureHDInsightPigJobDefinition azurePigJobDefinition, IJobSubmissionClient client)
{
var pigJobDefinition = new PigJobCreateParameters {
Query = azurePigJobDefinition.Query, File = azurePigJobDefinition.File
};
pigJobDefinition.StatusFolder = azurePigJobDefinition.StatusFolder;
pigJobDefinition.Arguments.AddRange(azurePigJobDefinition.Arguments);
pigJobDefinition.Files.AddRange(azurePigJobDefinition.Files);
var jobCreationResults = await client.CreatePigJobAsync(pigJobDefinition);
return(jobCreationResults);
}
public void CanCreateNewPigJob_StartJob()
{
var pigJobDefinition = new PigJobCreateParameters {
Query = "load table from 'A'"
};
INewAzureHDInsightPigJobDefinitionCommand newMapReduceJobDefinitionCommand =
ServiceLocator.Instance.Locate <IAzureHDInsightCommandFactory>().CreateNewPigJobDefinition();
newMapReduceJobDefinitionCommand.Query = pigJobDefinition.Query;
newMapReduceJobDefinitionCommand.EndProcessing();
AzureHDInsightPigJobDefinition pigJobFromCommand = newMapReduceJobDefinitionCommand.Output.ElementAt(0);
TestJobStart(pigJobFromCommand);
}
public void CanCreateNewPigDefinition()
{
var pigJobDefinition = new PigJobCreateParameters {
Query = "load 'passwd' using PigStorage(':'); B = foreach A generate $0 as id;"
};
INewAzureHDInsightPigJobDefinitionCommand newPigJobDefinitionCommand =
ServiceLocator.Instance.Locate <IAzureHDInsightCommandFactory>().CreateNewPigJobDefinition();
newPigJobDefinitionCommand.Query = pigJobDefinition.Query;
newPigJobDefinitionCommand.EndProcessing();
AzureHDInsightPigJobDefinition pigJobFromCommand = newPigJobDefinitionCommand.Output.ElementAt(0);
Assert.AreEqual(pigJobDefinition.Query, pigJobFromCommand.Query);
}
public void ICanCallThe_New_HDInsightPigJobDefinitionCmdlet()
{
var pigJobDefinition = new PigJobCreateParameters {
Query = "load 'passwd' using PigStorage(':'); B = foreach A generate $0 as id;"
};
using (IRunspace runspace = this.GetPowerShellRunspace())
{
IPipelineResult results =
runspace.NewPipeline()
.AddCommand(CmdletConstants.NewAzureHDInsightPigJobDefinition)
.WithParameter(CmdletConstants.Query, pigJobDefinition.Query)
.Invoke();
Assert.AreEqual(1, results.Results.Count);
AzureHDInsightPigJobDefinition pigJobFromPowershell = results.Results.ToEnumerable <AzureHDInsightPigJobDefinition>().First();
Assert.AreEqual(pigJobDefinition.Query, pigJobFromPowershell.Query);
}
}
public virtual void NewPigJob_StartJob_GetJob()
{
var pigJobDefinition = new PigJobCreateParameters {
Query = "load 'passwd' using PigStorage(':'); B = foreach A generate $0 as id;"
};
using (IRunspace runspace = this.GetPowerShellRunspace())
{
IPipelineResult results =
runspace.NewPipeline()
.AddCommand(CmdletConstants.NewAzureHDInsightPigJobDefinition)
.WithParameter(CmdletConstants.Query, pigJobDefinition.Query)
.Invoke();
Assert.AreEqual(1, results.Results.Count);
AzureHDInsightPigJobDefinition pigJobFromPowershell = results.Results.ToEnumerable <AzureHDInsightPigJobDefinition>().First();
AzureHDInsightJob pigJobfromHistory = RunJobAndGetWithId(runspace, pigJobFromPowershell);
Assert.AreEqual(pigJobfromHistory.Query, pigJobDefinition.Query, "Failed to retrieve query for executed pig jobDetails");
}
}
public virtual void ICanCallThe_NewPigJob_Then_Start_HDInsightJobsCmdlet()
{
var pigJobDefinition = new PigJobCreateParameters {
Query = "load table from 'A'"
};
using (IRunspace runspace = this.GetPowerShellRunspace())
{
IPipelineResult results =
runspace.NewPipeline()
.AddCommand(CmdletConstants.NewAzureHDInsightPigJobDefinition)
.WithParameter(CmdletConstants.Query, pigJobDefinition.Query)
.Invoke();
Assert.AreEqual(1, results.Results.Count);
AzureHDInsightPigJobDefinition pigJobFromPowershell = results.Results.ToEnumerable <AzureHDInsightPigJobDefinition>().First();
RunJobInPowershell(runspace, pigJobFromPowershell);
}
}
public void ICanCallThe_New_HDInsightPigJobDefinitionCmdlet_WithQueryFile()
{
var pigJobDefinition = new PigJobCreateParameters {
File = TestConstants.WabsProtocolSchemeName + "container@accountname/pigquery.q"
};
pigJobDefinition.Arguments.Add("map.input.tasks=1000");
pigJobDefinition.Arguments.Add("map.input.reducers=1000");
using (IRunspace runspace = this.GetPowerShellRunspace())
{
IPipelineResult results =
runspace.NewPipeline()
.AddCommand(CmdletConstants.NewAzureHDInsightPigJobDefinition)
.WithParameter(CmdletConstants.File, pigJobDefinition.File)
.WithParameter(CmdletConstants.Arguments, pigJobDefinition.Arguments)
.Invoke();
Assert.AreEqual(1, results.Results.Count);
AzureHDInsightPigJobDefinition pigJobFromPowershell = results.Results.ToEnumerable <AzureHDInsightPigJobDefinition>().First();
Assert.AreEqual(pigJobDefinition.File, pigJobFromPowershell.File);
}
}
public NewAzureHDInsightPigJobDefinitionCommand()
{
Arguments = new string[] { };
Files = new string[] { };
job = new AzureHDInsightPigJobDefinition();
}
public void CanCreateNewPigJob_WithoutJobName_WithQuery()
{
var pigJobDefinition = new AzureHDInsightPigJobDefinition { Query = "show tables" };
AzureHDInsightJob startedJob = TestJobStart(pigJobDefinition);
Assert.AreEqual(string.Empty, startedJob.Name);
}
public void CanCreateNewPigJob_WithoutJobName_WithFile()
{
var pigJobDefinition = new AzureHDInsightPigJobDefinition
{
File = TestConstants.WabsProtocolSchemeName + "container@hostname/Container1/myqueryfile.hql"
};
AzureHDInsightJob startedJob = TestJobStart(pigJobDefinition);
Assert.AreEqual(string.Empty, startedJob.Name);
}
