public void CanCreateNewSqoopDefinition_WithResources()
{
var pigJobDefinition = new SqoopJobCreateParameters
{
Command = "load 'passwd' using SqoopStorage(':'); B = foreach A generate $0 as id;"
};
pigJobDefinition.Files.Add("pidata.txt");
pigJobDefinition.Files.Add("pidate2.txt");
INewAzureHDInsightSqoopJobDefinitionCommand newSqoopJobDefinitionCommand =
ServiceLocator.Instance.Locate <IAzureHDInsightCommandFactory>().CreateNewSqoopDefinition();
newSqoopJobDefinitionCommand.Command = pigJobDefinition.Command;
newSqoopJobDefinitionCommand.Files = pigJobDefinition.Files.ToArray();
newSqoopJobDefinitionCommand.EndProcessing();
AzureHDInsightSqoopJobDefinition pigJobFromCommand = newSqoopJobDefinitionCommand.Output.ElementAt(0);
Assert.AreEqual(pigJobDefinition.Command, pigJobFromCommand.Command);
foreach (string resource in pigJobDefinition.Files)
{
Assert.IsTrue(
pigJobFromCommand.Files.Any(arg => string.Equals(resource, arg)),
"Unable to find File '{0}' in value returned from command",
resource);
}
}
public void ICanCallThe_New_HDInsightSqoopJobDefinitionCmdlet_WithResources()
{
var sqoopJobDefinition = new SqoopJobCreateParameters {
Command = "show tables"
};
sqoopJobDefinition.Files.Add("pidata.txt");
sqoopJobDefinition.Files.Add("pidate2.txt");
using (IRunspace runspace = this.GetPowerShellRunspace())
{
IPipelineResult results =
runspace.NewPipeline()
.AddCommand(CmdletConstants.NewAzureHDInsightSqoopJobDefinition)
.WithParameter(CmdletConstants.Command, sqoopJobDefinition.Command)
.WithParameter(CmdletConstants.Files, sqoopJobDefinition.Files)
.Invoke();
Assert.AreEqual(1, results.Results.Count);
AzureHDInsightSqoopJobDefinition SqoopJobFromPowershell = results.Results.ToEnumerable <AzureHDInsightSqoopJobDefinition>().First();
Assert.AreEqual(sqoopJobDefinition.Command, SqoopJobFromPowershell.Command);
foreach (string file in sqoopJobDefinition.Files)
{
Assert.IsTrue(
SqoopJobFromPowershell.Files.Any(arg => string.Equals(file, arg)),
"Unable to find File '{0}' in value returned from powershell",
file);
}
}
}
public void CanStartSqoopJob()
{
var sqoopJobDefinition = new AzureHDInsightSqoopJobDefinition {
Command = "show tables;"
};
TestJobStart(sqoopJobDefinition);
}
private static async Task <JobCreationResults> CreateSqoopJob(
AzureHDInsightSqoopJobDefinition azureSqoopJobDefinition, IJobSubmissionClient client)
{
SqoopJobCreateParameters sqoopJobDefinition = azureSqoopJobDefinition.ToSqoopJobCreateParameters();
var jobCreationResults = await client.CreateSqoopJobAsync(sqoopJobDefinition);
return(jobCreationResults);
}
private static async Task <JobCreationResults> CreateSqoopJob(
AzureHDInsightSqoopJobDefinition azureSqoopJobDefinition, IJobSubmissionClient client)
{
AssertQueryDoesNotContainRestrictedCharacters(azureSqoopJobDefinition.Command, "Command");
SqoopJobCreateParameters sqoopJobDefinition = azureSqoopJobDefinition.ToSqoopJobCreateParameters();
var jobCreationResults = await client.CreateSqoopJobAsync(sqoopJobDefinition);
return(jobCreationResults);
}
public void CanCreateNewSqoopDefinition_WithFile()
{
var pigJobDefinition = new SqoopJobCreateParameters {
File = "my local file"
};
INewAzureHDInsightSqoopJobDefinitionCommand newSqoopJobDefinitionCommand =
ServiceLocator.Instance.Locate <IAzureHDInsightCommandFactory>().CreateNewSqoopDefinition();
newSqoopJobDefinitionCommand.File = pigJobDefinition.File;
newSqoopJobDefinitionCommand.EndProcessing();
AzureHDInsightSqoopJobDefinition pigJobFromCommand = newSqoopJobDefinitionCommand.Output.ElementAt(0);
Assert.AreEqual(pigJobDefinition.File, pigJobFromCommand.File);
}
public void CanCreateNewSqoopDefinition()
{
var pigJobDefinition = new SqoopJobCreateParameters
{
Command = "load 'passwd' using SqoopStorage(':'); B = foreach A generate $0 as id;"
};
INewAzureHDInsightSqoopJobDefinitionCommand newSqoopJobDefinitionCommand =
ServiceLocator.Instance.Locate <IAzureHDInsightCommandFactory>().CreateNewSqoopDefinition();
newSqoopJobDefinitionCommand.Command = pigJobDefinition.Command;
newSqoopJobDefinitionCommand.EndProcessing();
AzureHDInsightSqoopJobDefinition pigJobFromCommand = newSqoopJobDefinitionCommand.Output.ElementAt(0);
Assert.AreEqual(pigJobDefinition.Command, pigJobFromCommand.Command);
}
public void CannotStartSqoopJob_WithRestrictedCharacters()
{
var sqoopJobDefinition = new AzureHDInsightSqoopJobDefinition {
Command = "show tables; %"
};
try
{
TestJobStart(sqoopJobDefinition);
Assert.Fail();
}
catch (AggregateException aggregateException)
{
var invalidOperationException = aggregateException.GetBaseException() as InvalidOperationException;
Assert.IsNotNull(invalidOperationException);
Assert.IsTrue(invalidOperationException.Message.Contains("Query contains restricted character :'%'"), "Exception not thrown for special character");
}
}
public void CannotStartSqoopJob_WithRestrictedCharacters()
{
var sqoopJobDefinition = new AzureHDInsightSqoopJobDefinition {
Command = "show tables; %"
};
try
{
TestJobStart(sqoopJobDefinition);
Assert.Fail();
}
catch (AggregateException aggregateException)
{
var invalidOperationException = aggregateException.GetBaseException() as InvalidOperationException;
Assert.IsNotNull(invalidOperationException);
Assert.AreEqual("Command text contains restricted character '%', please upload the query to a file in storage and re-submit the job using the -File parameter",
invalidOperationException.Message);
}
}
public void ICanCallThe_New_HDInsightSqoopJobDefinitionCmdlet_WithFileParameter()
{
var sqoopJobDefinition = new SqoopJobCreateParameters {
File = TestConstants.WabsProtocolSchemeName + "filepath.hql"
};
using (IRunspace runspace = this.GetPowerShellRunspace())
{
IPipelineResult results =
runspace.NewPipeline()
.AddCommand(CmdletConstants.NewAzureHDInsightSqoopJobDefinition)
.WithParameter(CmdletConstants.File, sqoopJobDefinition.File)
.Invoke();
Assert.AreEqual(1, results.Results.Count);
AzureHDInsightSqoopJobDefinition SqoopJobFromPowershell = results.Results.ToEnumerable <AzureHDInsightSqoopJobDefinition>().First();
Assert.AreEqual(sqoopJobDefinition.File, SqoopJobFromPowershell.File);
}
}
public void ICanCallThe_New_HDInsightSqoopJobDefinitionCmdlet()
{
var sqoopJobDefinition = new SqoopJobCreateParameters {
Command = "show tables"
};
using (IRunspace runspace = this.GetPowerShellRunspace())
{
IPipelineResult results =
runspace.NewPipeline()
.AddCommand(CmdletConstants.NewAzureHDInsightSqoopJobDefinition)
.WithParameter(CmdletConstants.Command, sqoopJobDefinition.Command)
.Invoke();
Assert.AreEqual(1, results.Results.Count);
AzureHDInsightSqoopJobDefinition SqoopJobFromPowershell = results.Results.ToEnumerable <AzureHDInsightSqoopJobDefinition>().First();
Assert.AreEqual(sqoopJobDefinition.Command, SqoopJobFromPowershell.Command);
}
}
public virtual void ICanCallThe_NewSqoopJob_Then_Start_HDInsightJobsCmdlet()
{
var sqoopJobDefinition = new SqoopJobCreateParameters {
Command = "load table from 'A'"
};
using (IRunspace runspace = this.GetPowerShellRunspace())
{
IPipelineResult results =
runspace.NewPipeline()
.AddCommand(CmdletConstants.NewAzureHDInsightSqoopJobDefinition)
.WithParameter(CmdletConstants.Command, sqoopJobDefinition.Command)
.Invoke();
Assert.AreEqual(1, results.Results.Count);
AzureHDInsightSqoopJobDefinition sqoopJobFromPowershell = results.Results.ToEnumerable <AzureHDInsightSqoopJobDefinition>().First();
RunJobInPowershell(runspace, sqoopJobFromPowershell);
}
}
public void CanCreateSDKObjectFromPowershellObject()
{
var sqoopJobDefinition = new AzureHDInsightSqoopJobDefinition
{
Command = "Import into sqlserver",
File = "http://myfileshare.txt",
StatusFolder = Guid.NewGuid().ToString(),
};
sqoopJobDefinition.Arguments.Add("arg1");
sqoopJobDefinition.Files.Add("file1.sqoop");
SqoopJobCreateParameters sdkObject = sqoopJobDefinition.ToSqoopJobCreateParameters();
Assert.AreEqual(sqoopJobDefinition.StatusFolder, sdkObject.StatusFolder);
Assert.AreEqual(sqoopJobDefinition.File, sdkObject.File);
Assert.AreEqual(sqoopJobDefinition.Command, sdkObject.Command);
foreach (string file in sqoopJobDefinition.Files)
{
Assert.IsTrue(sdkObject.Files.Contains(file), file);
}
}
public void CanCreateSDKObjectFromPowershellObject()
{
var sqoopJobDefinition = new AzureHDInsightSqoopJobDefinition
{
Command = "Import into sqlserver",
File = "http://myfileshare.txt",
StatusFolder = Guid.NewGuid().ToString(),
};
sqoopJobDefinition.Arguments.Add("arg1");
sqoopJobDefinition.Files.Add("file1.sqoop");
SqoopJobCreateParameters sdkObject = sqoopJobDefinition.ToSqoopJobCreateParameters();
Assert.AreEqual(sqoopJobDefinition.StatusFolder, sdkObject.StatusFolder);
Assert.AreEqual(sqoopJobDefinition.File, sdkObject.File);
Assert.AreEqual(sqoopJobDefinition.Command, sdkObject.Command);
foreach (string file in sqoopJobDefinition.Files)
{
Assert.IsTrue(sdkObject.Files.Contains(file), file);
}
}
public NewAzureHDInsightSqoopJobDefinitionCommand()
{
Files = new string[] { };
job = new AzureHDInsightSqoopJobDefinition();
}
public void CannotStartSqoopJob_WithRestrictedCharacters()
{
var sqoopJobDefinition = new AzureHDInsightSqoopJobDefinition { Command = "show tables; %" };
try
{
TestJobStart(sqoopJobDefinition);
Assert.Fail();
}
catch (AggregateException aggregateException)
{
var invalidOperationException = aggregateException.GetBaseException() as InvalidOperationException;
Assert.IsNotNull(invalidOperationException);
Assert.AreEqual("Command text contains restricted character '%', please upload the query to a file in storage and re-submit the job using the -File parameter",
invalidOperationException.Message);
}
}
public void CanStartSqoopJob()
{
var sqoopJobDefinition = new AzureHDInsightSqoopJobDefinition { Command = "show tables;" };
TestJobStart(sqoopJobDefinition);
}
