public Task <JobCreationResults> CreateSqoopJobAsync(SqoopJobCreateParameters sqoopJobCreateParameters)
{
var factory = ServiceLocator.Instance.Locate <IRemoteHadoopJobSubmissionPocoClientFactory>();
var pocoClient = factory.Create(this.credentials, this.Context, this.IgnoreSslErrors, this.userAgentString);
sqoopJobCreateParameters = this.PrepareQueryJob(sqoopJobCreateParameters);
return(pocoClient.SubmitSqoopJob(sqoopJobCreateParameters));
}
/// <summary>
/// Creates a SDK object from the PSCmdlet object type.
/// </summary>
/// <returns>A SDK Sqoop job definition object.</returns>
internal SqoopJobCreateParameters ToSqoopJobCreateParameters()
{
var soopJobDefinition = new SqoopJobCreateParameters { Command = this.Command, File = this.File, StatusFolder = this.StatusFolder };
if (this.Files.IsNotNull())
{
soopJobDefinition.Files.AddRange(this.Files);
}
return soopJobDefinition;
}
public void CanCreateNewSqoopDefinition_WithFile()
{
var pigJobDefinition = new SqoopJobCreateParameters { File = "my local file" };
INewAzureHDInsightSqoopJobDefinitionCommand newSqoopJobDefinitionCommand =
ServiceLocator.Instance.Locate<IAzureHDInsightCommandFactory>().CreateNewSqoopDefinition();
newSqoopJobDefinitionCommand.File = pigJobDefinition.File;
newSqoopJobDefinitionCommand.EndProcessing();
AzureHDInsightSqoopJobDefinition pigJobFromCommand = newSqoopJobDefinitionCommand.Output.ElementAt(0);
Assert.AreEqual(pigJobDefinition.File, pigJobFromCommand.File);
}
public void CanCreateNewSqoopDefinition()
{
var pigJobDefinition = new SqoopJobCreateParameters
{
Command = "load 'passwd' using SqoopStorage(':'); B = foreach A generate $0 as id;"
};
INewAzureHDInsightSqoopJobDefinitionCommand newSqoopJobDefinitionCommand =
ServiceLocator.Instance.Locate<IAzureHDInsightCommandFactory>().CreateNewSqoopDefinition();
newSqoopJobDefinitionCommand.Command = pigJobDefinition.Command;
newSqoopJobDefinitionCommand.EndProcessing();
AzureHDInsightSqoopJobDefinition pigJobFromCommand = newSqoopJobDefinitionCommand.Output.ElementAt(0);
Assert.AreEqual(pigJobDefinition.Command, pigJobFromCommand.Command);
}
public void ICanCallThe_New_HDInsightSqoopJobDefinitionCmdlet()
{
var sqoopJobDefinition = new SqoopJobCreateParameters { Command = "show tables" };
using (IRunspace runspace = this.GetPowerShellRunspace())
{
IPipelineResult results =
runspace.NewPipeline()
.AddCommand(CmdletConstants.NewAzureHDInsightSqoopJobDefinition)
.WithParameter(CmdletConstants.Command, sqoopJobDefinition.Command)
.Invoke();
Assert.AreEqual(1, results.Results.Count);
AzureHDInsightSqoopJobDefinition SqoopJobFromPowershell = results.Results.ToEnumerable<AzureHDInsightSqoopJobDefinition>().First();
Assert.AreEqual(sqoopJobDefinition.Command, SqoopJobFromPowershell.Command);
}
}
public void ICanCallThe_New_HDInsightSqoopJobDefinitionCmdlet_WithFileParameter()
{
var sqoopJobDefinition = new SqoopJobCreateParameters { File = TestConstants.WabsProtocolSchemeName + "filepath.hql" };
using (IRunspace runspace = this.GetPowerShellRunspace())
{
IPipelineResult results =
runspace.NewPipeline()
.AddCommand(CmdletConstants.NewAzureHDInsightSqoopJobDefinition)
.WithParameter(CmdletConstants.File, sqoopJobDefinition.File)
.Invoke();
Assert.AreEqual(1, results.Results.Count);
AzureHDInsightSqoopJobDefinition SqoopJobFromPowershell = results.Results.ToEnumerable<AzureHDInsightSqoopJobDefinition>().First();
Assert.AreEqual(sqoopJobDefinition.File, SqoopJobFromPowershell.File);
}
}
public Task<JobCreationResults> SubmitSqoopJob(SqoopJobCreateParameters sqoopJobCreateParameters)
{
this.SubmitSqoopJobCalled = true;
var job = new JobCreationResults() { JobId = JobId };
return Task.Run(() => job);
}
public void CanSerializeValidSqoopJobRequest_File()
{
var sqoopJob = new SqoopJobCreateParameters()
{
File = "file.sqoop"
};
var payloadConverter = new PayloadConverterBase();
var payload = payloadConverter.SerializeSqoopRequest("hadoop", sqoopJob);
Assert.IsTrue(payload.Contains(string.Format("{0}={1}", WebHCatConstants.File, Uri.EscapeDataString(sqoopJob.File))));
}
public Task<JobCreationResults> SubmitSqoopJob(SqoopJobCreateParameters sqoopJobCreateParameters)
{
if (sqoopJobCreateParameters == null)
{
throw new ArgumentNullException("sqoopJobCreateParameters");
}
var retval = this.CreateJobSuccessResult(new JobDetails()
{
Query = sqoopJobCreateParameters.Command,
StatusDirectory = sqoopJobCreateParameters.StatusFolder
},
string.Empty);
return Task.FromResult(retval);
}
public void ShouldNotSerializeSqoopJobName()
{
var sqoopJob = new SqoopJobCreateParameters();
sqoopJob.Command = "show tables";
var payloadConverter = new PayloadConverterBase();
var payload = payloadConverter.SerializeSqoopRequest("hadoop", sqoopJob);
Assert.IsFalse((payload.Contains(WebHCatConstants.DefineJobName)));
}
public void CanSerializeValidSqoopJobRequest_Command()
{
var sqoopJob = new SqoopJobCreateParameters()
{
Command = "show tables"
};
var payloadConverter = new PayloadConverterBase();
var payload = payloadConverter.SerializeSqoopRequest("hadoop", sqoopJob);
Assert.IsTrue(payload.Contains(string.Format("{0}={1}", WebHCatConstants.Command, Uri.EscapeDataString(sqoopJob.Command))));
}
public JobCreationResults CreateSqoopJob(SqoopJobCreateParameters sqoopJobCreateParameters)
{
return this.CreateSqoopJobAsync(sqoopJobCreateParameters).WaitForResult();
}
public virtual void ICanCallThe_NewSqoopJob_Then_Start_HDInsightJobsCmdlet()
{
var sqoopJobDefinition = new SqoopJobCreateParameters { Command = "load table from 'A'" };
using (IRunspace runspace = this.GetPowerShellRunspace())
{
IPipelineResult results =
runspace.NewPipeline()
.AddCommand(CmdletConstants.NewAzureHDInsightSqoopJobDefinition)
.WithParameter(CmdletConstants.Command, sqoopJobDefinition.Command)
.Invoke();
Assert.AreEqual(1, results.Results.Count);
AzureHDInsightSqoopJobDefinition sqoopJobFromPowershell = results.Results.ToEnumerable<AzureHDInsightSqoopJobDefinition>().First();
RunJobInPowershell(runspace, sqoopJobFromPowershell);
}
}
public JobCreationResults CreateSqoopJob(SqoopJobCreateParameters sqoopJobCreateParameters)
{
this.PrepareQueryJob(sqoopJobCreateParameters);
return this.CreateSqoopJobAsync(sqoopJobCreateParameters).WaitForResult();
}
public void ICannotCallThe_New_HDInsightSqoopJobDefinitionCmdlet_WithoutFileOrCommandParameter()
{
var sqoopJobDefinition = new SqoopJobCreateParameters { File = TestConstants.WabsProtocolSchemeName + "filepath.hql" };
try
{
using (IRunspace runspace = this.GetPowerShellRunspace())
{
runspace.NewPipeline().AddCommand(CmdletConstants.NewAzureHDInsightSqoopJobDefinition).Invoke();
Assert.Fail("test failed.");
}
}
catch (CmdletInvocationException invokeException)
{
var psArgumentException = invokeException.GetBaseException() as PSArgumentException;
Assert.IsNotNull(psArgumentException);
Assert.AreEqual("Either File or Command should be specified for Sqoop jobs.", psArgumentException.Message);
}
}
public void ICanCallThe_New_HDInsightSqoopJobDefinitionCmdlet_WithResources()
{
var sqoopJobDefinition = new SqoopJobCreateParameters { Command = "show tables" };
sqoopJobDefinition.Files.Add("pidata.txt");
sqoopJobDefinition.Files.Add("pidate2.txt");
using (IRunspace runspace = this.GetPowerShellRunspace())
{
IPipelineResult results =
runspace.NewPipeline()
.AddCommand(CmdletConstants.NewAzureHDInsightSqoopJobDefinition)
.WithParameter(CmdletConstants.Command, sqoopJobDefinition.Command)
.WithParameter(CmdletConstants.Files, sqoopJobDefinition.Files)
.Invoke();
Assert.AreEqual(1, results.Results.Count);
AzureHDInsightSqoopJobDefinition SqoopJobFromPowershell = results.Results.ToEnumerable<AzureHDInsightSqoopJobDefinition>().First();
Assert.AreEqual(sqoopJobDefinition.Command, SqoopJobFromPowershell.Command);
foreach (string file in sqoopJobDefinition.Files)
{
Assert.IsTrue(
SqoopJobFromPowershell.Files.Any(arg => string.Equals(file, arg)),
"Unable to find File '{0}' in value returned from powershell",
file);
}
}
}
/// <inheritdoc />
public JobCreationResults CreateSqoopJob(SqoopJobCreateParameters sqoopJobCreateParameters)
{
sqoopJobCreateParameters = this.PrepareQueryJob(sqoopJobCreateParameters);
return(this.CreateSqoopJobAsync(sqoopJobCreateParameters).WaitForResult());
}
public Task<JobCreationResults> CreateSqoopJobAsync(SqoopJobCreateParameters sqoopJobCreateParameters)
{
if (sqoopJobCreateParameters == null)
{
throw new ArgumentNullException("sqoopJobCreateParameters");
}
this.PrepareQueryJob(sqoopJobCreateParameters);
JobCreationResults retval =
this.CreateJobSuccessResult(
new JobDetails { Query = sqoopJobCreateParameters.Command, StatusDirectory = sqoopJobCreateParameters.StatusFolder }, string.Empty);
return TaskEx2.FromResult(retval);
}
public void CanCreateNewSqoopDefinition_WithResources()
{
var pigJobDefinition = new SqoopJobCreateParameters
{
Command = "load 'passwd' using SqoopStorage(':'); B = foreach A generate $0 as id;"
};
pigJobDefinition.Files.Add("pidata.txt");
pigJobDefinition.Files.Add("pidate2.txt");
INewAzureHDInsightSqoopJobDefinitionCommand newSqoopJobDefinitionCommand =
ServiceLocator.Instance.Locate<IAzureHDInsightCommandFactory>().CreateNewSqoopDefinition();
newSqoopJobDefinitionCommand.Command = pigJobDefinition.Command;
newSqoopJobDefinitionCommand.Files = pigJobDefinition.Files.ToArray();
newSqoopJobDefinitionCommand.EndProcessing();
AzureHDInsightSqoopJobDefinition pigJobFromCommand = newSqoopJobDefinitionCommand.Output.ElementAt(0);
Assert.AreEqual(pigJobDefinition.Command, pigJobFromCommand.Command);
foreach (string resource in pigJobDefinition.Files)
{
Assert.IsTrue(
pigJobFromCommand.Files.Any(arg => string.Equals(resource, arg)),
"Unable to find File '{0}' in value returned from command",
resource);
}
}
/// <inheritdoc />
public async Task<JobCreationResults> SubmitSqoopJob(SqoopJobCreateParameters sqoopJobCreateParameters)
{
var remoteClient = ServiceLocator.Instance.Locate<IRemoteHadoopJobSubmissionPocoClientFactory>().Create(this.remoteCreds, this.context, this.ignoreSslErrors, this.GetUserAgentString());
return await remoteClient.SubmitSqoopJob(sqoopJobCreateParameters);
}
