public void CanCreateNewSqoopDefinition_WithResources() { var pigJobDefinition = new SqoopJobCreateParameters { Command = "load 'passwd' using SqoopStorage(':'); B = foreach A generate $0 as id;" }; pigJobDefinition.Files.Add("pidata.txt"); pigJobDefinition.Files.Add("pidate2.txt"); INewAzureHDInsightSqoopJobDefinitionCommand newSqoopJobDefinitionCommand = ServiceLocator.Instance.Locate <IAzureHDInsightCommandFactory>().CreateNewSqoopDefinition(); newSqoopJobDefinitionCommand.Command = pigJobDefinition.Command; newSqoopJobDefinitionCommand.Files = pigJobDefinition.Files.ToArray(); newSqoopJobDefinitionCommand.EndProcessing(); AzureHDInsightSqoopJobDefinition pigJobFromCommand = newSqoopJobDefinitionCommand.Output.ElementAt(0); Assert.AreEqual(pigJobDefinition.Command, pigJobFromCommand.Command); foreach (string resource in pigJobDefinition.Files) { Assert.IsTrue( pigJobFromCommand.Files.Any(arg => string.Equals(resource, arg)), "Unable to find File '{0}' in value returned from command", resource); } }
public void ICanCallThe_New_HDInsightSqoopJobDefinitionCmdlet_WithResources() { var sqoopJobDefinition = new SqoopJobCreateParameters { Command = "show tables" }; sqoopJobDefinition.Files.Add("pidata.txt"); sqoopJobDefinition.Files.Add("pidate2.txt"); using (IRunspace runspace = this.GetPowerShellRunspace()) { IPipelineResult results = runspace.NewPipeline() .AddCommand(CmdletConstants.NewAzureHDInsightSqoopJobDefinition) .WithParameter(CmdletConstants.Command, sqoopJobDefinition.Command) .WithParameter(CmdletConstants.Files, sqoopJobDefinition.Files) .Invoke(); Assert.AreEqual(1, results.Results.Count); AzureHDInsightSqoopJobDefinition SqoopJobFromPowershell = results.Results.ToEnumerable <AzureHDInsightSqoopJobDefinition>().First(); Assert.AreEqual(sqoopJobDefinition.Command, SqoopJobFromPowershell.Command); foreach (string file in sqoopJobDefinition.Files) { Assert.IsTrue( SqoopJobFromPowershell.Files.Any(arg => string.Equals(file, arg)), "Unable to find File '{0}' in value returned from powershell", file); } } }
private static async Task <JobCreationResults> CreateSqoopJob( AzureHDInsightSqoopJobDefinition azureSqoopJobDefinition, IJobSubmissionClient client) { SqoopJobCreateParameters sqoopJobDefinition = azureSqoopJobDefinition.ToSqoopJobCreateParameters(); var jobCreationResults = await client.CreateSqoopJobAsync(sqoopJobDefinition); return(jobCreationResults); }
public Task <JobCreationResults> SubmitSqoopJob(SqoopJobCreateParameters sqoopJobCreateParameters) { this.SubmitSqoopJobCalled = true; var job = new JobCreationResults() { JobId = JobId }; return(Task.Run(() => job)); }
private static async Task <JobCreationResults> CreateSqoopJob( AzureHDInsightSqoopJobDefinition azureSqoopJobDefinition, IJobSubmissionClient client) { AssertQueryDoesNotContainRestrictedCharacters(azureSqoopJobDefinition.Command, "Command"); SqoopJobCreateParameters sqoopJobDefinition = azureSqoopJobDefinition.ToSqoopJobCreateParameters(); var jobCreationResults = await client.CreateSqoopJobAsync(sqoopJobDefinition); return(jobCreationResults); }
/// <summary> /// Creates a SDK object from the PSCmdlet object type. /// </summary> /// <returns>A SDK Sqoop job definition object.</returns> internal SqoopJobCreateParameters ToSqoopJobCreateParameters() { var soopJobDefinition = new SqoopJobCreateParameters { Command = this.Command, File = this.File, StatusFolder = this.StatusFolder }; if (this.Files.IsNotNull()) { soopJobDefinition.Files.AddRange(this.Files); } return soopJobDefinition; }
public void ShouldNotSerializeSqoopJobName() { var sqoopJob = new SqoopJobCreateParameters(); sqoopJob.Command = "show tables"; var payloadConverter = new PayloadConverterBase(); var payload = payloadConverter.SerializeSqoopRequest("hadoop", sqoopJob); Assert.IsFalse((payload.Contains(WebHCatConstants.DefineJobName))); }
public void CanSerializeValidSqoopJobRequest_File() { var sqoopJob = new SqoopJobCreateParameters() { File = "file.sqoop" }; var payloadConverter = new PayloadConverterBase(); var payload = payloadConverter.SerializeSqoopRequest("hadoop", sqoopJob); Assert.IsTrue(payload.Contains(string.Format("{0}={1}", WebHCatConstants.File, Uri.EscapeDataString(sqoopJob.File)))); }
public void CanSerializeValidSqoopJobRequest_Command() { var sqoopJob = new SqoopJobCreateParameters() { Command = "show tables" }; var payloadConverter = new PayloadConverterBase(); var payload = payloadConverter.SerializeSqoopRequest("hadoop", sqoopJob); Assert.IsTrue(payload.Contains(string.Format("{0}={1}", WebHCatConstants.Command, Uri.EscapeDataString(sqoopJob.Command)))); }
public async Task <JobCreationResults> SubmitSqoopJob(SqoopJobCreateParameters sqoopJobCreateParameters) { //NEIN: Any code modification here should add unit tests for this class var converter = new PayloadConverter(); var payload = converter.SerializeSqoopRequest(this.credentials.UserName, sqoopJobCreateParameters); var client = ServiceLocator.Instance.Locate <IHadoopRemoteJobSubmissionRestClientFactory>().Create(this.credentials, this.context, this.ignoreSslErrors, this.GetUserAgentString()); var result = await client.SubmitSqoopJob(payload); return(new JobCreationResults() { JobId = converter.DeserializeJobSubmissionResponse(result.Content) }); }
/// <summary> /// Creates a SDK object from the PSCmdlet object type. /// </summary> /// <returns>A SDK Sqoop job definition object.</returns> internal SqoopJobCreateParameters ToSqoopJobCreateParameters() { var soopJobDefinition = new SqoopJobCreateParameters { Command = this.Command, File = this.File, StatusFolder = this.StatusFolder }; if (this.Files.IsNotNull()) { soopJobDefinition.Files.AddRange(this.Files); } return(soopJobDefinition); }
public async Task<JobCreationResults> SubmitSqoopJob(SqoopJobCreateParameters sqoopJobCreateParameters) { var relative = new Uri( HadoopRemoteRestConstants.Pig + "?" + HadoopRemoteRestConstants.UserName + "=" + _connectionConfig.UserName.EscapeDataString(), UriKind.Relative); var requestContent = _converter.SerializeSqoopRequest(_connectionConfig.UserName, sqoopJobCreateParameters); var result = await MakeAsyncJobSubmissionRequest(relative, requestContent); return new JobCreationResults { JobId = _converter.DeserializeJobSubmissionResponse(result) }; }
public void CanCreateNewSqoopDefinition_WithFile() { var pigJobDefinition = new SqoopJobCreateParameters { File = "my local file" }; INewAzureHDInsightSqoopJobDefinitionCommand newSqoopJobDefinitionCommand = ServiceLocator.Instance.Locate<IAzureHDInsightCommandFactory>().CreateNewSqoopDefinition(); newSqoopJobDefinitionCommand.File = pigJobDefinition.File; newSqoopJobDefinitionCommand.EndProcessing(); AzureHDInsightSqoopJobDefinition pigJobFromCommand = newSqoopJobDefinitionCommand.Output.ElementAt(0); Assert.AreEqual(pigJobDefinition.File, pigJobFromCommand.File); }
public Task <JobCreationResults> CreateSqoopJobAsync(SqoopJobCreateParameters sqoopJobCreateParameters) { if (sqoopJobCreateParameters == null) { throw new ArgumentNullException("sqoopJobCreateParameters"); } JobCreationResults retval = this.CreateJobSuccessResult( new JobDetails { Query = sqoopJobCreateParameters.Command, StatusDirectory = sqoopJobCreateParameters.StatusFolder }, string.Empty); return(TaskEx2.FromResult(retval)); }
/// <inheritdoc/> public async Task <JobCreationResults> CreateSqoopJobAsync(SqoopJobCreateParameters sqoopJobCreateParameters) { sqoopJobCreateParameters = this.PrepareQueryJob(sqoopJobCreateParameters); var pocoClient = this.GetPocoClient(); try { return(await pocoClient.SubmitSqoopJob(sqoopJobCreateParameters)); } catch (UnauthorizedAccessException) { pocoClient = this.GetPocoClient(true); } return(await pocoClient.SubmitSqoopJob(sqoopJobCreateParameters)); }
public async Task <JobCreationResults> SubmitSqoopJob(SqoopJobCreateParameters sqoopJobCreateParameters) { var relative = new Uri( HadoopRemoteRestConstants.Pig + "?" + HadoopRemoteRestConstants.UserName + "=" + _connectionConfig.UserName.EscapeDataString(), UriKind.Relative); var requestContent = _converter.SerializeSqoopRequest(_connectionConfig.UserName, sqoopJobCreateParameters); var result = await MakeAsyncJobSubmissionRequest(relative, requestContent); return(new JobCreationResults { JobId = _converter.DeserializeJobSubmissionResponse(result) }); }
public void CanCreateNewSqoopDefinition() { var pigJobDefinition = new SqoopJobCreateParameters { Command = "load 'passwd' using SqoopStorage(':'); B = foreach A generate $0 as id;" }; INewAzureHDInsightSqoopJobDefinitionCommand newSqoopJobDefinitionCommand = ServiceLocator.Instance.Locate<IAzureHDInsightCommandFactory>().CreateNewSqoopDefinition(); newSqoopJobDefinitionCommand.Command = pigJobDefinition.Command; newSqoopJobDefinitionCommand.EndProcessing(); AzureHDInsightSqoopJobDefinition pigJobFromCommand = newSqoopJobDefinitionCommand.Output.ElementAt(0); Assert.AreEqual(pigJobDefinition.Command, pigJobFromCommand.Command); }
public void CanCreateNewSqoopDefinition_WithFile() { var pigJobDefinition = new SqoopJobCreateParameters { File = "my local file" }; INewAzureHDInsightSqoopJobDefinitionCommand newSqoopJobDefinitionCommand = ServiceLocator.Instance.Locate <IAzureHDInsightCommandFactory>().CreateNewSqoopDefinition(); newSqoopJobDefinitionCommand.File = pigJobDefinition.File; newSqoopJobDefinitionCommand.EndProcessing(); AzureHDInsightSqoopJobDefinition pigJobFromCommand = newSqoopJobDefinitionCommand.Output.ElementAt(0); Assert.AreEqual(pigJobDefinition.File, pigJobFromCommand.File); }
public void ICanCallThe_New_HDInsightSqoopJobDefinitionCmdlet_WithFileParameter() { var sqoopJobDefinition = new SqoopJobCreateParameters { File = TestConstants.WabsProtocolSchemeName + "filepath.hql" }; using (IRunspace runspace = this.GetPowerShellRunspace()) { IPipelineResult results = runspace.NewPipeline() .AddCommand(CmdletConstants.NewAzureHDInsightSqoopJobDefinition) .WithParameter(CmdletConstants.File, sqoopJobDefinition.File) .Invoke(); Assert.AreEqual(1, results.Results.Count); AzureHDInsightSqoopJobDefinition SqoopJobFromPowershell = results.Results.ToEnumerable<AzureHDInsightSqoopJobDefinition>().First(); Assert.AreEqual(sqoopJobDefinition.File, SqoopJobFromPowershell.File); } }
public void CanCreateNewSqoopDefinition() { var pigJobDefinition = new SqoopJobCreateParameters { Command = "load 'passwd' using SqoopStorage(':'); B = foreach A generate $0 as id;" }; INewAzureHDInsightSqoopJobDefinitionCommand newSqoopJobDefinitionCommand = ServiceLocator.Instance.Locate <IAzureHDInsightCommandFactory>().CreateNewSqoopDefinition(); newSqoopJobDefinitionCommand.Command = pigJobDefinition.Command; newSqoopJobDefinitionCommand.EndProcessing(); AzureHDInsightSqoopJobDefinition pigJobFromCommand = newSqoopJobDefinitionCommand.Output.ElementAt(0); Assert.AreEqual(pigJobDefinition.Command, pigJobFromCommand.Command); }
public void ICanCallThe_New_HDInsightSqoopJobDefinitionCmdlet() { var sqoopJobDefinition = new SqoopJobCreateParameters { Command = "show tables" }; using (IRunspace runspace = this.GetPowerShellRunspace()) { IPipelineResult results = runspace.NewPipeline() .AddCommand(CmdletConstants.NewAzureHDInsightSqoopJobDefinition) .WithParameter(CmdletConstants.Command, sqoopJobDefinition.Command) .Invoke(); Assert.AreEqual(1, results.Results.Count); AzureHDInsightSqoopJobDefinition SqoopJobFromPowershell = results.Results.ToEnumerable<AzureHDInsightSqoopJobDefinition>().First(); Assert.AreEqual(sqoopJobDefinition.Command, SqoopJobFromPowershell.Command); } }
public void ICanCallThe_New_HDInsightSqoopJobDefinitionCmdlet() { var sqoopJobDefinition = new SqoopJobCreateParameters { Command = "show tables" }; using (IRunspace runspace = this.GetPowerShellRunspace()) { IPipelineResult results = runspace.NewPipeline() .AddCommand(CmdletConstants.NewAzureHDInsightSqoopJobDefinition) .WithParameter(CmdletConstants.Command, sqoopJobDefinition.Command) .Invoke(); Assert.AreEqual(1, results.Results.Count); AzureHDInsightSqoopJobDefinition SqoopJobFromPowershell = results.Results.ToEnumerable <AzureHDInsightSqoopJobDefinition>().First(); Assert.AreEqual(sqoopJobDefinition.Command, SqoopJobFromPowershell.Command); } }
public void ICanCallThe_New_HDInsightSqoopJobDefinitionCmdlet_WithFileParameter() { var sqoopJobDefinition = new SqoopJobCreateParameters { File = TestConstants.WabsProtocolSchemeName + "filepath.hql" }; using (IRunspace runspace = this.GetPowerShellRunspace()) { IPipelineResult results = runspace.NewPipeline() .AddCommand(CmdletConstants.NewAzureHDInsightSqoopJobDefinition) .WithParameter(CmdletConstants.File, sqoopJobDefinition.File) .Invoke(); Assert.AreEqual(1, results.Results.Count); AzureHDInsightSqoopJobDefinition SqoopJobFromPowershell = results.Results.ToEnumerable <AzureHDInsightSqoopJobDefinition>().First(); Assert.AreEqual(sqoopJobDefinition.File, SqoopJobFromPowershell.File); } }
public virtual void ICanCallThe_NewSqoopJob_Then_Start_HDInsightJobsCmdlet() { var sqoopJobDefinition = new SqoopJobCreateParameters { Command = "load table from 'A'" }; using (IRunspace runspace = this.GetPowerShellRunspace()) { IPipelineResult results = runspace.NewPipeline() .AddCommand(CmdletConstants.NewAzureHDInsightSqoopJobDefinition) .WithParameter(CmdletConstants.Command, sqoopJobDefinition.Command) .Invoke(); Assert.AreEqual(1, results.Results.Count); AzureHDInsightSqoopJobDefinition sqoopJobFromPowershell = results.Results.ToEnumerable <AzureHDInsightSqoopJobDefinition>().First(); RunJobInPowershell(runspace, sqoopJobFromPowershell); } }
public void ICannotCallThe_New_HDInsightSqoopJobDefinitionCmdlet_WithoutFileOrCommandParameter() { var sqoopJobDefinition = new SqoopJobCreateParameters { File = TestConstants.WabsProtocolSchemeName + "filepath.hql" }; try { using (IRunspace runspace = this.GetPowerShellRunspace()) { runspace.NewPipeline().AddCommand(CmdletConstants.NewAzureHDInsightSqoopJobDefinition).Invoke(); Assert.Fail("test failed."); } } catch (CmdletInvocationException invokeException) { var psArgumentException = invokeException.GetBaseException() as PSArgumentException; Assert.IsNotNull(psArgumentException); Assert.AreEqual("Either File or Command should be specified for Sqoop jobs.", psArgumentException.Message); } }
public void CanCreateSDKObjectFromPowershellObject() { var sqoopJobDefinition = new AzureHDInsightSqoopJobDefinition { Command = "Import into sqlserver", File = "http://myfileshare.txt", StatusFolder = Guid.NewGuid().ToString(), }; sqoopJobDefinition.Arguments.Add("arg1"); sqoopJobDefinition.Files.Add("file1.sqoop"); SqoopJobCreateParameters sdkObject = sqoopJobDefinition.ToSqoopJobCreateParameters(); Assert.AreEqual(sqoopJobDefinition.StatusFolder, sdkObject.StatusFolder); Assert.AreEqual(sqoopJobDefinition.File, sdkObject.File); Assert.AreEqual(sqoopJobDefinition.Command, sdkObject.Command); foreach (string file in sqoopJobDefinition.Files) { Assert.IsTrue(sdkObject.Files.Contains(file), file); } }
/// <summary> /// Creates the payload for a Sqoop request. /// </summary> /// <param name="userName"> The user name.</param> /// <param name="details"> The details.</param> /// <returns>A string that represents the payload for the request.</returns> public string SerializeSqoopRequest(string userName, SqoopJobCreateParameters details) { details.ArgumentNotNull("details"); return this.SerializeQueryRequest(userName, details, string.Empty, details.File, details.Command, WebHCatConstants.Command, null, null); }
public JobCreationResults CreateSqoopJob(SqoopJobCreateParameters sqoopJobCreateParameters) { return this.CreateSqoopJobAsync(sqoopJobCreateParameters).WaitForResult(); }
/// <summary> /// Creates the payload for a Sqoop request. /// </summary> /// <param name="userName"> The user name.</param> /// <param name="details"> The details.</param> /// <returns>A string that represents the payload for the request.</returns> public string SerializeSqoopRequest(string userName, SqoopJobCreateParameters details) { details.ArgumentNotNull("details"); return(this.SerializeQueryRequest(userName, details, string.Empty, details.File, details.Command, WebHCatConstants.Command, null, null)); }
public void CanCreateNewSqoopDefinition_WithResources() { var pigJobDefinition = new SqoopJobCreateParameters { Command = "load 'passwd' using SqoopStorage(':'); B = foreach A generate $0 as id;" }; pigJobDefinition.Files.Add("pidata.txt"); pigJobDefinition.Files.Add("pidate2.txt"); INewAzureHDInsightSqoopJobDefinitionCommand newSqoopJobDefinitionCommand = ServiceLocator.Instance.Locate<IAzureHDInsightCommandFactory>().CreateNewSqoopDefinition(); newSqoopJobDefinitionCommand.Command = pigJobDefinition.Command; newSqoopJobDefinitionCommand.Files = pigJobDefinition.Files.ToArray(); newSqoopJobDefinitionCommand.EndProcessing(); AzureHDInsightSqoopJobDefinition pigJobFromCommand = newSqoopJobDefinitionCommand.Output.ElementAt(0); Assert.AreEqual(pigJobDefinition.Command, pigJobFromCommand.Command); foreach (string resource in pigJobDefinition.Files) { Assert.IsTrue( pigJobFromCommand.Files.Any(arg => string.Equals(resource, arg)), "Unable to find File '{0}' in value returned from command", resource); } }
public virtual void ICanCallThe_NewSqoopJob_Then_Start_HDInsightJobsCmdlet() { var sqoopJobDefinition = new SqoopJobCreateParameters { Command = "load table from 'A'" }; using (IRunspace runspace = this.GetPowerShellRunspace()) { IPipelineResult results = runspace.NewPipeline() .AddCommand(CmdletConstants.NewAzureHDInsightSqoopJobDefinition) .WithParameter(CmdletConstants.Command, sqoopJobDefinition.Command) .Invoke(); Assert.AreEqual(1, results.Results.Count); AzureHDInsightSqoopJobDefinition sqoopJobFromPowershell = results.Results.ToEnumerable<AzureHDInsightSqoopJobDefinition>().First(); RunJobInPowershell(runspace, sqoopJobFromPowershell); } }
public Task<JobCreationResults> CreateSqoopJobAsync(SqoopJobCreateParameters sqoopJobCreateParameters) { if (sqoopJobCreateParameters == null) { throw new ArgumentNullException("sqoopJobCreateParameters"); } JobCreationResults retval = this.CreateJobSuccessResult( new JobDetails { Query = sqoopJobCreateParameters.Command, StatusDirectory = sqoopJobCreateParameters.StatusFolder }, string.Empty); return TaskEx2.FromResult(retval); }
public JobCreationResults CreateSqoopJob(SqoopJobCreateParameters sqoopJobCreateParameters) { this.PrepareQueryJob(sqoopJobCreateParameters); return(this.CreateSqoopJobAsync(sqoopJobCreateParameters).WaitForResult()); }
public async Task<JobCreationResults> SubmitSqoopJob(SqoopJobCreateParameters sqoopJobCreateParameters) { //NEIN: Any code modification here should add unit tests for this class var converter = new PayloadConverter(); var payload = converter.SerializeSqoopRequest(this.credentials.UserName, sqoopJobCreateParameters); var client = ServiceLocator.Instance.Locate<IHadoopRemoteJobSubmissionRestClientFactory>().Create(this.credentials, this.context, this.ignoreSslErrors, this.GetUserAgentString()); var result = await client.SubmitSqoopJob(payload); return new JobCreationResults() { JobId = converter.DeserializeJobSubmissionResponse(result.Content) }; }
public void ICanCallThe_New_HDInsightSqoopJobDefinitionCmdlet_WithResources() { var sqoopJobDefinition = new SqoopJobCreateParameters { Command = "show tables" }; sqoopJobDefinition.Files.Add("pidata.txt"); sqoopJobDefinition.Files.Add("pidate2.txt"); using (IRunspace runspace = this.GetPowerShellRunspace()) { IPipelineResult results = runspace.NewPipeline() .AddCommand(CmdletConstants.NewAzureHDInsightSqoopJobDefinition) .WithParameter(CmdletConstants.Command, sqoopJobDefinition.Command) .WithParameter(CmdletConstants.Files, sqoopJobDefinition.Files) .Invoke(); Assert.AreEqual(1, results.Results.Count); AzureHDInsightSqoopJobDefinition SqoopJobFromPowershell = results.Results.ToEnumerable<AzureHDInsightSqoopJobDefinition>().First(); Assert.AreEqual(sqoopJobDefinition.Command, SqoopJobFromPowershell.Command); foreach (string file in sqoopJobDefinition.Files) { Assert.IsTrue( SqoopJobFromPowershell.Files.Any(arg => string.Equals(file, arg)), "Unable to find File '{0}' in value returned from powershell", file); } } }
public JobCreationResults CreateSqoopJob(SqoopJobCreateParameters sqoopJobCreateParameters) { return(this.CreateSqoopJobAsync(sqoopJobCreateParameters).WaitForResult()); }
/// <inheritdoc /> public async Task <JobCreationResults> SubmitSqoopJob(SqoopJobCreateParameters sqoopJobCreateParameters) { var remoteClient = ServiceLocator.Instance.Locate <IRemoteHadoopJobSubmissionPocoClientFactory>().Create(this.remoteCreds, this.context, this.ignoreSslErrors, this.GetUserAgentString()); return(await remoteClient.SubmitSqoopJob(sqoopJobCreateParameters)); }