public void CanCreateNewPigDefinition_WithArguments() { var pigJobDefinition = new PigJobCreateParameters { Query = "load 'passwd' using PigStorage(':'); B = foreach A generate $0 as id;" }; pigJobDefinition.Arguments.Add("map.input.tasks=1000"); pigJobDefinition.Arguments.Add("map.input.reducers=1000"); INewAzureHDInsightPigJobDefinitionCommand newPigJobDefinitionCommand = ServiceLocator.Instance.Locate<IAzureHDInsightCommandFactory>().CreateNewPigJobDefinition(); newPigJobDefinitionCommand.Query = pigJobDefinition.Query; newPigJobDefinitionCommand.Arguments = pigJobDefinition.Arguments.ToArray(); newPigJobDefinitionCommand.EndProcessing(); AzureHDInsightPigJobDefinition pigJobFromCommand = newPigJobDefinitionCommand.Output.ElementAt(0); Assert.AreEqual(pigJobDefinition.Query, pigJobFromCommand.Query); foreach (string parameter in pigJobDefinition.Arguments) { Assert.IsTrue( pigJobFromCommand.Arguments.Any(arg => string.Equals(parameter, arg)), "Unable to find parameter '{0}' in value returned from command", parameter); } }
public void ICanCallThe_New_HDInsightPigJobDefinitionCmdlet_WithArguments() { var pigJobDefinition = new PigJobCreateParameters { File = TestConstants.WabsProtocolSchemeName + "container@accountname/pigquery.q" }; pigJobDefinition.Arguments.Add("map.input.tasks=1000"); pigJobDefinition.Arguments.Add("map.input.reducers=1000"); using (IRunspace runspace = this.GetPowerShellRunspace()) { IPipelineResult results = runspace.NewPipeline() .AddCommand(CmdletConstants.NewAzureHDInsightPigJobDefinition) .WithParameter(CmdletConstants.File, pigJobDefinition.File) .WithParameter(CmdletConstants.Arguments, pigJobDefinition.Arguments) .Invoke(); Assert.AreEqual(1, results.Results.Count); AzureHDInsightPigJobDefinition pigJobFromPowershell = results.Results.ToEnumerable<AzureHDInsightPigJobDefinition>().First(); Assert.AreEqual(pigJobDefinition.File, pigJobFromPowershell.File); foreach (string argument in pigJobDefinition.Arguments) { Assert.IsTrue( pigJobFromPowershell.Arguments.Any(arg => string.Equals(argument, arg)), string.Format("Unable to find parameter '{0}' in value returned from powershell", argument)); } } }
public async Task<JobCreationResults> SubmitPigJob(PigJobCreateParameters pigJobCreateParameters) { var relative = new Uri( HadoopRemoteRestConstants.MapReduceStreaming + "?" + HadoopRemoteRestConstants.UserName + "=" + _connectionConfig.UserName.EscapeDataString(), UriKind.Relative); var requestContent = _converter.SerializePigRequest(_connectionConfig.UserName, pigJobCreateParameters); var result = await MakeAsyncJobSubmissionRequest(relative, requestContent); return new JobCreationResults {JobId = _converter.DeserializeJobSubmissionResponse(result)}; }
public void CanCreateNewPigDefinition() { var pigJobDefinition = new PigJobCreateParameters { Query = "load 'passwd' using PigStorage(':'); B = foreach A generate $0 as id;" }; INewAzureHDInsightPigJobDefinitionCommand newPigJobDefinitionCommand = ServiceLocator.Instance.Locate<IAzureHDInsightCommandFactory>().CreateNewPigJobDefinition(); newPigJobDefinitionCommand.Query = pigJobDefinition.Query; newPigJobDefinitionCommand.EndProcessing(); AzureHDInsightPigJobDefinition pigJobFromCommand = newPigJobDefinitionCommand.Output.ElementAt(0); Assert.AreEqual(pigJobDefinition.Query, pigJobFromCommand.Query); }
public virtual void ICanCallThe_NewPigJob_Then_Start_HDInsightJobsCmdlet() { var pigJobDefinition = new PigJobCreateParameters { Query = "load table from 'A'" }; using (IRunspace runspace = this.GetPowerShellRunspace()) { IPipelineResult results = runspace.NewPipeline() .AddCommand(CmdletConstants.NewAzureHDInsightPigJobDefinition) .WithParameter(CmdletConstants.Query, pigJobDefinition.Query) .Invoke(); Assert.AreEqual(1, results.Results.Count); AzureHDInsightPigJobDefinition pigJobFromPowershell = results.Results.ToEnumerable<AzureHDInsightPigJobDefinition>().First(); RunJobInPowershell(runspace, pigJobFromPowershell); } }
public void ICanCallThe_New_HDInsightPigJobDefinitionCmdlet() { var pigJobDefinition = new PigJobCreateParameters { Query = "load 'passwd' using PigStorage(':'); B = foreach A generate $0 as id;" }; using (IRunspace runspace = this.GetPowerShellRunspace()) { IPipelineResult results = runspace.NewPipeline() .AddCommand(CmdletConstants.NewAzureHDInsightPigJobDefinition) .WithParameter(CmdletConstants.Query, pigJobDefinition.Query) .Invoke(); Assert.AreEqual(1, results.Results.Count); AzureHDInsightPigJobDefinition pigJobFromPowershell = results.Results.ToEnumerable<AzureHDInsightPigJobDefinition>().First(); Assert.AreEqual(pigJobDefinition.Query, pigJobFromPowershell.Query); } }
public void ICanCallThe_New_HDInsightPigJobDefinitionCmdlet_WithResources() { var pigJobDefinition = new PigJobCreateParameters { Query = "load 'passwd' using PigStorage(':'); B = foreach A generate $0 as id;" }; pigJobDefinition.Files.Add("pidata.txt"); pigJobDefinition.Files.Add("pidate2.txt"); using (IRunspace runspace = this.GetPowerShellRunspace()) { IPipelineResult results = runspace.NewPipeline() .AddCommand(CmdletConstants.NewAzureHDInsightPigJobDefinition) .WithParameter(CmdletConstants.Query, pigJobDefinition.Query) .WithParameter(CmdletConstants.Files, pigJobDefinition.Files) .Invoke(); Assert.AreEqual(1, results.Results.Count); AzureHDInsightPigJobDefinition pigJobFromPowershell = results.Results.ToEnumerable<AzureHDInsightPigJobDefinition>().First(); Assert.AreEqual(pigJobDefinition.Query, pigJobFromPowershell.Query); foreach (string file in pigJobDefinition.Files) { Assert.IsTrue( pigJobFromPowershell.Files.Any(arg => string.Equals(file, arg)), "Unable to find File '{0}' in value returned from powershell", file); } } }
public void ICanCallThe_New_HDInsightPigJobDefinitionCmdlet_WithQueryFile() { var pigJobDefinition = new PigJobCreateParameters { File = TestConstants.WabsProtocolSchemeName + "container@accountname/pigquery.q" }; pigJobDefinition.Arguments.Add("map.input.tasks=1000"); pigJobDefinition.Arguments.Add("map.input.reducers=1000"); using (IRunspace runspace = this.GetPowerShellRunspace()) { IPipelineResult results = runspace.NewPipeline() .AddCommand(CmdletConstants.NewAzureHDInsightPigJobDefinition) .WithParameter(CmdletConstants.File, pigJobDefinition.File) .WithParameter(CmdletConstants.Arguments, pigJobDefinition.Arguments) .Invoke(); Assert.AreEqual(1, results.Results.Count); AzureHDInsightPigJobDefinition pigJobFromPowershell = results.Results.ToEnumerable<AzureHDInsightPigJobDefinition>().First(); Assert.AreEqual(pigJobDefinition.File, pigJobFromPowershell.File); } }
public void CannotCreateNewPigJob_WithRestrictedCharacters_StartJob() { var pigJobDefinition = new PigJobCreateParameters { Query = "load table from 'A' %" }; INewAzureHDInsightPigJobDefinitionCommand newMapReduceJobDefinitionCommand = ServiceLocator.Instance.Locate<IAzureHDInsightCommandFactory>().CreateNewPigJobDefinition(); newMapReduceJobDefinitionCommand.Query = pigJobDefinition.Query; newMapReduceJobDefinitionCommand.EndProcessing(); AzureHDInsightPigJobDefinition pigJobFromCommand = newMapReduceJobDefinitionCommand.Output.ElementAt(0); try { TestJobStart(pigJobFromCommand); Assert.Fail(); } catch (AggregateException aggregateException) { var invalidOperationException = aggregateException.GetBaseException() as InvalidOperationException; Assert.IsNotNull(invalidOperationException); Assert.AreEqual("Query text contains restricted character '%', please upload the query to a file in storage and re-submit the job using the -File parameter", invalidOperationException.Message); } }
public void CanCreateNewPigJob_StartJob() { var pigJobDefinition = new PigJobCreateParameters { Query = "load table from 'A'" }; INewAzureHDInsightPigJobDefinitionCommand newMapReduceJobDefinitionCommand = ServiceLocator.Instance.Locate<IAzureHDInsightCommandFactory>().CreateNewPigJobDefinition(); newMapReduceJobDefinitionCommand.Query = pigJobDefinition.Query; newMapReduceJobDefinitionCommand.EndProcessing(); AzureHDInsightPigJobDefinition pigJobFromCommand = newMapReduceJobDefinitionCommand.Output.ElementAt(0); TestJobStart(pigJobFromCommand); }
public JobCreationResults CreatePigJob(PigJobCreateParameters pigJobCreateParameters) { return(this.CreatePigJobAsync(pigJobCreateParameters).WaitForResult()); }
public JobCreationResults CreatePigJob(PigJobCreateParameters pigJobCreateParameters) { return this.CreatePigJobAsync(pigJobCreateParameters).WaitForResult(); }
public void CanCreateNewPigDefinition_WithResources() { var pigJobDefinition = new PigJobCreateParameters { Query = "load 'passwd' using PigStorage(':'); B = foreach A generate $0 as id;" }; pigJobDefinition.Files.Add("pidata.txt"); pigJobDefinition.Files.Add("pidate2.txt"); INewAzureHDInsightPigJobDefinitionCommand newPigJobDefinitionCommand = ServiceLocator.Instance.Locate<IAzureHDInsightCommandFactory>().CreateNewPigJobDefinition(); newPigJobDefinitionCommand.Query = pigJobDefinition.Query; newPigJobDefinitionCommand.Files = pigJobDefinition.Files.ToArray(); newPigJobDefinitionCommand.EndProcessing(); AzureHDInsightPigJobDefinition pigJobFromCommand = newPigJobDefinitionCommand.Output.ElementAt(0); Assert.AreEqual(pigJobDefinition.Query, pigJobFromCommand.Query); foreach (string resource in pigJobDefinition.Files) { Assert.IsTrue( pigJobFromCommand.Files.Any(arg => string.Equals(resource, arg)), "Unable to find File '{0}' in value returned from command", resource); } }
/// <summary> /// Creates the payload for a Pig request. /// </summary> /// <param name="userName"> The user name.</param> /// <param name="details"> The details.</param> /// <returns>A string that represents the payload for the request.</returns> public string SerializePigRequest(string userName, PigJobCreateParameters details) { details.ArgumentNotNull("details"); return this.SerializeQueryRequest(userName, details, string.Empty, details.File, details.Query, WebHCatConstants.Execute, details.Arguments, null); }
/// <inheritdoc /> public async Task<JobCreationResults> SubmitPigJob(PigJobCreateParameters pigJobCreateParameters) { //NEIN: Any code modification here should add unit tests for this class var converter = new PayloadConverter(); var payload = converter.SerializePigRequest(this.credentials.UserName, pigJobCreateParameters); var client = ServiceLocator.Instance.Locate<IHadoopRemoteJobSubmissionRestClientFactory>().Create(this.credentials, this.context, this.ignoreSslErrors, this.GetUserAgentString()); var result = await client.SubmitPigJob(payload); return new JobCreationResults() { JobId = converter.DeserializeJobSubmissionResponse(result.Content) }; }
public Task<JobCreationResults> CreatePigJobAsync(PigJobCreateParameters pigJobCreateParameters) { if (pigJobCreateParameters == null) { throw new ArgumentNullException("pigJobCreateParameters"); } JobCreationResults retval = this.CreateJobSuccessResult( new JobDetails { Query = pigJobCreateParameters.Query, StatusDirectory = pigJobCreateParameters.StatusFolder }, string.Empty); return TaskEx2.FromResult(retval); }
/// <inheritdoc /> public async Task <JobCreationResults> SubmitPigJob(PigJobCreateParameters pigJobCreateParameters) { var remoteClient = ServiceLocator.Instance.Locate <IRemoteHadoopJobSubmissionPocoClientFactory>().Create(this.remoteCreds, this.context, this.ignoreSslErrors, this.GetUserAgentString()); return(await remoteClient.SubmitPigJob(pigJobCreateParameters)); }