public void ICanCallThe_New_HDInsightPigJobDefinitionCmdlet_WithArguments() { var pigJobDefinition = new PigJobCreateParameters { File = TestConstants.WabsProtocolSchemeName + "container@accountname/pigquery.q" }; pigJobDefinition.Arguments.Add("map.input.tasks=1000"); pigJobDefinition.Arguments.Add("map.input.reducers=1000"); using (IRunspace runspace = this.GetPowerShellRunspace()) { IPipelineResult results = runspace.NewPipeline() .AddCommand(CmdletConstants.NewAzureHDInsightPigJobDefinition) .WithParameter(CmdletConstants.File, pigJobDefinition.File) .WithParameter(CmdletConstants.Arguments, pigJobDefinition.Arguments) .Invoke(); Assert.AreEqual(1, results.Results.Count); AzureHDInsightPigJobDefinition pigJobFromPowershell = results.Results.ToEnumerable<AzureHDInsightPigJobDefinition>().First(); Assert.AreEqual(pigJobDefinition.File, pigJobFromPowershell.File); foreach (string argument in pigJobDefinition.Arguments) { Assert.IsTrue( pigJobFromPowershell.Arguments.Any(arg => string.Equals(argument, arg)), string.Format("Unable to find parameter '{0}' in value returned from powershell", argument)); } } }
public void CanCreateNewPigDefinition_WithArguments() { var pigJobDefinition = new PigJobCreateParameters { Query = "load 'passwd' using PigStorage(':'); B = foreach A generate $0 as id;" }; pigJobDefinition.Arguments.Add("map.input.tasks=1000"); pigJobDefinition.Arguments.Add("map.input.reducers=1000"); INewAzureHDInsightPigJobDefinitionCommand newPigJobDefinitionCommand = ServiceLocator.Instance.Locate<IAzureHDInsightCommandFactory>().CreateNewPigJobDefinition(); newPigJobDefinitionCommand.Query = pigJobDefinition.Query; newPigJobDefinitionCommand.Arguments = pigJobDefinition.Arguments.ToArray(); newPigJobDefinitionCommand.EndProcessing(); AzureHDInsightPigJobDefinition pigJobFromCommand = newPigJobDefinitionCommand.Output.ElementAt(0); Assert.AreEqual(pigJobDefinition.Query, pigJobFromCommand.Query); foreach (string parameter in pigJobDefinition.Arguments) { Assert.IsTrue( pigJobFromCommand.Arguments.Any(arg => string.Equals(parameter, arg)), "Unable to find parameter '{0}' in value returned from command", parameter); } }
public void CanCreateNewPigDefinition() { var pigJobDefinition = new PigJobCreateParameters { Query = "load 'passwd' using PigStorage(':'); B = foreach A generate $0 as id;" }; INewAzureHDInsightPigJobDefinitionCommand newPigJobDefinitionCommand = ServiceLocator.Instance.Locate<IAzureHDInsightCommandFactory>().CreateNewPigJobDefinition(); newPigJobDefinitionCommand.Query = pigJobDefinition.Query; newPigJobDefinitionCommand.EndProcessing(); AzureHDInsightPigJobDefinition pigJobFromCommand = newPigJobDefinitionCommand.Output.ElementAt(0); Assert.AreEqual(pigJobDefinition.Query, pigJobFromCommand.Query); }
public virtual void ICanCallThe_NewPigJob_Then_Start_HDInsightJobsCmdlet() { var pigJobDefinition = new PigJobCreateParameters { Query = "load table from 'A'" }; using (IRunspace runspace = this.GetPowerShellRunspace()) { IPipelineResult results = runspace.NewPipeline() .AddCommand(CmdletConstants.NewAzureHDInsightPigJobDefinition) .WithParameter(CmdletConstants.Query, pigJobDefinition.Query) .Invoke(); Assert.AreEqual(1, results.Results.Count); AzureHDInsightPigJobDefinition pigJobFromPowershell = results.Results.ToEnumerable<AzureHDInsightPigJobDefinition>().First(); RunJobInPowershell(runspace, pigJobFromPowershell); } }
public void ICanCallThe_New_HDInsightPigJobDefinitionCmdlet() { var pigJobDefinition = new PigJobCreateParameters { Query = "load 'passwd' using PigStorage(':'); B = foreach A generate $0 as id;" }; using (IRunspace runspace = this.GetPowerShellRunspace()) { IPipelineResult results = runspace.NewPipeline() .AddCommand(CmdletConstants.NewAzureHDInsightPigJobDefinition) .WithParameter(CmdletConstants.Query, pigJobDefinition.Query) .Invoke(); Assert.AreEqual(1, results.Results.Count); AzureHDInsightPigJobDefinition pigJobFromPowershell = results.Results.ToEnumerable<AzureHDInsightPigJobDefinition>().First(); Assert.AreEqual(pigJobDefinition.Query, pigJobFromPowershell.Query); } }
public void CanSerializeValidPigJobRequest() { var pigJob = new PigJobCreateParameters() { Query = "show tables" }; var payloadConverter = new PayloadConverterBase(); var payload = payloadConverter.SerializePigRequest("hadoop", pigJob); Assert.IsTrue(payload.Contains(string.Format("{0}={1}", WebHCatConstants.Execute, Uri.EscapeDataString(pigJob.Query)))); }
public void ShouldNotSerializePigJobName() { var pigJob = new PigJobCreateParameters(); pigJob.Arguments.Add("16"); pigJob.Query = "show tables"; pigJob.Arguments.Add("10000"); var payloadConverter = new PayloadConverterBase(); var payload = payloadConverter.SerializePigRequest("hadoop", pigJob); Assert.IsFalse((payload.Contains(WebHCatConstants.DefineJobName))); }
public void CanSerializeValidJobRequest_Arguments() { var pigJob = new PigJobCreateParameters(); pigJob.Arguments.Add("16"); pigJob.Query = "show tables"; pigJob.Arguments.Add("10000"); var payloadConverter = new PayloadConverterBase(); var payload = payloadConverter.SerializePigRequest("hadoop", pigJob); foreach (var argument in pigJob.Arguments) { Assert.IsTrue(payload.Contains(string.Format("{0}={1}", WebHCatConstants.Arg, argument))); } }
/// <inheritdoc /> public async Task<JobCreationResults> SubmitPigJob(PigJobCreateParameters pigJobCreateParameters) { var remoteClient = ServiceLocator.Instance.Locate<IRemoteHadoopJobSubmissionPocoClientFactory>().Create(this.remoteCreds, this.context, this.ignoreSslErrors, this.GetUserAgentString()); return await remoteClient.SubmitPigJob(pigJobCreateParameters); }
public Task<JobCreationResults> SubmitPigJob(PigJobCreateParameters pigJobCreateParameters) { throw new System.NotImplementedException(); }
public void ICanCallThe_New_HDInsightPigJobDefinitionCmdlet_WithResources() { var pigJobDefinition = new PigJobCreateParameters { Query = "load 'passwd' using PigStorage(':'); B = foreach A generate $0 as id;" }; pigJobDefinition.Files.Add("pidata.txt"); pigJobDefinition.Files.Add("pidate2.txt"); using (IRunspace runspace = this.GetPowerShellRunspace()) { IPipelineResult results = runspace.NewPipeline() .AddCommand(CmdletConstants.NewAzureHDInsightPigJobDefinition) .WithParameter(CmdletConstants.Query, pigJobDefinition.Query) .WithParameter(CmdletConstants.Files, pigJobDefinition.Files) .Invoke(); Assert.AreEqual(1, results.Results.Count); AzureHDInsightPigJobDefinition pigJobFromPowershell = results.Results.ToEnumerable<AzureHDInsightPigJobDefinition>().First(); Assert.AreEqual(pigJobDefinition.Query, pigJobFromPowershell.Query); foreach (string file in pigJobDefinition.Files) { Assert.IsTrue( pigJobFromPowershell.Files.Any(arg => string.Equals(file, arg)), "Unable to find File '{0}' in value returned from powershell", file); } } }
public void ICanCallThe_New_HDInsightPigJobDefinitionCmdlet_WithQueryFile() { var pigJobDefinition = new PigJobCreateParameters { File = TestConstants.WabsProtocolSchemeName + "container@accountname/pigquery.q" }; pigJobDefinition.Arguments.Add("map.input.tasks=1000"); pigJobDefinition.Arguments.Add("map.input.reducers=1000"); using (IRunspace runspace = this.GetPowerShellRunspace()) { IPipelineResult results = runspace.NewPipeline() .AddCommand(CmdletConstants.NewAzureHDInsightPigJobDefinition) .WithParameter(CmdletConstants.File, pigJobDefinition.File) .WithParameter(CmdletConstants.Arguments, pigJobDefinition.Arguments) .Invoke(); Assert.AreEqual(1, results.Results.Count); AzureHDInsightPigJobDefinition pigJobFromPowershell = results.Results.ToEnumerable<AzureHDInsightPigJobDefinition>().First(); Assert.AreEqual(pigJobDefinition.File, pigJobFromPowershell.File); } }
public Task<JobCreationResults> CreatePigJobAsync(PigJobCreateParameters pigJobCreateParameters) { if (pigJobCreateParameters == null) { throw new ArgumentNullException("pigJobCreateParameters"); } this.PrepareQueryJob(pigJobCreateParameters); JobCreationResults retval = this.CreateJobSuccessResult( new JobDetails { Query = pigJobCreateParameters.Query, StatusDirectory = pigJobCreateParameters.StatusFolder }, string.Empty); return TaskEx2.FromResult(retval); }
public JobCreationResults CreatePigJob(PigJobCreateParameters pigJobCreateParameters) { this.PrepareQueryJob(pigJobCreateParameters); return this.CreatePigJobAsync(pigJobCreateParameters).WaitForResult(); }
public JobCreationResults CreatePigJob(PigJobCreateParameters pigJobCreateParameters) { return this.CreatePigJobAsync(pigJobCreateParameters).WaitForResult(); }
public void CanSerializeValidPigJobRequest_WithFile() { var pigJob = new PigJobCreateParameters() { File = Constants.WabsProtocolSchemeName + "filepath.hql" }; var payloadConverter = new PayloadConverterBase(); var payload = payloadConverter.SerializePigRequest("hadoop", pigJob); Assert.IsTrue(payload.Contains(string.Format("{0}={1}", WebHCatConstants.File, Uri.EscapeDataString(pigJob.File)))); }
public Task<JobCreationResults> SubmitPigJob(PigJobCreateParameters pigJobCreateParameters) { if (pigJobCreateParameters == null) { throw new ArgumentNullException("pigJobCreateParameters"); } var retval = this.CreateJobSuccessResult(new JobDetails() { Query = pigJobCreateParameters.Query, StatusDirectory = pigJobCreateParameters.StatusFolder }, string.Empty); return Task.FromResult(retval); }
public void CanCreateNewPigJob_StartJob() { var pigJobDefinition = new PigJobCreateParameters { Query = "load table from 'A'" }; INewAzureHDInsightPigJobDefinitionCommand newMapReduceJobDefinitionCommand = ServiceLocator.Instance.Locate<IAzureHDInsightCommandFactory>().CreateNewPigJobDefinition(); newMapReduceJobDefinitionCommand.Query = pigJobDefinition.Query; newMapReduceJobDefinitionCommand.EndProcessing(); AzureHDInsightPigJobDefinition pigJobFromCommand = newMapReduceJobDefinitionCommand.Output.ElementAt(0); TestJobStart(pigJobFromCommand); }
public void CannotCreateNewPigJob_WithRestrictedCharacters_StartJob() { var pigJobDefinition = new PigJobCreateParameters { Query = "load table from 'A' %" }; INewAzureHDInsightPigJobDefinitionCommand newMapReduceJobDefinitionCommand = ServiceLocator.Instance.Locate<IAzureHDInsightCommandFactory>().CreateNewPigJobDefinition(); newMapReduceJobDefinitionCommand.Query = pigJobDefinition.Query; newMapReduceJobDefinitionCommand.EndProcessing(); AzureHDInsightPigJobDefinition pigJobFromCommand = newMapReduceJobDefinitionCommand.Output.ElementAt(0); try { TestJobStart(pigJobFromCommand); Assert.Fail(); } catch (AggregateException aggregateException) { var invalidOperationException = aggregateException.GetBaseException() as InvalidOperationException; Assert.IsNotNull(invalidOperationException); Assert.AreEqual("Query text contains restricted character '%', please upload the query to a file in storage and re-submit the job using the -File parameter", invalidOperationException.Message); } }
private static void SubmitJobs() { // Get HDInsight cluster configuration settings string clusterName = ConfigurationManager.AppSettings["ClusterName"]; string userName = ConfigurationManager.AppSettings["UserName"]; string password = ConfigurationManager.AppSettings["Password"]; // Create basic authentication credential for cluster BasicAuthCredential bcred = new BasicAuthCredential(); bcred.Server = new Uri("https://" + clusterName + ".azurehdinsight.net"); bcred.UserName = userName; bcred.Password = password; // Create and submit Pig job PigJobCreateParameters pigJob = new PigJobCreateParameters() { StatusFolder = "/data/racecar/scripts/processdatastatus", File = "/data/racecar/scripts/processdata.pig" }; var pigJobClient = JobSubmissionClientFactory.Connect(bcred); JobCreationResults pigJobResults = pigJobClient.CreatePigJob(pigJob); WaitForJobCompletion(pigJobResults, pigJobClient); // Create and submit Hive job HiveJobCreateParameters hiveJob = new HiveJobCreateParameters() { JobName = "Create Hive tables", StatusFolder = "/data/racecar/scripts/createtablestatus", File = "/data/racecar/scripts/createtables.hql" }; var hiveJobClient = JobSubmissionClientFactory.Connect(bcred); JobCreationResults hiveJobResults = hiveJobClient.CreateHiveJob(hiveJob); WaitForJobCompletion(hiveJobResults, hiveJobClient); }
public void CanCreateNewPigDefinition_WithResources() { var pigJobDefinition = new PigJobCreateParameters { Query = "load 'passwd' using PigStorage(':'); B = foreach A generate $0 as id;" }; pigJobDefinition.Files.Add("pidata.txt"); pigJobDefinition.Files.Add("pidate2.txt"); INewAzureHDInsightPigJobDefinitionCommand newPigJobDefinitionCommand = ServiceLocator.Instance.Locate<IAzureHDInsightCommandFactory>().CreateNewPigJobDefinition(); newPigJobDefinitionCommand.Query = pigJobDefinition.Query; newPigJobDefinitionCommand.Files = pigJobDefinition.Files.ToArray(); newPigJobDefinitionCommand.EndProcessing(); AzureHDInsightPigJobDefinition pigJobFromCommand = newPigJobDefinitionCommand.Output.ElementAt(0); Assert.AreEqual(pigJobDefinition.Query, pigJobFromCommand.Query); foreach (string resource in pigJobDefinition.Files) { Assert.IsTrue( pigJobFromCommand.Files.Any(arg => string.Equals(resource, arg)), "Unable to find File '{0}' in value returned from command", resource); } }
public void CannotCreateNewPigJob_WithRestrictedCharacters_StartJob() { var pigJobDefinition = new PigJobCreateParameters { Query = "load table from 'A' %" }; INewAzureHDInsightPigJobDefinitionCommand newMapReduceJobDefinitionCommand = ServiceLocator.Instance.Locate<IAzureHDInsightCommandFactory>().CreateNewPigJobDefinition(); newMapReduceJobDefinitionCommand.Query = pigJobDefinition.Query; newMapReduceJobDefinitionCommand.EndProcessing(); AzureHDInsightPigJobDefinition pigJobFromCommand = newMapReduceJobDefinitionCommand.Output.ElementAt(0); try { TestJobStart(pigJobFromCommand); Assert.Fail(); } catch (AggregateException aggregateException) { var invalidOperationException = aggregateException.GetBaseException() as InvalidOperationException; Assert.IsNotNull(invalidOperationException); Assert.IsTrue(invalidOperationException.Message.Contains("Query contains restricted character :'%'"), "Exception not thrown for special character"); } }