public void ICanCallThe_New_HDInsightStreamingMapReduceJobDefinitionCmdlet() { var streamingMapReduceJobDefinition = new StreamingMapReduceJobCreateParameters { JobName = "pi estimation jobDetails", Input = TestConstants.WabsProtocolSchemeName + "input", Output = TestConstants.WabsProtocolSchemeName + "input", Mapper = TestConstants.WabsProtocolSchemeName + "combiner", Reducer = TestConstants.WabsProtocolSchemeName + "combiner", StatusFolder = TestConstants.WabsProtocolSchemeName + "someotherlocation" }; using (IRunspace runspace = this.GetPowerShellRunspace()) { IPipelineResult results = runspace.NewPipeline() .AddCommand(CmdletConstants.NewAzureHDInsightStreamingMapReduceJobDefinition) .WithParameter(CmdletConstants.JobName, streamingMapReduceJobDefinition.JobName) .WithParameter(CmdletConstants.Input, streamingMapReduceJobDefinition.Input) .WithParameter(CmdletConstants.Output, streamingMapReduceJobDefinition.Output) .WithParameter(CmdletConstants.Mapper, streamingMapReduceJobDefinition.Mapper) .WithParameter(CmdletConstants.Reducer, streamingMapReduceJobDefinition.Reducer) .WithParameter(CmdletConstants.StatusFolder, streamingMapReduceJobDefinition.StatusFolder) .Invoke(); Assert.AreEqual(1, results.Results.Count); AzureHDInsightStreamingMapReduceJobDefinition streamingMapReduceJobFromPowershell = results.Results.ToEnumerable<AzureHDInsightStreamingMapReduceJobDefinition>().First(); AssertJobDefinitionsEqual(streamingMapReduceJobDefinition, streamingMapReduceJobFromPowershell); } }
/// <inheritdoc /> public Task <JobCreationResults> CreateStreamingJobAsync(StreamingMapReduceJobCreateParameters streamingMapReduceJobCreateParameters) { var factory = ServiceLocator.Instance.Locate <IRemoteHadoopJobSubmissionPocoClientFactory>(); var pocoClient = factory.Create(this.credentials, this.Context, this.IgnoreSslErrors, this.GetCustomUserAgent()); return(pocoClient.SubmitStreamingJob(streamingMapReduceJobCreateParameters)); }
public Task<JobCreationResults> SubmitStreamingJob(StreamingMapReduceJobCreateParameters pigJobCreateParameters) { throw new System.NotImplementedException(); }
public void ICanCallThe_New_HDInsightStreamingMapReduceJobDefinitionCmdlet_WithArguments() { var streamingMapReduceJobDefinition = new StreamingMapReduceJobCreateParameters { JobName = "pi estimation jobDetails", Input = TestConstants.WabsProtocolSchemeName + "input", Output = TestConstants.WabsProtocolSchemeName + "input", Mapper = TestConstants.WabsProtocolSchemeName + "combiner", Reducer = TestConstants.WabsProtocolSchemeName + "combiner", StatusFolder = TestConstants.WabsProtocolSchemeName + "someotherlocation" }; streamingMapReduceJobDefinition.Arguments.Add("16"); streamingMapReduceJobDefinition.Arguments.Add("10000"); using (IRunspace runspace = this.GetPowerShellRunspace()) { IPipelineResult results = runspace.NewPipeline() .AddCommand(CmdletConstants.NewAzureHDInsightStreamingMapReduceJobDefinition) .WithParameter(CmdletConstants.JobName, streamingMapReduceJobDefinition.JobName) .WithParameter(CmdletConstants.Input, streamingMapReduceJobDefinition.Input) .WithParameter(CmdletConstants.Output, streamingMapReduceJobDefinition.Output) .WithParameter(CmdletConstants.Mapper, streamingMapReduceJobDefinition.Mapper) .WithParameter(CmdletConstants.Reducer, streamingMapReduceJobDefinition.Reducer) .WithParameter(CmdletConstants.StatusFolder, streamingMapReduceJobDefinition.StatusFolder) .WithParameter(CmdletConstants.Arguments, streamingMapReduceJobDefinition.Arguments) .Invoke(); Assert.AreEqual(1, results.Results.Count); AzureHDInsightStreamingMapReduceJobDefinition streamingMapReduceJobFromPowershell = results.Results.ToEnumerable<AzureHDInsightStreamingMapReduceJobDefinition>().First(); AssertJobDefinitionsEqual(streamingMapReduceJobDefinition, streamingMapReduceJobFromPowershell); foreach (string argument in streamingMapReduceJobDefinition.Arguments) { Assert.IsTrue( streamingMapReduceJobFromPowershell.Arguments.Any(arg => string.Equals(argument, arg)), "Unable to find argument '{0}' in value returned from powershell", argument); } } }
internal static void AssertJobDefinitionsEqual( StreamingMapReduceJobCreateParameters streamingMapReduceJobDefinition, AzureHDInsightStreamingMapReduceJobDefinition streamingMapReduceJobFromPowershell) { Assert.AreEqual(streamingMapReduceJobDefinition.JobName, streamingMapReduceJobFromPowershell.JobName); Assert.AreEqual(streamingMapReduceJobDefinition.Input, streamingMapReduceJobFromPowershell.Input); Assert.AreEqual(streamingMapReduceJobDefinition.Output, streamingMapReduceJobFromPowershell.Output); Assert.AreEqual(streamingMapReduceJobDefinition.Mapper, streamingMapReduceJobFromPowershell.Mapper); Assert.AreEqual(streamingMapReduceJobDefinition.Reducer, streamingMapReduceJobFromPowershell.Reducer); Assert.AreEqual(streamingMapReduceJobDefinition.StatusFolder, streamingMapReduceJobFromPowershell.StatusFolder); }
public Task<JobCreationResults> SubmitStreamingJob(StreamingMapReduceJobCreateParameters streamingMapReduceJobCreateParameters) { if (streamingMapReduceJobCreateParameters == null) { throw new ArgumentNullException("streamingMapReduceJobCreateParameters"); } return Task.FromResult(this.CreateJobSuccessResult(streamingMapReduceJobCreateParameters, streamingMapReduceJobCreateParameters.JobName)); }
public void CanSerializeValidStreamingMapReduceJobRequest_WithNoReducer() { var streamingMapReduceJob = new StreamingMapReduceJobCreateParameters() { Input = Constants.WabsProtocolSchemeName + "input", Output = Constants.WabsProtocolSchemeName + "output", Mapper = Constants.WabsProtocolSchemeName + "mapper" }; var payloadConverter = new PayloadConverterBase(); var payload = payloadConverter.SerializeStreamingMapReduceRequest("hadoop", streamingMapReduceJob); Assert.IsTrue(payload.Contains(string.Format("{0}={1}", WebHCatConstants.Input, Uri.EscapeDataString(streamingMapReduceJob.Input)))); Assert.IsTrue(payload.Contains(string.Format("{0}={1}", WebHCatConstants.Output, Uri.EscapeDataString(streamingMapReduceJob.Output)))); Assert.IsTrue(payload.Contains(string.Format("{0}={1}", WebHCatConstants.Mapper, Uri.EscapeDataString(streamingMapReduceJob.Mapper)))); }
public void CanSerializeValidStreamingMapReduceJobRequest_WithFiles() { var streamingMapReduceJob = new StreamingMapReduceJobCreateParameters() { Input = Constants.WabsProtocolSchemeName + "input", Output = Constants.WabsProtocolSchemeName + "output", Mapper = Constants.WabsProtocolSchemeName + "mapper", Reducer = Constants.WabsProtocolSchemeName + "reducer" }; var resourceFile1 = "asv://container@hostname/myfile1"; var resourceFile2 = "asv://container@hostname/myfile2"; var payloadConverter = new PayloadConverterBase(); streamingMapReduceJob.Files.Add(resourceFile1); streamingMapReduceJob.Files.Add(resourceFile2); var payload = payloadConverter.SerializeStreamingMapReduceRequest("hadoop", streamingMapReduceJob); Assert.IsTrue(payload.Contains(string.Format("{0}={1}", WebHCatConstants.Input, Uri.EscapeDataString(streamingMapReduceJob.Input)))); Assert.IsTrue(payload.Contains(string.Format("{0}={1}", WebHCatConstants.Output, Uri.EscapeDataString(streamingMapReduceJob.Output)))); Assert.IsTrue(payload.Contains(string.Format("{0}={1}", WebHCatConstants.Mapper, Uri.EscapeDataString(streamingMapReduceJob.Mapper)))); Assert.IsTrue(payload.Contains(string.Format("{0}={1}", WebHCatConstants.Reducer, Uri.EscapeDataString(streamingMapReduceJob.Reducer)))); Assert.IsTrue(payload.Contains(Uri.EscapeDataString(resourceFile1))); Assert.IsTrue(payload.Contains(Uri.EscapeDataString(resourceFile2))); }
public void CanSerializeValidStreamingMapReduceJobRequest_CmdEnv() { var streamingMapReduceJob = new StreamingMapReduceJobCreateParameters() { Input = "asv://input", Output = "asv://output", Mapper = "asv://mapper", Reducer = "asv://reducer" }; streamingMapReduceJob.CommandEnvironment.Add("Name1=Value1"); streamingMapReduceJob.CommandEnvironment.Add("Name2=Value2"); var payloadConverter = new PayloadConverterBase(); var payload = payloadConverter.SerializeStreamingMapReduceRequest("hadoop", streamingMapReduceJob); Assert.IsTrue(payload.Contains(string.Format("{0}={1}", WebHCatConstants.Input, Uri.EscapeDataString(streamingMapReduceJob.Input)))); Assert.IsTrue(payload.Contains(string.Format("{0}={1}", WebHCatConstants.Output, Uri.EscapeDataString(streamingMapReduceJob.Output)))); Assert.IsTrue(payload.Contains(string.Format("{0}={1}", WebHCatConstants.Mapper, Uri.EscapeDataString(streamingMapReduceJob.Mapper)))); Assert.IsTrue(payload.Contains(string.Format("{0}={1}", WebHCatConstants.Reducer, Uri.EscapeDataString(streamingMapReduceJob.Reducer)))); foreach (var cmdEnvArgument in streamingMapReduceJob.CommandEnvironment) { Assert.IsTrue(payload.Contains(string.Format("{0}={1}", WebHCatConstants.CmdEnv, Uri.EscapeDataString(cmdEnvArgument)))); } }
public void CanSerializeValidStreamingMapReduceJobRequest_Defines() { var streamingMapReduceJob = new StreamingMapReduceJobCreateParameters() { Input = "asv://input", Output = "asv://output", Mapper = "asv://mapper", Reducer = "asv://reducer", }; streamingMapReduceJob.Defines.Add(new KeyValuePair<string, string>("definekey1", "definevalue1")); streamingMapReduceJob.Defines.Add(new KeyValuePair<string, string>("definekey2", "definevalue2")); streamingMapReduceJob.Defines.Add(new KeyValuePair<string, string>("definekey3", "definevalue3")); var payloadConverter = new PayloadConverterBase(); var payload = payloadConverter.SerializeStreamingMapReduceRequest("hadoop", streamingMapReduceJob); Assert.IsTrue(payload.Contains(string.Format("{0}={1}", WebHCatConstants.Input, Uri.EscapeDataString(streamingMapReduceJob.Input)))); Assert.IsTrue(payload.Contains(string.Format("{0}={1}", WebHCatConstants.Output, Uri.EscapeDataString(streamingMapReduceJob.Output)))); Assert.IsTrue(payload.Contains(string.Format("{0}={1}", WebHCatConstants.Mapper, Uri.EscapeDataString(streamingMapReduceJob.Mapper)))); Assert.IsTrue(payload.Contains(string.Format("{0}={1}", WebHCatConstants.Reducer, Uri.EscapeDataString(streamingMapReduceJob.Reducer)))); foreach (var define in streamingMapReduceJob.Defines) { Assert.IsTrue(payload.Contains(Uri.EscapeDataString(string.Format("{0}={1}", define.Key, define.Value)))); } }
public void CanSerializeValidStreamingMapReduceJobRequest() { var streamingMapReduceJob = new StreamingMapReduceJobCreateParameters() { Input = "asv://input", Output = "asv://output", Mapper = "asv://mapper", Reducer = "asv://reducer", Combiner = "asv://combiner" }; var payloadConverter = new PayloadConverterBase(); var payload = payloadConverter.SerializeStreamingMapReduceRequest("hadoop", streamingMapReduceJob); Assert.IsTrue(payload.Contains(string.Format("{0}={1}", WebHCatConstants.Input, Uri.EscapeDataString(streamingMapReduceJob.Input)))); Assert.IsTrue(payload.Contains(string.Format("{0}={1}", WebHCatConstants.Output, Uri.EscapeDataString(streamingMapReduceJob.Output)))); Assert.IsTrue(payload.Contains(string.Format("{0}={1}", WebHCatConstants.Mapper, Uri.EscapeDataString(streamingMapReduceJob.Mapper)))); Assert.IsTrue(payload.Contains(string.Format("{0}={1}", WebHCatConstants.Reducer, Uri.EscapeDataString(streamingMapReduceJob.Reducer)))); Assert.IsTrue(payload.Contains(string.Format("{0}={1}", WebHCatConstants.Combiner, Uri.EscapeDataString(streamingMapReduceJob.Combiner)))); }
/// <inheritdoc /> public JobCreationResults CreateStreamingJob(StreamingMapReduceJobCreateParameters streamingMapReduceJobCreateParameters) { return(this.CreateStreamingJobAsync(streamingMapReduceJobCreateParameters).WaitForResult()); }
public void CanCreateNewStreamingJob_StartJob() { var streamingMapReduceJobDefinition = new StreamingMapReduceJobCreateParameters { JobName = "pi estimation jobDetails", Input = TestConstants.WabsProtocolSchemeName + "container@hostname/input", Output = TestConstants.WabsProtocolSchemeName + "container@hostname/input", Mapper = TestConstants.WabsProtocolSchemeName + "container@hostname/combiner", Reducer = TestConstants.WabsProtocolSchemeName + "container@hostname/combiner", StatusFolder = TestConstants.WabsProtocolSchemeName + "container@hostname/someotherlocation" }; INewAzureHDInsightStreamingJobDefinitionCommand newStreamingMapReduceJobDefinitionCommand = ServiceLocator.Instance.Locate<IAzureHDInsightCommandFactory>().CreateNewStreamingMapReduceDefinition(); newStreamingMapReduceJobDefinitionCommand.JobName = streamingMapReduceJobDefinition.JobName; newStreamingMapReduceJobDefinitionCommand.JobName = streamingMapReduceJobDefinition.JobName; newStreamingMapReduceJobDefinitionCommand.InputPath = streamingMapReduceJobDefinition.Input; newStreamingMapReduceJobDefinitionCommand.OutputPath = streamingMapReduceJobDefinition.Output; newStreamingMapReduceJobDefinitionCommand.Mapper = streamingMapReduceJobDefinition.Mapper; newStreamingMapReduceJobDefinitionCommand.Reducer = streamingMapReduceJobDefinition.Reducer; newStreamingMapReduceJobDefinitionCommand.StatusFolder = streamingMapReduceJobDefinition.StatusFolder; newStreamingMapReduceJobDefinitionCommand.EndProcessing(); AzureHDInsightStreamingMapReduceJobDefinition streamingJobFromCommand = newStreamingMapReduceJobDefinitionCommand.Output.ElementAt(0); TestJobStart(streamingJobFromCommand); }
public JobCreationResults CreateStreamingJob(StreamingMapReduceJobCreateParameters streamingMapReduceJobCreateParameters) { return this.CreateStreamingJobAsync(streamingMapReduceJobCreateParameters).WaitForResult(); }
internal StreamingMapReduceJobCreateParameters ToStreamingMapReduceJobCreateParameters() { var streamingJobDefinition = new StreamingMapReduceJobCreateParameters { JobName = this.JobName, Input = this.Input, Output = this.Output, Reducer = this.Reducer, Combiner = this.Combiner, Mapper = this.Mapper, StatusFolder = this.StatusFolder }; streamingJobDefinition.CommandEnvironment.AddRange(this.CommandEnvironment); streamingJobDefinition.Arguments.AddRange(this.Arguments); streamingJobDefinition.Defines.AddRange(this.Defines); streamingJobDefinition.Files.AddRange(this.Files); return streamingJobDefinition; }
/// <inheritdoc /> public async Task<JobCreationResults> SubmitStreamingJob(StreamingMapReduceJobCreateParameters pigJobCreateParameters) { var remoteClient = ServiceLocator.Instance.Locate<IRemoteHadoopJobSubmissionPocoClientFactory>().Create(this.remoteCreds, this.context, this.ignoreSslErrors, this.GetUserAgentString()); return await remoteClient.SubmitStreamingJob(pigJobCreateParameters); }