public void ICanCallThe_New_HDInsightStreamingMapReduceJobDefinitionCmdlet()
        {
            var streamingMapReduceJobDefinition = new StreamingMapReduceJobCreateParameters
            {
                JobName = "pi estimation jobDetails",
                Input = TestConstants.WabsProtocolSchemeName + "input",
                Output = TestConstants.WabsProtocolSchemeName + "input",
                Mapper = TestConstants.WabsProtocolSchemeName + "combiner",
                Reducer = TestConstants.WabsProtocolSchemeName + "combiner",
                StatusFolder = TestConstants.WabsProtocolSchemeName + "someotherlocation"
            };

            using (IRunspace runspace = this.GetPowerShellRunspace())
            {
                IPipelineResult results =
                    runspace.NewPipeline()
                            .AddCommand(CmdletConstants.NewAzureHDInsightStreamingMapReduceJobDefinition)
                            .WithParameter(CmdletConstants.JobName, streamingMapReduceJobDefinition.JobName)
                            .WithParameter(CmdletConstants.Input, streamingMapReduceJobDefinition.Input)
                            .WithParameter(CmdletConstants.Output, streamingMapReduceJobDefinition.Output)
                            .WithParameter(CmdletConstants.Mapper, streamingMapReduceJobDefinition.Mapper)
                            .WithParameter(CmdletConstants.Reducer, streamingMapReduceJobDefinition.Reducer)
                            .WithParameter(CmdletConstants.StatusFolder, streamingMapReduceJobDefinition.StatusFolder)
                            .Invoke();
                Assert.AreEqual(1, results.Results.Count);
                AzureHDInsightStreamingMapReduceJobDefinition streamingMapReduceJobFromPowershell =
                    results.Results.ToEnumerable<AzureHDInsightStreamingMapReduceJobDefinition>().First();

                AssertJobDefinitionsEqual(streamingMapReduceJobDefinition, streamingMapReduceJobFromPowershell);
            }
        }
Example #2
0
        /// <inheritdoc />
        public Task <JobCreationResults> CreateStreamingJobAsync(StreamingMapReduceJobCreateParameters streamingMapReduceJobCreateParameters)
        {
            var factory    = ServiceLocator.Instance.Locate <IRemoteHadoopJobSubmissionPocoClientFactory>();
            var pocoClient = factory.Create(this.credentials, this.Context, this.IgnoreSslErrors, this.GetCustomUserAgent());

            return(pocoClient.SubmitStreamingJob(streamingMapReduceJobCreateParameters));
        }
 public Task<JobCreationResults> SubmitStreamingJob(StreamingMapReduceJobCreateParameters pigJobCreateParameters)
 {
     throw new System.NotImplementedException();
 }
        public void ICanCallThe_New_HDInsightStreamingMapReduceJobDefinitionCmdlet_WithArguments()
        {
            var streamingMapReduceJobDefinition = new StreamingMapReduceJobCreateParameters
            {
                JobName = "pi estimation jobDetails",
                Input = TestConstants.WabsProtocolSchemeName + "input",
                Output = TestConstants.WabsProtocolSchemeName + "input",
                Mapper = TestConstants.WabsProtocolSchemeName + "combiner",
                Reducer = TestConstants.WabsProtocolSchemeName + "combiner",
                StatusFolder = TestConstants.WabsProtocolSchemeName + "someotherlocation"
            };
            streamingMapReduceJobDefinition.Arguments.Add("16");
            streamingMapReduceJobDefinition.Arguments.Add("10000");

            using (IRunspace runspace = this.GetPowerShellRunspace())
            {
                IPipelineResult results =
                    runspace.NewPipeline()
                            .AddCommand(CmdletConstants.NewAzureHDInsightStreamingMapReduceJobDefinition)
                            .WithParameter(CmdletConstants.JobName, streamingMapReduceJobDefinition.JobName)
                            .WithParameter(CmdletConstants.Input, streamingMapReduceJobDefinition.Input)
                            .WithParameter(CmdletConstants.Output, streamingMapReduceJobDefinition.Output)
                            .WithParameter(CmdletConstants.Mapper, streamingMapReduceJobDefinition.Mapper)
                            .WithParameter(CmdletConstants.Reducer, streamingMapReduceJobDefinition.Reducer)
                            .WithParameter(CmdletConstants.StatusFolder, streamingMapReduceJobDefinition.StatusFolder)
                            .WithParameter(CmdletConstants.Arguments, streamingMapReduceJobDefinition.Arguments)
                            .Invoke();
                Assert.AreEqual(1, results.Results.Count);
                AzureHDInsightStreamingMapReduceJobDefinition streamingMapReduceJobFromPowershell =
                    results.Results.ToEnumerable<AzureHDInsightStreamingMapReduceJobDefinition>().First();

                AssertJobDefinitionsEqual(streamingMapReduceJobDefinition, streamingMapReduceJobFromPowershell);
                foreach (string argument in streamingMapReduceJobDefinition.Arguments)
                {
                    Assert.IsTrue(
                        streamingMapReduceJobFromPowershell.Arguments.Any(arg => string.Equals(argument, arg)),
                        "Unable to find argument '{0}' in value returned from powershell",
                        argument);
                }
            }
        }
 internal static void AssertJobDefinitionsEqual(
     StreamingMapReduceJobCreateParameters streamingMapReduceJobDefinition,
     AzureHDInsightStreamingMapReduceJobDefinition streamingMapReduceJobFromPowershell)
 {
     Assert.AreEqual(streamingMapReduceJobDefinition.JobName, streamingMapReduceJobFromPowershell.JobName);
     Assert.AreEqual(streamingMapReduceJobDefinition.Input, streamingMapReduceJobFromPowershell.Input);
     Assert.AreEqual(streamingMapReduceJobDefinition.Output, streamingMapReduceJobFromPowershell.Output);
     Assert.AreEqual(streamingMapReduceJobDefinition.Mapper, streamingMapReduceJobFromPowershell.Mapper);
     Assert.AreEqual(streamingMapReduceJobDefinition.Reducer, streamingMapReduceJobFromPowershell.Reducer);
     Assert.AreEqual(streamingMapReduceJobDefinition.StatusFolder, streamingMapReduceJobFromPowershell.StatusFolder);
 }
        public Task<JobCreationResults> SubmitStreamingJob(StreamingMapReduceJobCreateParameters streamingMapReduceJobCreateParameters)
        {
            if (streamingMapReduceJobCreateParameters == null)
            {
                throw new ArgumentNullException("streamingMapReduceJobCreateParameters");
            }

            return Task.FromResult(this.CreateJobSuccessResult(streamingMapReduceJobCreateParameters, streamingMapReduceJobCreateParameters.JobName));
        }
        public void CanSerializeValidStreamingMapReduceJobRequest_WithNoReducer()
        {
            var streamingMapReduceJob = new StreamingMapReduceJobCreateParameters()
            {
                Input = Constants.WabsProtocolSchemeName + "input",
                Output = Constants.WabsProtocolSchemeName + "output",
                Mapper = Constants.WabsProtocolSchemeName + "mapper"
            };

            var payloadConverter = new PayloadConverterBase();
            var payload = payloadConverter.SerializeStreamingMapReduceRequest("hadoop", streamingMapReduceJob);

            Assert.IsTrue(payload.Contains(string.Format("{0}={1}", WebHCatConstants.Input, Uri.EscapeDataString(streamingMapReduceJob.Input))));
            Assert.IsTrue(payload.Contains(string.Format("{0}={1}", WebHCatConstants.Output, Uri.EscapeDataString(streamingMapReduceJob.Output))));
            Assert.IsTrue(payload.Contains(string.Format("{0}={1}", WebHCatConstants.Mapper, Uri.EscapeDataString(streamingMapReduceJob.Mapper))));
        }
        public void CanSerializeValidStreamingMapReduceJobRequest_WithFiles()
        {
            var streamingMapReduceJob = new StreamingMapReduceJobCreateParameters()
            {
                Input = Constants.WabsProtocolSchemeName + "input",
                Output = Constants.WabsProtocolSchemeName + "output",
                Mapper = Constants.WabsProtocolSchemeName + "mapper",
                Reducer = Constants.WabsProtocolSchemeName + "reducer"
            };

            var resourceFile1 = "asv://container@hostname/myfile1";
            var resourceFile2 = "asv://container@hostname/myfile2";
            var payloadConverter = new PayloadConverterBase();
            streamingMapReduceJob.Files.Add(resourceFile1);
            streamingMapReduceJob.Files.Add(resourceFile2);

            var payload = payloadConverter.SerializeStreamingMapReduceRequest("hadoop", streamingMapReduceJob);

            Assert.IsTrue(payload.Contains(string.Format("{0}={1}", WebHCatConstants.Input, Uri.EscapeDataString(streamingMapReduceJob.Input))));
            Assert.IsTrue(payload.Contains(string.Format("{0}={1}", WebHCatConstants.Output, Uri.EscapeDataString(streamingMapReduceJob.Output))));
            Assert.IsTrue(payload.Contains(string.Format("{0}={1}", WebHCatConstants.Mapper, Uri.EscapeDataString(streamingMapReduceJob.Mapper))));
            Assert.IsTrue(payload.Contains(string.Format("{0}={1}", WebHCatConstants.Reducer, Uri.EscapeDataString(streamingMapReduceJob.Reducer))));
            Assert.IsTrue(payload.Contains(Uri.EscapeDataString(resourceFile1)));
            Assert.IsTrue(payload.Contains(Uri.EscapeDataString(resourceFile2)));
        }
        public void CanSerializeValidStreamingMapReduceJobRequest_CmdEnv()
        {
            var streamingMapReduceJob = new StreamingMapReduceJobCreateParameters()
            {
                Input = "asv://input",
                Output = "asv://output",
                Mapper = "asv://mapper",
                Reducer = "asv://reducer"
            };

            streamingMapReduceJob.CommandEnvironment.Add("Name1=Value1");
            streamingMapReduceJob.CommandEnvironment.Add("Name2=Value2");

            var payloadConverter = new PayloadConverterBase();
            var payload = payloadConverter.SerializeStreamingMapReduceRequest("hadoop", streamingMapReduceJob);

            Assert.IsTrue(payload.Contains(string.Format("{0}={1}", WebHCatConstants.Input, Uri.EscapeDataString(streamingMapReduceJob.Input))));
            Assert.IsTrue(payload.Contains(string.Format("{0}={1}", WebHCatConstants.Output, Uri.EscapeDataString(streamingMapReduceJob.Output))));
            Assert.IsTrue(payload.Contains(string.Format("{0}={1}", WebHCatConstants.Mapper, Uri.EscapeDataString(streamingMapReduceJob.Mapper))));
            Assert.IsTrue(payload.Contains(string.Format("{0}={1}", WebHCatConstants.Reducer, Uri.EscapeDataString(streamingMapReduceJob.Reducer))));

            foreach (var cmdEnvArgument in streamingMapReduceJob.CommandEnvironment)
            {
                Assert.IsTrue(payload.Contains(string.Format("{0}={1}", WebHCatConstants.CmdEnv, Uri.EscapeDataString(cmdEnvArgument))));
            }
        }
        public void CanSerializeValidStreamingMapReduceJobRequest_Defines()
        {
            var streamingMapReduceJob = new StreamingMapReduceJobCreateParameters()
            {
                Input = "asv://input",
                Output = "asv://output",
                Mapper = "asv://mapper",
                Reducer = "asv://reducer",
            };

            streamingMapReduceJob.Defines.Add(new KeyValuePair<string, string>("definekey1", "definevalue1"));
            streamingMapReduceJob.Defines.Add(new KeyValuePair<string, string>("definekey2", "definevalue2"));
            streamingMapReduceJob.Defines.Add(new KeyValuePair<string, string>("definekey3", "definevalue3"));
            var payloadConverter = new PayloadConverterBase();
            var payload = payloadConverter.SerializeStreamingMapReduceRequest("hadoop", streamingMapReduceJob);
            Assert.IsTrue(payload.Contains(string.Format("{0}={1}", WebHCatConstants.Input, Uri.EscapeDataString(streamingMapReduceJob.Input))));
            Assert.IsTrue(payload.Contains(string.Format("{0}={1}", WebHCatConstants.Output, Uri.EscapeDataString(streamingMapReduceJob.Output))));
            Assert.IsTrue(payload.Contains(string.Format("{0}={1}", WebHCatConstants.Mapper, Uri.EscapeDataString(streamingMapReduceJob.Mapper))));
            Assert.IsTrue(payload.Contains(string.Format("{0}={1}", WebHCatConstants.Reducer, Uri.EscapeDataString(streamingMapReduceJob.Reducer))));

            foreach (var define in streamingMapReduceJob.Defines)
            {
                Assert.IsTrue(payload.Contains(Uri.EscapeDataString(string.Format("{0}={1}", define.Key, define.Value))));
            }
        }
        public void CanSerializeValidStreamingMapReduceJobRequest()
        {
            var streamingMapReduceJob = new StreamingMapReduceJobCreateParameters()
            {
                Input = "asv://input",
                Output = "asv://output",
                Mapper = "asv://mapper",
                Reducer = "asv://reducer",
                Combiner = "asv://combiner"
            };

            var payloadConverter = new PayloadConverterBase();
            var payload = payloadConverter.SerializeStreamingMapReduceRequest("hadoop", streamingMapReduceJob);

            Assert.IsTrue(payload.Contains(string.Format("{0}={1}", WebHCatConstants.Input, Uri.EscapeDataString(streamingMapReduceJob.Input))));
            Assert.IsTrue(payload.Contains(string.Format("{0}={1}", WebHCatConstants.Output, Uri.EscapeDataString(streamingMapReduceJob.Output))));
            Assert.IsTrue(payload.Contains(string.Format("{0}={1}", WebHCatConstants.Mapper, Uri.EscapeDataString(streamingMapReduceJob.Mapper))));
            Assert.IsTrue(payload.Contains(string.Format("{0}={1}", WebHCatConstants.Reducer, Uri.EscapeDataString(streamingMapReduceJob.Reducer))));
            Assert.IsTrue(payload.Contains(string.Format("{0}={1}", WebHCatConstants.Combiner, Uri.EscapeDataString(streamingMapReduceJob.Combiner))));
        }
Example #12
0
 /// <inheritdoc />
 public JobCreationResults CreateStreamingJob(StreamingMapReduceJobCreateParameters streamingMapReduceJobCreateParameters)
 {
     return(this.CreateStreamingJobAsync(streamingMapReduceJobCreateParameters).WaitForResult());
 }
        public void CanCreateNewStreamingJob_StartJob()
        {
            var streamingMapReduceJobDefinition = new StreamingMapReduceJobCreateParameters
            {
                JobName = "pi estimation jobDetails",
                Input = TestConstants.WabsProtocolSchemeName + "container@hostname/input",
                Output = TestConstants.WabsProtocolSchemeName + "container@hostname/input",
                Mapper = TestConstants.WabsProtocolSchemeName + "container@hostname/combiner",
                Reducer = TestConstants.WabsProtocolSchemeName + "container@hostname/combiner",
                StatusFolder = TestConstants.WabsProtocolSchemeName + "container@hostname/someotherlocation"
            };

            INewAzureHDInsightStreamingJobDefinitionCommand newStreamingMapReduceJobDefinitionCommand =
                ServiceLocator.Instance.Locate<IAzureHDInsightCommandFactory>().CreateNewStreamingMapReduceDefinition();
            newStreamingMapReduceJobDefinitionCommand.JobName = streamingMapReduceJobDefinition.JobName;
            newStreamingMapReduceJobDefinitionCommand.JobName = streamingMapReduceJobDefinition.JobName;
            newStreamingMapReduceJobDefinitionCommand.InputPath = streamingMapReduceJobDefinition.Input;
            newStreamingMapReduceJobDefinitionCommand.OutputPath = streamingMapReduceJobDefinition.Output;
            newStreamingMapReduceJobDefinitionCommand.Mapper = streamingMapReduceJobDefinition.Mapper;
            newStreamingMapReduceJobDefinitionCommand.Reducer = streamingMapReduceJobDefinition.Reducer;
            newStreamingMapReduceJobDefinitionCommand.StatusFolder = streamingMapReduceJobDefinition.StatusFolder;
            newStreamingMapReduceJobDefinitionCommand.EndProcessing();

            AzureHDInsightStreamingMapReduceJobDefinition streamingJobFromCommand = newStreamingMapReduceJobDefinitionCommand.Output.ElementAt(0);

            TestJobStart(streamingJobFromCommand);
        }
 public JobCreationResults CreateStreamingJob(StreamingMapReduceJobCreateParameters streamingMapReduceJobCreateParameters)
 {
     return this.CreateStreamingJobAsync(streamingMapReduceJobCreateParameters).WaitForResult();
 }
        internal StreamingMapReduceJobCreateParameters ToStreamingMapReduceJobCreateParameters()
        {
            var streamingJobDefinition = new StreamingMapReduceJobCreateParameters
            {
                JobName = this.JobName,
                Input = this.Input,
                Output = this.Output,
                Reducer = this.Reducer,
                Combiner = this.Combiner,
                Mapper = this.Mapper,
                StatusFolder = this.StatusFolder
            };

            streamingJobDefinition.CommandEnvironment.AddRange(this.CommandEnvironment);
            streamingJobDefinition.Arguments.AddRange(this.Arguments);
            streamingJobDefinition.Defines.AddRange(this.Defines);
            streamingJobDefinition.Files.AddRange(this.Files);

            return streamingJobDefinition;
        }
 /// <inheritdoc />
 public async Task<JobCreationResults> SubmitStreamingJob(StreamingMapReduceJobCreateParameters pigJobCreateParameters)
 {
     var remoteClient = ServiceLocator.Instance.Locate<IRemoteHadoopJobSubmissionPocoClientFactory>().Create(this.remoteCreds, this.context, this.ignoreSslErrors, this.GetUserAgentString());
     return await remoteClient.SubmitStreamingJob(pigJobCreateParameters);
 }