public void ICanCallThe_New_HDInsightStreamingMapReduceJobDefinitionCmdlet() { var streamingMapReduceJobDefinition = new StreamingMapReduceJobCreateParameters { JobName = "pi estimation jobDetails", Input = TestConstants.WabsProtocolSchemeName + "input", Output = TestConstants.WabsProtocolSchemeName + "input", Mapper = TestConstants.WabsProtocolSchemeName + "combiner", Reducer = TestConstants.WabsProtocolSchemeName + "combiner", StatusFolder = TestConstants.WabsProtocolSchemeName + "someotherlocation" }; using (IRunspace runspace = this.GetPowerShellRunspace()) { IPipelineResult results = runspace.NewPipeline() .AddCommand(CmdletConstants.NewAzureHDInsightStreamingMapReduceJobDefinition) .WithParameter(CmdletConstants.JobName, streamingMapReduceJobDefinition.JobName) .WithParameter(CmdletConstants.Input, streamingMapReduceJobDefinition.Input) .WithParameter(CmdletConstants.Output, streamingMapReduceJobDefinition.Output) .WithParameter(CmdletConstants.Mapper, streamingMapReduceJobDefinition.Mapper) .WithParameter(CmdletConstants.Reducer, streamingMapReduceJobDefinition.Reducer) .WithParameter(CmdletConstants.StatusFolder, streamingMapReduceJobDefinition.StatusFolder) .Invoke(); Assert.AreEqual(1, results.Results.Count); AzureHDInsightStreamingMapReduceJobDefinition streamingMapReduceJobFromPowershell = results.Results.ToEnumerable<AzureHDInsightStreamingMapReduceJobDefinition>().First(); AssertJobDefinitionsEqual(streamingMapReduceJobDefinition, streamingMapReduceJobFromPowershell); } }
public virtual void NewStreamingMapReduceJob_StartJob_GetJob() { var streamingMapReduceJobDefinition = new StreamingMapReduceJobCreateParameters { JobName = "environment variable jobDetails", Mapper = "environmentvariables.exe", Input = "/example/apps/environmentvariables.exe", Output = Guid.NewGuid().ToString() }; var files = new List <string> { streamingMapReduceJobDefinition.Mapper }; using (IRunspace runspace = this.GetPowerShellRunspace()) { IPipelineResult results = runspace.NewPipeline() .AddCommand(CmdletConstants.NewAzureHDInsightStreamingMapReduceJobDefinition) .WithParameter(CmdletConstants.JobName, streamingMapReduceJobDefinition.JobName) .WithParameter(CmdletConstants.Mapper, streamingMapReduceJobDefinition.Mapper) .WithParameter(CmdletConstants.Input, streamingMapReduceJobDefinition.Input) .WithParameter(CmdletConstants.Output, streamingMapReduceJobDefinition.Output) .WithParameter(CmdletConstants.Files, files) .Invoke(); Assert.AreEqual(1, results.Results.Count); AzureHDInsightStreamingMapReduceJobDefinition streamingMapReduceJobFromPowershell = results.Results.ToEnumerable <AzureHDInsightStreamingMapReduceJobDefinition>().First(); RunJobAndGetWithId(runspace, streamingMapReduceJobFromPowershell); } }
/// <summary> /// Creates the payload for a Streaming MapReduce request. /// </summary> /// <param name="userName"> /// The user name. /// </param> /// <param name="details"> /// The details. /// </param> /// <returns> /// A string that represents the payload for the request. /// </returns> public string SerializeStreamingMapReduceRequest(string userName, StreamingMapReduceJobCreateParameters details) { details.ArgumentNotNull("details"); var values = new List <KeyValuePair <string, string> >(); values.Add(new KeyValuePair <string, string>(WebHCatConstants.Input, details.Input)); values.Add(new KeyValuePair <string, string>(WebHCatConstants.Output, details.Output)); values.Add(new KeyValuePair <string, string>(WebHCatConstants.Mapper, details.Mapper)); if (!details.Combiner.IsNullOrEmpty()) { values.Add(new KeyValuePair <string, string>(WebHCatConstants.Combiner, details.Combiner)); } if (details.Reducer.IsNullOrEmpty()) { values.Add(new KeyValuePair <string, string>(WebHCatConstants.Reducer, NoneReducer)); } else { values.Add(new KeyValuePair <string, string>(WebHCatConstants.Reducer, details.Reducer)); } values.AddRange(this.SerializeJobRequest(userName, details, details.JobName, details.Arguments, details.Defines)); values.AddRange(this.BuildList(WebHCatConstants.CmdEnv, details.CommandEnvironment)); var retval = this.ConvertItems(values.Where(kvp => kvp.Value != null)); return(retval); }
public void CanSerializeValidStreamingMapReduceJobRequest_CmdEnv() { var streamingMapReduceJob = new StreamingMapReduceJobCreateParameters() { Input = "asv://input", Output = "asv://output", Mapper = "asv://mapper", Reducer = "asv://reducer" }; streamingMapReduceJob.CommandEnvironment.Add("Name1=Value1"); streamingMapReduceJob.CommandEnvironment.Add("Name2=Value2"); var payloadConverter = new PayloadConverterBase(); var payload = payloadConverter.SerializeStreamingMapReduceRequest("hadoop", streamingMapReduceJob); Assert.IsTrue(payload.Contains(string.Format("{0}={1}", WebHCatConstants.Input, Uri.EscapeDataString(streamingMapReduceJob.Input)))); Assert.IsTrue(payload.Contains(string.Format("{0}={1}", WebHCatConstants.Output, Uri.EscapeDataString(streamingMapReduceJob.Output)))); Assert.IsTrue(payload.Contains(string.Format("{0}={1}", WebHCatConstants.Mapper, Uri.EscapeDataString(streamingMapReduceJob.Mapper)))); Assert.IsTrue(payload.Contains(string.Format("{0}={1}", WebHCatConstants.Reducer, Uri.EscapeDataString(streamingMapReduceJob.Reducer)))); foreach (var cmdEnvArgument in streamingMapReduceJob.CommandEnvironment) { Assert.IsTrue(payload.Contains(string.Format("{0}={1}", WebHCatConstants.CmdEnv, Uri.EscapeDataString(cmdEnvArgument)))); } }
public void CanSerializeValidStreamingMapReduceJobRequest_WithFiles() { var streamingMapReduceJob = new StreamingMapReduceJobCreateParameters() { Input = Constants.WabsProtocolSchemeName + "input", Output = Constants.WabsProtocolSchemeName + "output", Mapper = Constants.WabsProtocolSchemeName + "mapper", Reducer = Constants.WabsProtocolSchemeName + "reducer" }; var resourceFile1 = "asv://container@hostname/myfile1"; var resourceFile2 = "asv://container@hostname/myfile2"; var payloadConverter = new PayloadConverterBase(); streamingMapReduceJob.Files.Add(resourceFile1); streamingMapReduceJob.Files.Add(resourceFile2); var payload = payloadConverter.SerializeStreamingMapReduceRequest("hadoop", streamingMapReduceJob); Assert.IsTrue(payload.Contains(string.Format("{0}={1}", WebHCatConstants.Input, Uri.EscapeDataString(streamingMapReduceJob.Input)))); Assert.IsTrue(payload.Contains(string.Format("{0}={1}", WebHCatConstants.Output, Uri.EscapeDataString(streamingMapReduceJob.Output)))); Assert.IsTrue(payload.Contains(string.Format("{0}={1}", WebHCatConstants.Mapper, Uri.EscapeDataString(streamingMapReduceJob.Mapper)))); Assert.IsTrue(payload.Contains(string.Format("{0}={1}", WebHCatConstants.Reducer, Uri.EscapeDataString(streamingMapReduceJob.Reducer)))); Assert.IsTrue(payload.Contains(Uri.EscapeDataString(resourceFile1))); Assert.IsTrue(payload.Contains(Uri.EscapeDataString(resourceFile2))); }
public void CanSerializeValidStreamingMapReduceJobRequest_Defines() { var streamingMapReduceJob = new StreamingMapReduceJobCreateParameters() { Input = "asv://input", Output = "asv://output", Mapper = "asv://mapper", Reducer = "asv://reducer", }; streamingMapReduceJob.Defines.Add(new KeyValuePair <string, string>("definekey1", "definevalue1")); streamingMapReduceJob.Defines.Add(new KeyValuePair <string, string>("definekey2", "definevalue2")); streamingMapReduceJob.Defines.Add(new KeyValuePair <string, string>("definekey3", "definevalue3")); var payloadConverter = new PayloadConverterBase(); var payload = payloadConverter.SerializeStreamingMapReduceRequest("hadoop", streamingMapReduceJob); Assert.IsTrue(payload.Contains(string.Format("{0}={1}", WebHCatConstants.Input, Uri.EscapeDataString(streamingMapReduceJob.Input)))); Assert.IsTrue(payload.Contains(string.Format("{0}={1}", WebHCatConstants.Output, Uri.EscapeDataString(streamingMapReduceJob.Output)))); Assert.IsTrue(payload.Contains(string.Format("{0}={1}", WebHCatConstants.Mapper, Uri.EscapeDataString(streamingMapReduceJob.Mapper)))); Assert.IsTrue(payload.Contains(string.Format("{0}={1}", WebHCatConstants.Reducer, Uri.EscapeDataString(streamingMapReduceJob.Reducer)))); foreach (var define in streamingMapReduceJob.Defines) { Assert.IsTrue(payload.Contains(Uri.EscapeDataString(string.Format("{0}={1}", define.Key, define.Value)))); } }
public void CanCreateNewStreamingMapReduceDefinition_WithArguments() { StreamingMapReduceJobCreateParameters streamingMapReduceJobDefinition = this.GetStreamingMapReduceJob(); streamingMapReduceJobDefinition.Arguments.Add("16"); streamingMapReduceJobDefinition.Arguments.Add("10000"); INewAzureHDInsightStreamingJobDefinitionCommand newStreamingMapReduceJobDefinitionCommand = ServiceLocator.Instance.Locate <IAzureHDInsightCommandFactory>().CreateNewStreamingMapReduceDefinition(); newStreamingMapReduceJobDefinitionCommand.JobName = streamingMapReduceJobDefinition.JobName; newStreamingMapReduceJobDefinitionCommand.JobName = streamingMapReduceJobDefinition.JobName; newStreamingMapReduceJobDefinitionCommand.InputPath = streamingMapReduceJobDefinition.Input; newStreamingMapReduceJobDefinitionCommand.OutputPath = streamingMapReduceJobDefinition.Output; newStreamingMapReduceJobDefinitionCommand.Mapper = streamingMapReduceJobDefinition.Mapper; newStreamingMapReduceJobDefinitionCommand.Reducer = streamingMapReduceJobDefinition.Reducer; newStreamingMapReduceJobDefinitionCommand.StatusFolder = streamingMapReduceJobDefinition.StatusFolder; newStreamingMapReduceJobDefinitionCommand.Arguments = streamingMapReduceJobDefinition.Arguments.ToArray(); newStreamingMapReduceJobDefinitionCommand.EndProcessing(); AzureHDInsightStreamingMapReduceJobDefinition streamingMapReduceJobFromCommand = newStreamingMapReduceJobDefinitionCommand.Output.ElementAt(0); NewStreamingMapReduceJobCmdLetTests.AssertJobDefinitionsEqual(streamingMapReduceJobDefinition, streamingMapReduceJobFromCommand); foreach (string argument in streamingMapReduceJobDefinition.Arguments) { Assert.IsTrue( streamingMapReduceJobFromCommand.Arguments.Any(arg => string.Equals(argument, arg)), "Unable to find argument '{0}' in value returned from command", argument); } }
public void CanCreateNewStreamingJob_StartJob() { var streamingMapReduceJobDefinition = new StreamingMapReduceJobCreateParameters { JobName = "pi estimation jobDetails", Input = TestConstants.WabsProtocolSchemeName + "container@hostname/input", Output = TestConstants.WabsProtocolSchemeName + "container@hostname/input", Mapper = TestConstants.WabsProtocolSchemeName + "container@hostname/combiner", Reducer = TestConstants.WabsProtocolSchemeName + "container@hostname/combiner", StatusFolder = TestConstants.WabsProtocolSchemeName + "container@hostname/someotherlocation" }; INewAzureHDInsightStreamingJobDefinitionCommand newStreamingMapReduceJobDefinitionCommand = ServiceLocator.Instance.Locate <IAzureHDInsightCommandFactory>().CreateNewStreamingMapReduceDefinition(); newStreamingMapReduceJobDefinitionCommand.JobName = streamingMapReduceJobDefinition.JobName; newStreamingMapReduceJobDefinitionCommand.JobName = streamingMapReduceJobDefinition.JobName; newStreamingMapReduceJobDefinitionCommand.InputPath = streamingMapReduceJobDefinition.Input; newStreamingMapReduceJobDefinitionCommand.OutputPath = streamingMapReduceJobDefinition.Output; newStreamingMapReduceJobDefinitionCommand.Mapper = streamingMapReduceJobDefinition.Mapper; newStreamingMapReduceJobDefinitionCommand.Reducer = streamingMapReduceJobDefinition.Reducer; newStreamingMapReduceJobDefinitionCommand.StatusFolder = streamingMapReduceJobDefinition.StatusFolder; newStreamingMapReduceJobDefinitionCommand.EndProcessing(); AzureHDInsightStreamingMapReduceJobDefinition streamingJobFromCommand = newStreamingMapReduceJobDefinitionCommand.Output.ElementAt(0); TestJobStart(streamingJobFromCommand); }
/// <summary> /// Creates the payload for a Streaming MapReduce request. /// </summary> /// <param name="userName"> /// The user name. /// </param> /// <param name="details"> /// The details. /// </param> /// <returns> /// A string that represents the payload for the request. /// </returns> public string SerializeStreamingMapReduceRequest(string userName, StreamingMapReduceJobCreateParameters details) { details.ArgumentNotNull("details"); var values = new List<KeyValuePair<string, string>>(); values.Add(new KeyValuePair<string, string>(WebHCatConstants.Input, details.Input)); values.Add(new KeyValuePair<string, string>(WebHCatConstants.Output, details.Output)); values.Add(new KeyValuePair<string, string>(WebHCatConstants.Mapper, details.Mapper)); if (!details.Combiner.IsNullOrEmpty()) { values.Add(new KeyValuePair<string, string>(WebHCatConstants.Combiner, details.Combiner)); } if (details.Reducer.IsNullOrEmpty()) { values.Add(new KeyValuePair<string, string>(WebHCatConstants.Reducer, NoneReducer)); } else { values.Add(new KeyValuePair<string, string>(WebHCatConstants.Reducer, details.Reducer)); } values.AddRange(this.SerializeJobRequest(userName, details, details.JobName, details.Arguments, details.Defines)); values.AddRange(this.BuildList(WebHCatConstants.CmdEnv, details.CommandEnvironment)); var retval = this.ConvertItems(values.Where(kvp => kvp.Value != null)); return retval; }
public virtual void ICanCallThe_NewStreamingJob_Then_Start_HDInsightJobsCmdlet() { var streamingMapReduceJobDefinition = new StreamingMapReduceJobCreateParameters { JobName = "pi estimation jobDetails", Input = TestConstants.WabsProtocolSchemeName + "input", Output = TestConstants.WabsProtocolSchemeName + "input", Mapper = TestConstants.WabsProtocolSchemeName + "combiner", Reducer = TestConstants.WabsProtocolSchemeName + "combiner", StatusFolder = TestConstants.WabsProtocolSchemeName + "someotherlocation" }; using (IRunspace runspace = this.GetPowerShellRunspace()) { IPipelineResult results = runspace.NewPipeline() .AddCommand(CmdletConstants.NewAzureHDInsightStreamingMapReduceJobDefinition) .WithParameter(CmdletConstants.JobName, streamingMapReduceJobDefinition.JobName) .WithParameter(CmdletConstants.Input, streamingMapReduceJobDefinition.Input) .WithParameter(CmdletConstants.Output, streamingMapReduceJobDefinition.Output) .WithParameter(CmdletConstants.Mapper, streamingMapReduceJobDefinition.Mapper) .WithParameter(CmdletConstants.Reducer, streamingMapReduceJobDefinition.Reducer) .WithParameter(CmdletConstants.StatusFolder, streamingMapReduceJobDefinition.StatusFolder) .Invoke(); Assert.AreEqual(1, results.Results.Count); AzureHDInsightStreamingMapReduceJobDefinition streamingJobFromPowershell = results.Results.ToEnumerable <AzureHDInsightStreamingMapReduceJobDefinition>().First(); RunJobInPowershell(runspace, streamingJobFromPowershell); } }
public Task <JobCreationResults> SubmitStreamingJob(StreamingMapReduceJobCreateParameters streamingMapReduceJobCreateParameters) { if (streamingMapReduceJobCreateParameters == null) { throw new ArgumentNullException("streamingMapReduceJobCreateParameters"); } return(Task.FromResult(this.CreateJobSuccessResult(streamingMapReduceJobCreateParameters, streamingMapReduceJobCreateParameters.JobName))); }
internal static void AssertJobDefinitionsEqual( StreamingMapReduceJobCreateParameters streamingMapReduceJobDefinition, AzureHDInsightStreamingMapReduceJobDefinition streamingMapReduceJobFromPowershell) { Assert.AreEqual(streamingMapReduceJobDefinition.JobName, streamingMapReduceJobFromPowershell.JobName); Assert.AreEqual(streamingMapReduceJobDefinition.Input, streamingMapReduceJobFromPowershell.Input); Assert.AreEqual(streamingMapReduceJobDefinition.Output, streamingMapReduceJobFromPowershell.Output); Assert.AreEqual(streamingMapReduceJobDefinition.Mapper, streamingMapReduceJobFromPowershell.Mapper); Assert.AreEqual(streamingMapReduceJobDefinition.Reducer, streamingMapReduceJobFromPowershell.Reducer); Assert.AreEqual(streamingMapReduceJobDefinition.StatusFolder, streamingMapReduceJobFromPowershell.StatusFolder); }
/// <inheritdoc /> public async Task <JobCreationResults> SubmitStreamingJob(StreamingMapReduceJobCreateParameters streamingJobCreateParameters) { //NEIN: Any code modification here should add unit tests for this class var converter = new PayloadConverter(); var payload = converter.SerializeStreamingMapReduceRequest(this.credentials.UserName, streamingJobCreateParameters); var client = ServiceLocator.Instance.Locate <IHadoopRemoteJobSubmissionRestClientFactory>().Create(this.credentials, this.context, this.ignoreSslErrors, this.GetUserAgentString()); var result = await client.SubmitStreamingMapReduceJob(payload); return(new JobCreationResults() { JobId = converter.DeserializeJobSubmissionResponse(result.Content) }); }
/// <inheritdoc/> public async Task <JobCreationResults> CreateStreamingJobAsync(StreamingMapReduceJobCreateParameters streamingMapReduceJobCreateParameters) { var pocoClient = this.GetPocoClient(); try { return(await pocoClient.SubmitStreamingJob(streamingMapReduceJobCreateParameters)); } catch (UnauthorizedAccessException) { pocoClient = this.GetPocoClient(true); } return(await pocoClient.SubmitStreamingJob(streamingMapReduceJobCreateParameters)); }
public async Task <JobCreationResults> SubmitStreamingJob(StreamingMapReduceJobCreateParameters pigJobCreateParameters) { var relative = new Uri( HadoopRemoteRestConstants.Sqoop + "?" + HadoopRemoteRestConstants.UserName + "=" + _connectionConfig.UserName.EscapeDataString(), UriKind.Relative); var requestContent = _converter.SerializeStreamingMapReduceRequest(_connectionConfig.UserName, pigJobCreateParameters); var result = await MakeAsyncJobSubmissionRequest(relative, requestContent); return(new JobCreationResults { JobId = _converter.DeserializeJobSubmissionResponse(result) }); }
public void CanSerializeValidStreamingMapReduceJobRequest_WithNoReducer() { var streamingMapReduceJob = new StreamingMapReduceJobCreateParameters() { Input = Constants.WabsProtocolSchemeName + "input", Output = Constants.WabsProtocolSchemeName + "output", Mapper = Constants.WabsProtocolSchemeName + "mapper" }; var payloadConverter = new PayloadConverterBase(); var payload = payloadConverter.SerializeStreamingMapReduceRequest("hadoop", streamingMapReduceJob); Assert.IsTrue(payload.Contains(string.Format("{0}={1}", WebHCatConstants.Input, Uri.EscapeDataString(streamingMapReduceJob.Input)))); Assert.IsTrue(payload.Contains(string.Format("{0}={1}", WebHCatConstants.Output, Uri.EscapeDataString(streamingMapReduceJob.Output)))); Assert.IsTrue(payload.Contains(string.Format("{0}={1}", WebHCatConstants.Mapper, Uri.EscapeDataString(streamingMapReduceJob.Mapper)))); }
public ClusterJob Submit(XDocument config, Uri errorLocation) { // upload config xml doc to blob // launch single task map only job to submit xonfig xml to yarn launcher // get job output which will be the application id Uri userDfs = this.DfsClient.Combine(this.baseUri, "user", Environment.UserName, "staging"); Guid jobGuid = Guid.NewGuid(); StreamingMapReduceJobCreateParameters jobParams = new StreamingMapReduceJobCreateParameters(); jobParams.Defines.Add("mapreduce.job.reduces", "0"); jobParams.Defines.Add("mapreduce.job.maps", "1"); jobParams.JobName = "YarnMapLauncher"; jobParams.Mapper = "Microsoft.Research.Peloponnese.YarnMapLauncher.exe"; Uri outputDir = this.DfsClient.Combine(userDfs, jobGuid.ToString()); //dfsClient removes the leading slash, but hadoop assumes job output is relative to user working directory jobParams.Output = this.dfsClient.GetClusterInternalUri(outputDir).AbsoluteUri; using (MemoryStream upload = new MemoryStream()) { XmlWriterSettings settings = new XmlWriterSettings(); settings.Encoding = new UTF8Encoding(false); using (XmlWriter xw = XmlWriter.Create(upload, settings)) { config.WriteTo(xw); } XElement elem = ConfigHelpers.MakeDfsResourceFromBuffer(jobGuid + ".xml", upload.ToArray(), this.DfsClient, userDfs); jobParams.Input = elem.Value; } Uri peloponneseDfs = this.DfsClient.Combine(this.baseUri, "staging", "peloponnese"); string[] files = { jobParams.Mapper, "Microsoft.Research.Peloponnese.YarnLauncher.jar" }; foreach (string file in files) { XElement elem = ConfigHelpers.MakeDfsResourceFromFile(Path.Combine(peloponneseHome, file), this.DfsClient, peloponneseDfs); // hash mark in resources allows a target name to be specified for the symlink jobParams.Files.Add(string.Format("{0}#{1}", elem.Value, file)); } var launchJob = JobClient.CreateStreamingJob(jobParams); return(new AzureYarnJob(this, errorLocation, outputDir, launchJob)); }
public void ICanCallThe_New_HDInsightStreamingMapReduceJobDefinitionCmdlet_WithParameters() { var streamingMapReduceJobDefinition = new StreamingMapReduceJobCreateParameters { JobName = "pi estimation jobDetails", Input = TestConstants.WabsProtocolSchemeName + "input", Output = TestConstants.WabsProtocolSchemeName + "input", Mapper = TestConstants.WabsProtocolSchemeName + "combiner", Reducer = TestConstants.WabsProtocolSchemeName + "combiner", StatusFolder = TestConstants.WabsProtocolSchemeName + "someotherlocation" }; streamingMapReduceJobDefinition.Defines.Add("map.input.tasks", "1000"); streamingMapReduceJobDefinition.Defines.Add("map.input.reducers", "1000"); using (IRunspace runspace = this.GetPowerShellRunspace()) { IPipelineResult results = runspace.NewPipeline() .AddCommand(CmdletConstants.NewAzureHDInsightStreamingMapReduceJobDefinition) .WithParameter(CmdletConstants.JobName, streamingMapReduceJobDefinition.JobName) .WithParameter(CmdletConstants.Input, streamingMapReduceJobDefinition.Input) .WithParameter(CmdletConstants.Output, streamingMapReduceJobDefinition.Output) .WithParameter(CmdletConstants.Mapper, streamingMapReduceJobDefinition.Mapper) .WithParameter(CmdletConstants.Reducer, streamingMapReduceJobDefinition.Reducer) .WithParameter(CmdletConstants.StatusFolder, streamingMapReduceJobDefinition.StatusFolder) .WithParameter(CmdletConstants.Parameters, streamingMapReduceJobDefinition.Defines) .Invoke(); Assert.AreEqual(1, results.Results.Count); AzureHDInsightStreamingMapReduceJobDefinition streamingMapReduceJobFromPowershell = results.Results.ToEnumerable <AzureHDInsightStreamingMapReduceJobDefinition>().First(); AssertJobDefinitionsEqual(streamingMapReduceJobDefinition, streamingMapReduceJobFromPowershell); foreach (var parameter in streamingMapReduceJobDefinition.Defines) { Assert.IsTrue( streamingMapReduceJobFromPowershell.Defines.Any( arg => string.Equals(parameter.Key, arg.Key) && string.Equals(parameter.Value, arg.Value)), "Unable to find parameter '{0}' in value returned from powershell", parameter.Key); } } }
public void CanSerializeValidStreamingMapReduceJobRequest() { var streamingMapReduceJob = new StreamingMapReduceJobCreateParameters() { Input = "asv://input", Output = "asv://output", Mapper = "asv://mapper", Reducer = "asv://reducer", Combiner = "asv://combiner" }; var payloadConverter = new PayloadConverterBase(); var payload = payloadConverter.SerializeStreamingMapReduceRequest("hadoop", streamingMapReduceJob); Assert.IsTrue(payload.Contains(string.Format("{0}={1}", WebHCatConstants.Input, Uri.EscapeDataString(streamingMapReduceJob.Input)))); Assert.IsTrue(payload.Contains(string.Format("{0}={1}", WebHCatConstants.Output, Uri.EscapeDataString(streamingMapReduceJob.Output)))); Assert.IsTrue(payload.Contains(string.Format("{0}={1}", WebHCatConstants.Mapper, Uri.EscapeDataString(streamingMapReduceJob.Mapper)))); Assert.IsTrue(payload.Contains(string.Format("{0}={1}", WebHCatConstants.Reducer, Uri.EscapeDataString(streamingMapReduceJob.Reducer)))); Assert.IsTrue(payload.Contains(string.Format("{0}={1}", WebHCatConstants.Combiner, Uri.EscapeDataString(streamingMapReduceJob.Combiner)))); }
internal StreamingMapReduceJobCreateParameters ToStreamingMapReduceJobCreateParameters() { var streamingJobDefinition = new StreamingMapReduceJobCreateParameters { JobName = this.JobName, Input = this.Input, Output = this.Output, Reducer = this.Reducer, Combiner = this.Combiner, Mapper = this.Mapper, StatusFolder = this.StatusFolder }; streamingJobDefinition.CommandEnvironment.AddRange(this.CommandEnvironment); streamingJobDefinition.Arguments.AddRange(this.Arguments); streamingJobDefinition.Defines.AddRange(this.Defines); streamingJobDefinition.Files.AddRange(this.Files); return streamingJobDefinition; }
internal StreamingMapReduceJobCreateParameters ToStreamingMapReduceJobCreateParameters() { var streamingJobDefinition = new StreamingMapReduceJobCreateParameters { JobName = this.JobName, Input = this.Input, Output = this.Output, Reducer = this.Reducer, Combiner = this.Combiner, Mapper = this.Mapper, StatusFolder = this.StatusFolder }; streamingJobDefinition.CommandEnvironment.AddRange(this.CommandEnvironment); streamingJobDefinition.Arguments.AddRange(this.Arguments); streamingJobDefinition.Defines.AddRange(this.Defines); streamingJobDefinition.Files.AddRange(this.Files); return(streamingJobDefinition); }
public void ICanCallThe_New_HDInsightStreamingMapReduceJobDefinitionCmdlet_WithArguments() { var streamingMapReduceJobDefinition = new StreamingMapReduceJobCreateParameters { JobName = "pi estimation jobDetails", Input = TestConstants.WabsProtocolSchemeName + "input", Output = TestConstants.WabsProtocolSchemeName + "input", Mapper = TestConstants.WabsProtocolSchemeName + "combiner", Reducer = TestConstants.WabsProtocolSchemeName + "combiner", StatusFolder = TestConstants.WabsProtocolSchemeName + "someotherlocation" }; streamingMapReduceJobDefinition.Arguments.Add("16"); streamingMapReduceJobDefinition.Arguments.Add("10000"); using (IRunspace runspace = this.GetPowerShellRunspace()) { IPipelineResult results = runspace.NewPipeline() .AddCommand(CmdletConstants.NewAzureHDInsightStreamingMapReduceJobDefinition) .WithParameter(CmdletConstants.JobName, streamingMapReduceJobDefinition.JobName) .WithParameter(CmdletConstants.Input, streamingMapReduceJobDefinition.Input) .WithParameter(CmdletConstants.Output, streamingMapReduceJobDefinition.Output) .WithParameter(CmdletConstants.Mapper, streamingMapReduceJobDefinition.Mapper) .WithParameter(CmdletConstants.Reducer, streamingMapReduceJobDefinition.Reducer) .WithParameter(CmdletConstants.StatusFolder, streamingMapReduceJobDefinition.StatusFolder) .WithParameter(CmdletConstants.Arguments, streamingMapReduceJobDefinition.Arguments) .Invoke(); Assert.AreEqual(1, results.Results.Count); AzureHDInsightStreamingMapReduceJobDefinition streamingMapReduceJobFromPowershell = results.Results.ToEnumerable<AzureHDInsightStreamingMapReduceJobDefinition>().First(); AssertJobDefinitionsEqual(streamingMapReduceJobDefinition, streamingMapReduceJobFromPowershell); foreach (string argument in streamingMapReduceJobDefinition.Arguments) { Assert.IsTrue( streamingMapReduceJobFromPowershell.Arguments.Any(arg => string.Equals(argument, arg)), "Unable to find argument '{0}' in value returned from powershell", argument); } } }
public void CanCreateNewStreamingMapReduceDefinition() { StreamingMapReduceJobCreateParameters streamingMapReduceJobDefinition = this.GetStreamingMapReduceJob(); INewAzureHDInsightStreamingJobDefinitionCommand newStreamingMapReduceJobDefinitionCommand = ServiceLocator.Instance.Locate <IAzureHDInsightCommandFactory>().CreateNewStreamingMapReduceDefinition(); newStreamingMapReduceJobDefinitionCommand.JobName = streamingMapReduceJobDefinition.JobName; newStreamingMapReduceJobDefinitionCommand.JobName = streamingMapReduceJobDefinition.JobName; newStreamingMapReduceJobDefinitionCommand.InputPath = streamingMapReduceJobDefinition.Input; newStreamingMapReduceJobDefinitionCommand.OutputPath = streamingMapReduceJobDefinition.Output; newStreamingMapReduceJobDefinitionCommand.Mapper = streamingMapReduceJobDefinition.Mapper; newStreamingMapReduceJobDefinitionCommand.Reducer = streamingMapReduceJobDefinition.Reducer; newStreamingMapReduceJobDefinitionCommand.StatusFolder = streamingMapReduceJobDefinition.StatusFolder; newStreamingMapReduceJobDefinitionCommand.EndProcessing(); AzureHDInsightStreamingMapReduceJobDefinition streamingMapReduceJobFromCommand = newStreamingMapReduceJobDefinitionCommand.Output.ElementAt(0); NewStreamingMapReduceJobCmdLetTests.AssertJobDefinitionsEqual(streamingMapReduceJobDefinition, streamingMapReduceJobFromCommand); }
public void CanCreateNewStreamingMapReduceDefinition_WithParameters() { StreamingMapReduceJobCreateParameters streamingMapReduceJobDefinition = this.GetStreamingMapReduceJob(); streamingMapReduceJobDefinition.Defines.Add("map.input.tasks", "1000"); streamingMapReduceJobDefinition.Defines.Add("map.input.reducers", "1000"); INewAzureHDInsightStreamingJobDefinitionCommand newStreamingMapReduceJobDefinitionCommand = ServiceLocator.Instance.Locate <IAzureHDInsightCommandFactory>().CreateNewStreamingMapReduceDefinition(); newStreamingMapReduceJobDefinitionCommand.JobName = streamingMapReduceJobDefinition.JobName; newStreamingMapReduceJobDefinitionCommand.JobName = streamingMapReduceJobDefinition.JobName; newStreamingMapReduceJobDefinitionCommand.InputPath = streamingMapReduceJobDefinition.Input; newStreamingMapReduceJobDefinitionCommand.OutputPath = streamingMapReduceJobDefinition.Output; newStreamingMapReduceJobDefinitionCommand.Mapper = streamingMapReduceJobDefinition.Mapper; newStreamingMapReduceJobDefinitionCommand.Reducer = streamingMapReduceJobDefinition.Reducer; newStreamingMapReduceJobDefinitionCommand.StatusFolder = streamingMapReduceJobDefinition.StatusFolder; newStreamingMapReduceJobDefinitionCommand.Defines = new Hashtable(); foreach (var define in streamingMapReduceJobDefinition.Defines) { newStreamingMapReduceJobDefinitionCommand.Defines.Add(define.Key, define.Value); } newStreamingMapReduceJobDefinitionCommand.EndProcessing(); AzureHDInsightStreamingMapReduceJobDefinition streamingMapReduceJobFromCommand = newStreamingMapReduceJobDefinitionCommand.Output.ElementAt(0); NewStreamingMapReduceJobCmdLetTests.AssertJobDefinitionsEqual(streamingMapReduceJobDefinition, streamingMapReduceJobFromCommand); foreach (var parameter in streamingMapReduceJobDefinition.Defines) { Assert.IsTrue( streamingMapReduceJobFromCommand.Defines.Any( arg => string.Equals(parameter.Key, arg.Key) && string.Equals(parameter.Value, arg.Value)), "Unable to find parameter '{0}' in value returned from command", parameter.Key); } }
private static async Task <JobCreationResults> CreateStreamingJob( AzureHDInsightStreamingMapReduceJobDefinition azureStreamingJobDefinition, IJobSubmissionClient client) { var streamingJobDefinition = new StreamingMapReduceJobCreateParameters { JobName = azureStreamingJobDefinition.JobName, Input = azureStreamingJobDefinition.Input, Output = azureStreamingJobDefinition.Output, Reducer = azureStreamingJobDefinition.Reducer, Combiner = azureStreamingJobDefinition.Combiner, Mapper = azureStreamingJobDefinition.Mapper }; streamingJobDefinition.StatusFolder = azureStreamingJobDefinition.StatusFolder; streamingJobDefinition.CommandEnvironment.AddRange(azureStreamingJobDefinition.CommandEnvironment); streamingJobDefinition.Arguments.AddRange(azureStreamingJobDefinition.Arguments); streamingJobDefinition.Defines.AddRange(azureStreamingJobDefinition.Defines); streamingJobDefinition.Files.AddRange(azureStreamingJobDefinition.Files); var jobCreationResults = await client.CreateStreamingJobAsync(streamingJobDefinition); return(jobCreationResults); }
public virtual void PipeliningStreamingMapReduceJobExecution() { var streamingMapReduceJobDefinition = new StreamingMapReduceJobCreateParameters { JobName = "pi estimation job", Mapper = "environmentvariables.exe", Input = "/example/apps/environmentvariables.exe", Output = Guid.NewGuid().ToString() }; var files = new List <string> { streamingMapReduceJobDefinition.Mapper }; using (IRunspace runspace = this.GetPowerShellRunspace()) { ClusterDetails testCluster = GetHttpAccessEnabledCluster(); IPipelineResult results = runspace.NewPipeline() .AddCommand(CmdletConstants.NewAzureHDInsightStreamingMapReduceJobDefinition) .WithParameter(CmdletConstants.JobName, streamingMapReduceJobDefinition.JobName) .WithParameter(CmdletConstants.Mapper, streamingMapReduceJobDefinition.Mapper) .WithParameter(CmdletConstants.Input, streamingMapReduceJobDefinition.Input) .WithParameter(CmdletConstants.Output, streamingMapReduceJobDefinition.Output) .WithParameter(CmdletConstants.Files, files) .AddCommand(CmdletConstants.StartAzureHDInsightJob) .WithParameter(CmdletConstants.Cluster, testCluster.Name) .AddCommand(CmdletConstants.WaitAzureHDInsightJob) .WithParameter(CmdletConstants.WaitTimeoutInSeconds, 10) .AddCommand(CmdletConstants.GetAzureHDInsightJobOutput) .WithParameter(CmdletConstants.Cluster, testCluster.Name) .Invoke(); Assert.IsNotNull(results.Results.ToEnumerable <string>()); Assert.IsTrue(results.Results.ToEnumerable <string>().Any(str => str == "3.142")); } }
/// <inheritdoc /> public async Task <JobCreationResults> SubmitStreamingJob(StreamingMapReduceJobCreateParameters pigJobCreateParameters) { var remoteClient = ServiceLocator.Instance.Locate <IRemoteHadoopJobSubmissionPocoClientFactory>().Create(this.remoteCreds, this.context, this.ignoreSslErrors, this.GetUserAgentString()); return(await remoteClient.SubmitStreamingJob(pigJobCreateParameters)); }
public Task <JobCreationResults> SubmitStreamingJob(StreamingMapReduceJobCreateParameters pigJobCreateParameters) { throw new System.NotImplementedException(); }
public JobCreationResults CreateStreamingJob(StreamingMapReduceJobCreateParameters streamingMapReduceJobCreateParameters) { return this.CreateStreamingJobAsync(streamingMapReduceJobCreateParameters).WaitForResult(); }
public Task<JobCreationResults> CreateStreamingJobAsync(StreamingMapReduceJobCreateParameters streamingMapReduceJobCreateParameters) { if (streamingMapReduceJobCreateParameters == null) { throw new ArgumentNullException("streamingMapReduceJobCreateParameters"); } return TaskEx2.FromResult(this.CreateJobSuccessResult(streamingMapReduceJobCreateParameters, streamingMapReduceJobCreateParameters.JobName)); }
/// <inheritdoc /> public async Task<JobCreationResults> SubmitStreamingJob(StreamingMapReduceJobCreateParameters streamingJobCreateParameters) { //NEIN: Any code modification here should add unit tests for this class var converter = new PayloadConverter(); var payload = converter.SerializeStreamingMapReduceRequest(this.credentials.UserName, streamingJobCreateParameters); var client = ServiceLocator.Instance.Locate<IHadoopRemoteJobSubmissionRestClientFactory>().Create(this.credentials, this.context, this.ignoreSslErrors, this.GetUserAgentString()); var result = await client.SubmitStreamingMapReduceJob(payload); return new JobCreationResults() { JobId = converter.DeserializeJobSubmissionResponse(result.Content) }; }
public async Task<JobCreationResults> SubmitStreamingJob(StreamingMapReduceJobCreateParameters pigJobCreateParameters) { var relative = new Uri( HadoopRemoteRestConstants.Sqoop + "?" + HadoopRemoteRestConstants.UserName + "=" + _connectionConfig.UserName.EscapeDataString(), UriKind.Relative); var requestContent = _converter.SerializeStreamingMapReduceRequest(_connectionConfig.UserName, pigJobCreateParameters); var result = await MakeAsyncJobSubmissionRequest(relative, requestContent); return new JobCreationResults { JobId = _converter.DeserializeJobSubmissionResponse(result) }; }
public void CanCreateNewStreamingJob_StartJob() { var streamingMapReduceJobDefinition = new StreamingMapReduceJobCreateParameters { JobName = "pi estimation jobDetails", Input = TestConstants.WabsProtocolSchemeName + "container@hostname/input", Output = TestConstants.WabsProtocolSchemeName + "container@hostname/input", Mapper = TestConstants.WabsProtocolSchemeName + "container@hostname/combiner", Reducer = TestConstants.WabsProtocolSchemeName + "container@hostname/combiner", StatusFolder = TestConstants.WabsProtocolSchemeName + "container@hostname/someotherlocation" }; INewAzureHDInsightStreamingJobDefinitionCommand newStreamingMapReduceJobDefinitionCommand = ServiceLocator.Instance.Locate<IAzureHDInsightCommandFactory>().CreateNewStreamingMapReduceDefinition(); newStreamingMapReduceJobDefinitionCommand.JobName = streamingMapReduceJobDefinition.JobName; newStreamingMapReduceJobDefinitionCommand.JobName = streamingMapReduceJobDefinition.JobName; newStreamingMapReduceJobDefinitionCommand.InputPath = streamingMapReduceJobDefinition.Input; newStreamingMapReduceJobDefinitionCommand.OutputPath = streamingMapReduceJobDefinition.Output; newStreamingMapReduceJobDefinitionCommand.Mapper = streamingMapReduceJobDefinition.Mapper; newStreamingMapReduceJobDefinitionCommand.Reducer = streamingMapReduceJobDefinition.Reducer; newStreamingMapReduceJobDefinitionCommand.StatusFolder = streamingMapReduceJobDefinition.StatusFolder; newStreamingMapReduceJobDefinitionCommand.EndProcessing(); AzureHDInsightStreamingMapReduceJobDefinition streamingJobFromCommand = newStreamingMapReduceJobDefinitionCommand.Output.ElementAt(0); TestJobStart(streamingJobFromCommand); }
public JobCreationResults CreateStreamingJob(StreamingMapReduceJobCreateParameters streamingMapReduceJobCreateParameters) { return(this.CreateStreamingJobAsync(streamingMapReduceJobCreateParameters).WaitForResult()); }