public NewAzureHDInsightStreamingMapReduceJobDefinitionCommand() { Arguments = new string[] {}; CommandEnvironment = new string[] {}; Defines = new Hashtable(); job = new AzureHDInsightStreamingMapReduceJobDefinition(); }
public virtual void ICanCallThe_NewStreamingJob_Then_Start_HDInsightJobsCmdlet() { var streamingMapReduceJobDefinition = new StreamingMapReduceJobCreateParameters { JobName = "pi estimation jobDetails", Input = TestConstants.WabsProtocolSchemeName + "input", Output = TestConstants.WabsProtocolSchemeName + "input", Mapper = TestConstants.WabsProtocolSchemeName + "combiner", Reducer = TestConstants.WabsProtocolSchemeName + "combiner", StatusFolder = TestConstants.WabsProtocolSchemeName + "someotherlocation" }; using (IRunspace runspace = this.GetPowerShellRunspace()) { IPipelineResult results = runspace.NewPipeline() .AddCommand(CmdletConstants.NewAzureHDInsightStreamingMapReduceJobDefinition) .WithParameter(CmdletConstants.JobName, streamingMapReduceJobDefinition.JobName) .WithParameter(CmdletConstants.Input, streamingMapReduceJobDefinition.Input) .WithParameter(CmdletConstants.Output, streamingMapReduceJobDefinition.Output) .WithParameter(CmdletConstants.Mapper, streamingMapReduceJobDefinition.Mapper) .WithParameter(CmdletConstants.Reducer, streamingMapReduceJobDefinition.Reducer) .WithParameter(CmdletConstants.StatusFolder, streamingMapReduceJobDefinition.StatusFolder) .Invoke(); Assert.AreEqual(1, results.Results.Count); AzureHDInsightStreamingMapReduceJobDefinition streamingJobFromPowershell = results.Results.ToEnumerable <AzureHDInsightStreamingMapReduceJobDefinition>().First(); RunJobInPowershell(runspace, streamingJobFromPowershell); } }
public virtual void NewStreamingMapReduceJob_StartJob_GetJob() { var streamingMapReduceJobDefinition = new StreamingMapReduceJobCreateParameters { JobName = "environment variable jobDetails", Mapper = "environmentvariables.exe", Input = "/example/apps/environmentvariables.exe", Output = Guid.NewGuid().ToString() }; var files = new List <string> { streamingMapReduceJobDefinition.Mapper }; using (IRunspace runspace = this.GetPowerShellRunspace()) { IPipelineResult results = runspace.NewPipeline() .AddCommand(CmdletConstants.NewAzureHDInsightStreamingMapReduceJobDefinition) .WithParameter(CmdletConstants.JobName, streamingMapReduceJobDefinition.JobName) .WithParameter(CmdletConstants.Mapper, streamingMapReduceJobDefinition.Mapper) .WithParameter(CmdletConstants.Input, streamingMapReduceJobDefinition.Input) .WithParameter(CmdletConstants.Output, streamingMapReduceJobDefinition.Output) .WithParameter(CmdletConstants.Files, files) .Invoke(); Assert.AreEqual(1, results.Results.Count); AzureHDInsightStreamingMapReduceJobDefinition streamingMapReduceJobFromPowershell = results.Results.ToEnumerable <AzureHDInsightStreamingMapReduceJobDefinition>().First(); RunJobAndGetWithId(runspace, streamingMapReduceJobFromPowershell); } }
public void CanCreateNewStreamingMapReduceDefinition_WithArguments() { StreamingMapReduceJobCreateParameters streamingMapReduceJobDefinition = this.GetStreamingMapReduceJob(); streamingMapReduceJobDefinition.Arguments.Add("16"); streamingMapReduceJobDefinition.Arguments.Add("10000"); INewAzureHDInsightStreamingJobDefinitionCommand newStreamingMapReduceJobDefinitionCommand = ServiceLocator.Instance.Locate <IAzureHDInsightCommandFactory>().CreateNewStreamingMapReduceDefinition(); newStreamingMapReduceJobDefinitionCommand.JobName = streamingMapReduceJobDefinition.JobName; newStreamingMapReduceJobDefinitionCommand.JobName = streamingMapReduceJobDefinition.JobName; newStreamingMapReduceJobDefinitionCommand.InputPath = streamingMapReduceJobDefinition.Input; newStreamingMapReduceJobDefinitionCommand.OutputPath = streamingMapReduceJobDefinition.Output; newStreamingMapReduceJobDefinitionCommand.Mapper = streamingMapReduceJobDefinition.Mapper; newStreamingMapReduceJobDefinitionCommand.Reducer = streamingMapReduceJobDefinition.Reducer; newStreamingMapReduceJobDefinitionCommand.StatusFolder = streamingMapReduceJobDefinition.StatusFolder; newStreamingMapReduceJobDefinitionCommand.Arguments = streamingMapReduceJobDefinition.Arguments.ToArray(); newStreamingMapReduceJobDefinitionCommand.EndProcessing(); AzureHDInsightStreamingMapReduceJobDefinition streamingMapReduceJobFromCommand = newStreamingMapReduceJobDefinitionCommand.Output.ElementAt(0); NewStreamingMapReduceJobCmdLetTests.AssertJobDefinitionsEqual(streamingMapReduceJobDefinition, streamingMapReduceJobFromCommand); foreach (string argument in streamingMapReduceJobDefinition.Arguments) { Assert.IsTrue( streamingMapReduceJobFromCommand.Arguments.Any(arg => string.Equals(argument, arg)), "Unable to find argument '{0}' in value returned from command", argument); } }
public void CanCreateNewStreamingJob_StartJob() { var streamingMapReduceJobDefinition = new StreamingMapReduceJobCreateParameters { JobName = "pi estimation jobDetails", Input = TestConstants.WabsProtocolSchemeName + "container@hostname/input", Output = TestConstants.WabsProtocolSchemeName + "container@hostname/input", Mapper = TestConstants.WabsProtocolSchemeName + "container@hostname/combiner", Reducer = TestConstants.WabsProtocolSchemeName + "container@hostname/combiner", StatusFolder = TestConstants.WabsProtocolSchemeName + "container@hostname/someotherlocation" }; INewAzureHDInsightStreamingJobDefinitionCommand newStreamingMapReduceJobDefinitionCommand = ServiceLocator.Instance.Locate <IAzureHDInsightCommandFactory>().CreateNewStreamingMapReduceDefinition(); newStreamingMapReduceJobDefinitionCommand.JobName = streamingMapReduceJobDefinition.JobName; newStreamingMapReduceJobDefinitionCommand.JobName = streamingMapReduceJobDefinition.JobName; newStreamingMapReduceJobDefinitionCommand.InputPath = streamingMapReduceJobDefinition.Input; newStreamingMapReduceJobDefinitionCommand.OutputPath = streamingMapReduceJobDefinition.Output; newStreamingMapReduceJobDefinitionCommand.Mapper = streamingMapReduceJobDefinition.Mapper; newStreamingMapReduceJobDefinitionCommand.Reducer = streamingMapReduceJobDefinition.Reducer; newStreamingMapReduceJobDefinitionCommand.StatusFolder = streamingMapReduceJobDefinition.StatusFolder; newStreamingMapReduceJobDefinitionCommand.EndProcessing(); AzureHDInsightStreamingMapReduceJobDefinition streamingJobFromCommand = newStreamingMapReduceJobDefinitionCommand.Output.ElementAt(0); TestJobStart(streamingJobFromCommand); }
internal static void AssertJobDefinitionsEqual( StreamingMapReduceJobCreateParameters streamingMapReduceJobDefinition, AzureHDInsightStreamingMapReduceJobDefinition streamingMapReduceJobFromPowershell) { Assert.AreEqual(streamingMapReduceJobDefinition.JobName, streamingMapReduceJobFromPowershell.JobName); Assert.AreEqual(streamingMapReduceJobDefinition.Input, streamingMapReduceJobFromPowershell.Input); Assert.AreEqual(streamingMapReduceJobDefinition.Output, streamingMapReduceJobFromPowershell.Output); Assert.AreEqual(streamingMapReduceJobDefinition.Mapper, streamingMapReduceJobFromPowershell.Mapper); Assert.AreEqual(streamingMapReduceJobDefinition.Reducer, streamingMapReduceJobFromPowershell.Reducer); Assert.AreEqual(streamingMapReduceJobDefinition.StatusFolder, streamingMapReduceJobFromPowershell.StatusFolder); }
public void CanCreateNewStreamingMapReduceJob_WithoutJobName_FilesRelative() { var streamingMapReduceJobDefinition = new AzureHDInsightStreamingMapReduceJobDefinition { Input = "input", Output = "output", Mapper = "/examples/mapper.exe", Reducer = "/examples/combiner.exe", StatusFolder = "/someotherlocation" }; AzureHDInsightJob startedJob = TestJobStart(streamingMapReduceJobDefinition); Assert.AreEqual("mapper.exe", startedJob.Name); }
public void CanCreateNewStreamingMapReduceJob_WithoutJobName() { var streamingMapReduceJobDefinition = new AzureHDInsightStreamingMapReduceJobDefinition { Input = TestConstants.WabsProtocolSchemeName + "container@hostname/input", Output = TestConstants.WabsProtocolSchemeName + "container@hostname/input", Mapper = TestConstants.WabsProtocolSchemeName + "container@hostname/mapper", Reducer = TestConstants.WabsProtocolSchemeName + "container@hostname/combiner", StatusFolder = TestConstants.WabsProtocolSchemeName + "container@hostname/someotherlocation" }; AzureHDInsightJob startedJob = TestJobStart(streamingMapReduceJobDefinition); Assert.AreEqual(startedJob.Name, "mapper"); }
public void ICanCallThe_New_HDInsightStreamingMapReduceJobDefinitionCmdlet_WithParameters() { var streamingMapReduceJobDefinition = new StreamingMapReduceJobCreateParameters { JobName = "pi estimation jobDetails", Input = TestConstants.WabsProtocolSchemeName + "input", Output = TestConstants.WabsProtocolSchemeName + "input", Mapper = TestConstants.WabsProtocolSchemeName + "combiner", Reducer = TestConstants.WabsProtocolSchemeName + "combiner", StatusFolder = TestConstants.WabsProtocolSchemeName + "someotherlocation" }; streamingMapReduceJobDefinition.Defines.Add("map.input.tasks", "1000"); streamingMapReduceJobDefinition.Defines.Add("map.input.reducers", "1000"); using (IRunspace runspace = this.GetPowerShellRunspace()) { IPipelineResult results = runspace.NewPipeline() .AddCommand(CmdletConstants.NewAzureHDInsightStreamingMapReduceJobDefinition) .WithParameter(CmdletConstants.JobName, streamingMapReduceJobDefinition.JobName) .WithParameter(CmdletConstants.Input, streamingMapReduceJobDefinition.Input) .WithParameter(CmdletConstants.Output, streamingMapReduceJobDefinition.Output) .WithParameter(CmdletConstants.Mapper, streamingMapReduceJobDefinition.Mapper) .WithParameter(CmdletConstants.Reducer, streamingMapReduceJobDefinition.Reducer) .WithParameter(CmdletConstants.StatusFolder, streamingMapReduceJobDefinition.StatusFolder) .WithParameter(CmdletConstants.Parameters, streamingMapReduceJobDefinition.Defines) .Invoke(); Assert.AreEqual(1, results.Results.Count); AzureHDInsightStreamingMapReduceJobDefinition streamingMapReduceJobFromPowershell = results.Results.ToEnumerable <AzureHDInsightStreamingMapReduceJobDefinition>().First(); AssertJobDefinitionsEqual(streamingMapReduceJobDefinition, streamingMapReduceJobFromPowershell); foreach (var parameter in streamingMapReduceJobDefinition.Defines) { Assert.IsTrue( streamingMapReduceJobFromPowershell.Defines.Any( arg => string.Equals(parameter.Key, arg.Key) && string.Equals(parameter.Value, arg.Value)), "Unable to find parameter '{0}' in value returned from powershell", parameter.Key); } } }
public void CanCreateNewStreamingMapReduceDefinition() { StreamingMapReduceJobCreateParameters streamingMapReduceJobDefinition = this.GetStreamingMapReduceJob(); INewAzureHDInsightStreamingJobDefinitionCommand newStreamingMapReduceJobDefinitionCommand = ServiceLocator.Instance.Locate <IAzureHDInsightCommandFactory>().CreateNewStreamingMapReduceDefinition(); newStreamingMapReduceJobDefinitionCommand.JobName = streamingMapReduceJobDefinition.JobName; newStreamingMapReduceJobDefinitionCommand.JobName = streamingMapReduceJobDefinition.JobName; newStreamingMapReduceJobDefinitionCommand.InputPath = streamingMapReduceJobDefinition.Input; newStreamingMapReduceJobDefinitionCommand.OutputPath = streamingMapReduceJobDefinition.Output; newStreamingMapReduceJobDefinitionCommand.Mapper = streamingMapReduceJobDefinition.Mapper; newStreamingMapReduceJobDefinitionCommand.Reducer = streamingMapReduceJobDefinition.Reducer; newStreamingMapReduceJobDefinitionCommand.StatusFolder = streamingMapReduceJobDefinition.StatusFolder; newStreamingMapReduceJobDefinitionCommand.EndProcessing(); AzureHDInsightStreamingMapReduceJobDefinition streamingMapReduceJobFromCommand = newStreamingMapReduceJobDefinitionCommand.Output.ElementAt(0); NewStreamingMapReduceJobCmdLetTests.AssertJobDefinitionsEqual(streamingMapReduceJobDefinition, streamingMapReduceJobFromCommand); }
public void CanCreateNewStreamingMapReduceDefinition_WithParameters() { StreamingMapReduceJobCreateParameters streamingMapReduceJobDefinition = this.GetStreamingMapReduceJob(); streamingMapReduceJobDefinition.Defines.Add("map.input.tasks", "1000"); streamingMapReduceJobDefinition.Defines.Add("map.input.reducers", "1000"); INewAzureHDInsightStreamingJobDefinitionCommand newStreamingMapReduceJobDefinitionCommand = ServiceLocator.Instance.Locate <IAzureHDInsightCommandFactory>().CreateNewStreamingMapReduceDefinition(); newStreamingMapReduceJobDefinitionCommand.JobName = streamingMapReduceJobDefinition.JobName; newStreamingMapReduceJobDefinitionCommand.JobName = streamingMapReduceJobDefinition.JobName; newStreamingMapReduceJobDefinitionCommand.InputPath = streamingMapReduceJobDefinition.Input; newStreamingMapReduceJobDefinitionCommand.OutputPath = streamingMapReduceJobDefinition.Output; newStreamingMapReduceJobDefinitionCommand.Mapper = streamingMapReduceJobDefinition.Mapper; newStreamingMapReduceJobDefinitionCommand.Reducer = streamingMapReduceJobDefinition.Reducer; newStreamingMapReduceJobDefinitionCommand.StatusFolder = streamingMapReduceJobDefinition.StatusFolder; newStreamingMapReduceJobDefinitionCommand.Defines = new Hashtable(); foreach (var define in streamingMapReduceJobDefinition.Defines) { newStreamingMapReduceJobDefinitionCommand.Defines.Add(define.Key, define.Value); } newStreamingMapReduceJobDefinitionCommand.EndProcessing(); AzureHDInsightStreamingMapReduceJobDefinition streamingMapReduceJobFromCommand = newStreamingMapReduceJobDefinitionCommand.Output.ElementAt(0); NewStreamingMapReduceJobCmdLetTests.AssertJobDefinitionsEqual(streamingMapReduceJobDefinition, streamingMapReduceJobFromCommand); foreach (var parameter in streamingMapReduceJobDefinition.Defines) { Assert.IsTrue( streamingMapReduceJobFromCommand.Defines.Any( arg => string.Equals(parameter.Key, arg.Key) && string.Equals(parameter.Value, arg.Value)), "Unable to find parameter '{0}' in value returned from command", parameter.Key); } }
private static async Task <JobCreationResults> CreateStreamingJob( AzureHDInsightStreamingMapReduceJobDefinition azureStreamingJobDefinition, IJobSubmissionClient client) { var streamingJobDefinition = new StreamingMapReduceJobCreateParameters { JobName = azureStreamingJobDefinition.JobName, Input = azureStreamingJobDefinition.Input, Output = azureStreamingJobDefinition.Output, Reducer = azureStreamingJobDefinition.Reducer, Combiner = azureStreamingJobDefinition.Combiner, Mapper = azureStreamingJobDefinition.Mapper }; streamingJobDefinition.StatusFolder = azureStreamingJobDefinition.StatusFolder; streamingJobDefinition.CommandEnvironment.AddRange(azureStreamingJobDefinition.CommandEnvironment); streamingJobDefinition.Arguments.AddRange(azureStreamingJobDefinition.Arguments); streamingJobDefinition.Defines.AddRange(azureStreamingJobDefinition.Defines); streamingJobDefinition.Files.AddRange(azureStreamingJobDefinition.Files); var jobCreationResults = await client.CreateStreamingJobAsync(streamingJobDefinition); return(jobCreationResults); }
public void CanCreateNewStreamingMapReduceJob_WithoutJobName_FilesRelative() { var streamingMapReduceJobDefinition = new AzureHDInsightStreamingMapReduceJobDefinition { Input = "input", Output = "output", Mapper = "/examples/mapper.exe", Reducer = "/examples/combiner.exe", StatusFolder = "/someotherlocation" }; AzureHDInsightJob startedJob = TestJobStart(streamingMapReduceJobDefinition); Assert.AreEqual("mapper.exe", startedJob.Name); }
public void CanCreateNewStreamingMapReduceJob_WithoutJobName() { var streamingMapReduceJobDefinition = new AzureHDInsightStreamingMapReduceJobDefinition { Input = TestConstants.WabsProtocolSchemeName + "container@hostname/input", Output = TestConstants.WabsProtocolSchemeName + "container@hostname/input", Mapper = TestConstants.WabsProtocolSchemeName + "container@hostname/mapper", Reducer = TestConstants.WabsProtocolSchemeName + "container@hostname/combiner", StatusFolder = TestConstants.WabsProtocolSchemeName + "container@hostname/someotherlocation" }; AzureHDInsightJob startedJob = TestJobStart(streamingMapReduceJobDefinition); Assert.AreEqual(startedJob.Name, "mapper"); }
internal static void AssertJobDefinitionsEqual( StreamingMapReduceJobCreateParameters streamingMapReduceJobDefinition, AzureHDInsightStreamingMapReduceJobDefinition streamingMapReduceJobFromPowershell) { Assert.AreEqual(streamingMapReduceJobDefinition.JobName, streamingMapReduceJobFromPowershell.JobName); Assert.AreEqual(streamingMapReduceJobDefinition.Input, streamingMapReduceJobFromPowershell.Input); Assert.AreEqual(streamingMapReduceJobDefinition.Output, streamingMapReduceJobFromPowershell.Output); Assert.AreEqual(streamingMapReduceJobDefinition.Mapper, streamingMapReduceJobFromPowershell.Mapper); Assert.AreEqual(streamingMapReduceJobDefinition.Reducer, streamingMapReduceJobFromPowershell.Reducer); Assert.AreEqual(streamingMapReduceJobDefinition.StatusFolder, streamingMapReduceJobFromPowershell.StatusFolder); }