public void CanCreateNewMapReduceDefinition_WithArguments() { var mapReduceJobDefinition = new MapReduceJobCreateParameters { JobName = "pi estimation jobDetails", ClassName = "pi", JarFile = TestConstants.WabsProtocolSchemeName + "container@hostname/examples.jar" }; mapReduceJobDefinition.Arguments.Add("16"); mapReduceJobDefinition.Arguments.Add("10000"); INewAzureHDInsightMapReduceJobDefinitionCommand newMapReduceJobDefinitionCommand = ServiceLocator.Instance.Locate <IAzureHDInsightCommandFactory>().CreateNewMapReduceDefinition(); newMapReduceJobDefinitionCommand.JobName = mapReduceJobDefinition.JobName; newMapReduceJobDefinitionCommand.JarFile = mapReduceJobDefinition.JarFile; newMapReduceJobDefinitionCommand.ClassName = mapReduceJobDefinition.ClassName; newMapReduceJobDefinitionCommand.Arguments = mapReduceJobDefinition.Arguments.ToArray(); newMapReduceJobDefinitionCommand.EndProcessing(); AzureHDInsightMapReduceJobDefinition mapReduceJobFromCommand = newMapReduceJobDefinitionCommand.Output.ElementAt(0); Assert.AreEqual(mapReduceJobDefinition.JobName, mapReduceJobFromCommand.JobName); Assert.AreEqual(mapReduceJobDefinition.ClassName, mapReduceJobFromCommand.ClassName); Assert.AreEqual(mapReduceJobDefinition.JarFile, mapReduceJobFromCommand.JarFile); foreach (string argument in mapReduceJobDefinition.Arguments) { Assert.IsTrue( mapReduceJobFromCommand.Arguments.Any(arg => string.Equals(argument, arg)), "Unable to find argument '{0}' in value returned from command", argument); } }
public virtual void ICanCallThe_NewMapReduceJob_Then_Start_HDInsightJobsCmdlet() { var mapReduceJobDefinition = new MapReduceJobCreateParameters { JobName = "pi estimation jobDetails", ClassName = "pi", JarFile = "/example/hadoop-examples.jar" }; mapReduceJobDefinition.Arguments.Add("16"); mapReduceJobDefinition.Arguments.Add("10000"); using (IRunspace runspace = this.GetPowerShellRunspace()) { IPipelineResult results = runspace.NewPipeline() .AddCommand(CmdletConstants.NewAzureHDInsightMapReduceJobDefinition) .WithParameter(CmdletConstants.JobName, mapReduceJobDefinition.JobName) .WithParameter(CmdletConstants.JarFile, mapReduceJobDefinition.JarFile) .WithParameter(CmdletConstants.ClassName, mapReduceJobDefinition.ClassName) .WithParameter(CmdletConstants.Arguments, mapReduceJobDefinition.Arguments) .Invoke(); Assert.AreEqual(1, results.Results.Count); AzureHDInsightMapReduceJobDefinition mapReduceJobFromPowershell = results.Results.ToEnumerable <AzureHDInsightMapReduceJobDefinition>().First(); RunJobInPowershell(runspace, mapReduceJobFromPowershell); } }
public void ICanCallThe_New_HDInsightMapReduceJobDefinitionCmdlet() { var mapReduceJobDefinition = new MapReduceJobCreateParameters { JobName = "pi estimation jobDetails", ClassName = "pi", JarFile = TestConstants.WabsProtocolSchemeName + "container@hostname/examples.jar", }; mapReduceJobDefinition.LibJars.Add("some.jarfile.jar"); using (IRunspace runspace = this.GetPowerShellRunspace()) { IPipelineResult results = runspace.NewPipeline() .AddCommand(CmdletConstants.NewAzureHDInsightMapReduceJobDefinition) .WithParameter(CmdletConstants.JobName, mapReduceJobDefinition.JobName) .WithParameter(CmdletConstants.JarFile, mapReduceJobDefinition.JarFile) .WithParameter(CmdletConstants.ClassName, mapReduceJobDefinition.ClassName) .WithParameter(CmdletConstants.LibJars, mapReduceJobDefinition.LibJars) .Invoke(); Assert.AreEqual(1, results.Results.Count); AzureHDInsightMapReduceJobDefinition mapReduceJobFromPowershell = results.Results.ToEnumerable <AzureHDInsightMapReduceJobDefinition>().First(); Assert.AreEqual(mapReduceJobDefinition.JobName, mapReduceJobFromPowershell.JobName); Assert.AreEqual(mapReduceJobDefinition.ClassName, mapReduceJobFromPowershell.ClassName); Assert.AreEqual(mapReduceJobDefinition.JarFile, mapReduceJobFromPowershell.JarFile); Assert.AreEqual(mapReduceJobDefinition.LibJars.Count, mapReduceJobFromPowershell.LibJars.Count); Assert.AreEqual(mapReduceJobDefinition.LibJars.First(), mapReduceJobFromPowershell.LibJars.First()); } }
public NewAzureHDInsightMapReduceJobDefinitionCommand() { Arguments = new string[] { }; Files = new string[] { }; Defines = new Hashtable(); LibJars = new string[] { }; job = new AzureHDInsightMapReduceJobDefinition(); }
private static async Task <JobCreationResults> SubmitMapReduceJob( AzureHDInsightMapReduceJobDefinition azureMapReduceJobDefinition, IJobSubmissionClient client) { MapReduceJobCreateParameters mapReduceJobDefinition = azureMapReduceJobDefinition.ToMapReduceJobCreateParameters(); var jobCreationResults = await client.CreateMapReduceJobAsync(mapReduceJobDefinition); return(jobCreationResults); }
public void CanStartMapReduceJob() { var mapReduceJobDefinition = new AzureHDInsightMapReduceJobDefinition { JobName = "pi estimation jobDetails", ClassName = "pi", JarFile = TestConstants.WabsProtocolSchemeName + "container@hostname/examples.jar" }; TestJobStart(mapReduceJobDefinition); }
public void CanAutoGenerateStatusDirectoryForMapReduceJob() { var mapReduceJobDefinition = new AzureHDInsightMapReduceJobDefinition { JobName = "pi estimation jobDetails", ClassName = "pi", JarFile = TestConstants.WabsProtocolSchemeName + "container@hostname/examples.jar" }; AzureHDInsightJob startedJob = TestJobStart(mapReduceJobDefinition); Assert.IsFalse(string.IsNullOrEmpty(startedJob.StatusDirectory)); }
public virtual void ICanCallThe_Start_HDInsightJobsCmdlet_WithoutName() { var mapReduceJobDefinition = new AzureHDInsightMapReduceJobDefinition { ClassName = "pi", JarFile = TestConstants.WabsProtocolSchemeName + "container@hostname/examples.jar" }; using (IRunspace runspace = this.GetPowerShellRunspace()) { RunJobInPowershell(runspace, mapReduceJobDefinition); } }
public void CanAutoGenerateStatusDirectoryForMapReduceJob() { var mapReduceJobDefinition = new AzureHDInsightMapReduceJobDefinition { JobName = "pi estimation jobDetails", ClassName = "pi", JarFile = TestConstants.WabsProtocolSchemeName + "container@hostname/examples.jar" }; AzureHDInsightJob startedJob = TestJobStart(mapReduceJobDefinition); Assert.IsFalse(string.IsNullOrEmpty(startedJob.StatusDirectory)); }
public void CanCreateNewMapReduceJob_WithoutJobName() { var mapReduceJobDefinition = new AzureHDInsightMapReduceJobDefinition { ClassName = "pi", JarFile = TestConstants.WabsProtocolSchemeName + "container@hostname/examples.jar", StatusFolder = "/myoutputfolder" }; AzureHDInsightJob startedJob = TestJobStart(mapReduceJobDefinition); Assert.AreEqual("pi", startedJob.Name); }
public void UserCanSupplyStatusDirectoryForMapReduceJob() { var mapReduceJobDefinition = new AzureHDInsightMapReduceJobDefinition { JobName = "pi estimation jobDetails", ClassName = "pi", JarFile = TestConstants.WabsProtocolSchemeName + "container@hostname/examples.jar", StatusFolder = "/myoutputfolder" }; AzureHDInsightJob startedJob = TestJobStart(mapReduceJobDefinition); Assert.AreEqual(startedJob.StatusDirectory, mapReduceJobDefinition.StatusFolder); }
public void ICanCallThe_Start_HDInsightJobsCmdlet_WithDebug() { var mapReduceJobDefinition = new AzureHDInsightMapReduceJobDefinition { JobName = "pi estimation jobDetails", ClassName = "pi", JarFile = TestConstants.WabsProtocolSchemeName + "container@hostname/examples.jar" }; using (IRunspace runspace = this.GetPowerShellRunspace()) { string expectedLogMessage = string.Format(CultureInfo.InvariantCulture, "Starting jobDetails '{0}'.", mapReduceJobDefinition.JobName); RunJobInPowershell( runspace, mapReduceJobDefinition, CmdletScenariosTestCaseBase.GetHttpAccessEnabledCluster(), true, expectedLogMessage); } }
public void CanCreateNewMapReduceJob_StartJob() { var mapReduceJobDefinition = new MapReduceJobCreateParameters { JobName = "pi estimation jobDetails", ClassName = "pi", JarFile = TestConstants.WabsProtocolSchemeName + "container@hostname/examples.jar" }; INewAzureHDInsightMapReduceJobDefinitionCommand newMapReduceJobDefinitionCommand = ServiceLocator.Instance.Locate <IAzureHDInsightCommandFactory>().CreateNewMapReduceDefinition(); newMapReduceJobDefinitionCommand.JobName = mapReduceJobDefinition.JobName; newMapReduceJobDefinitionCommand.JarFile = mapReduceJobDefinition.JarFile; newMapReduceJobDefinitionCommand.ClassName = mapReduceJobDefinition.ClassName; newMapReduceJobDefinitionCommand.EndProcessing(); AzureHDInsightMapReduceJobDefinition mapReduceJobFromCommand = newMapReduceJobDefinitionCommand.Output.ElementAt(0); TestJobStart(mapReduceJobFromCommand); }
public void ICanCallThe_New_HDInsightMapReduceJobDefinitionCmdlet_WithParameters() { var mapReduceJobDefinition = new MapReduceJobCreateParameters { JobName = "pi estimation jobDetails", ClassName = "pi", JarFile = TestConstants.WabsProtocolSchemeName + "container@hostname/examples.jar" }; mapReduceJobDefinition.Defines.Add("map.input.tasks", "1000"); mapReduceJobDefinition.Defines.Add("map.input.reducers", "1000"); using (IRunspace runspace = this.GetPowerShellRunspace()) { IPipelineResult results = runspace.NewPipeline() .AddCommand(CmdletConstants.NewAzureHDInsightMapReduceJobDefinition) .WithParameter(CmdletConstants.JobName, mapReduceJobDefinition.JobName) .WithParameter(CmdletConstants.JarFile, mapReduceJobDefinition.JarFile) .WithParameter(CmdletConstants.ClassName, mapReduceJobDefinition.ClassName) .WithParameter(CmdletConstants.Parameters, mapReduceJobDefinition.Defines) .Invoke(); Assert.AreEqual(1, results.Results.Count); AzureHDInsightMapReduceJobDefinition mapReduceJobFromPowershell = results.Results.ToEnumerable <AzureHDInsightMapReduceJobDefinition>().First(); Assert.AreEqual(mapReduceJobDefinition.JobName, mapReduceJobFromPowershell.JobName); Assert.AreEqual(mapReduceJobDefinition.ClassName, mapReduceJobFromPowershell.ClassName); Assert.AreEqual(mapReduceJobDefinition.JarFile, mapReduceJobFromPowershell.JarFile); foreach (var parameter in mapReduceJobDefinition.Defines) { Assert.IsTrue( mapReduceJobFromPowershell.Defines.Any( arg => string.Equals(parameter.Key, arg.Key) && string.Equals(parameter.Value, arg.Value)), "Unable to find parameter '{0}' in value returned from powershell", parameter.Key); } } }
public void CanCreateNewMapReduceDefinition_WithParameters() { var mapReduceJobDefinition = new MapReduceJobCreateParameters { JobName = "pi estimation jobDetails", ClassName = "pi", JarFile = TestConstants.WabsProtocolSchemeName + "container@hostname/examples.jar" }; mapReduceJobDefinition.Defines.Add("map.input.tasks", "1000"); mapReduceJobDefinition.Defines.Add("map.input.reducers", "1000"); INewAzureHDInsightMapReduceJobDefinitionCommand newMapReduceJobDefinitionCommand = ServiceLocator.Instance.Locate <IAzureHDInsightCommandFactory>().CreateNewMapReduceDefinition(); newMapReduceJobDefinitionCommand.JobName = mapReduceJobDefinition.JobName; newMapReduceJobDefinitionCommand.JarFile = mapReduceJobDefinition.JarFile; newMapReduceJobDefinitionCommand.ClassName = mapReduceJobDefinition.ClassName; foreach (var define in mapReduceJobDefinition.Defines) { newMapReduceJobDefinitionCommand.Defines.Add(define.Key, define.Value); } newMapReduceJobDefinitionCommand.EndProcessing(); AzureHDInsightMapReduceJobDefinition mapReduceJobFromCommand = newMapReduceJobDefinitionCommand.Output.ElementAt(0); Assert.AreEqual(mapReduceJobDefinition.JobName, mapReduceJobFromCommand.JobName); Assert.AreEqual(mapReduceJobDefinition.ClassName, mapReduceJobFromCommand.ClassName); Assert.AreEqual(mapReduceJobDefinition.JarFile, mapReduceJobFromCommand.JarFile); foreach (var parameter in mapReduceJobDefinition.Defines) { Assert.IsTrue( mapReduceJobFromCommand.Defines.Any(arg => string.Equals(parameter.Key, arg.Key) && string.Equals(parameter.Value, arg.Value)), "Unable to find parameter '{0}' in value returned from command", parameter.Key); } }
public void ICanCallThe_Start_HDInsightJobsCmdlet_WithDebug() { var mapReduceJobDefinition = new AzureHDInsightMapReduceJobDefinition { JobName = "pi estimation jobDetails", ClassName = "pi", JarFile = TestConstants.WabsProtocolSchemeName + "container@hostname/examples.jar" }; using (IRunspace runspace = this.GetPowerShellRunspace()) { string expectedLogMessage = string.Format(CultureInfo.InvariantCulture, "Starting jobDetails '{0}'.", mapReduceJobDefinition.JobName); RunJobInPowershell( runspace, mapReduceJobDefinition, CmdletScenariosTestCaseBase.GetHttpAccessEnabledCluster(), true, expectedLogMessage); } }
public virtual void ICanCallThe_Start_HDInsightJobsCmdlet_WithoutName() { var mapReduceJobDefinition = new AzureHDInsightMapReduceJobDefinition { ClassName = "pi", JarFile = TestConstants.WabsProtocolSchemeName + "container@hostname/examples.jar" }; using (IRunspace runspace = this.GetPowerShellRunspace()) { RunJobInPowershell(runspace, mapReduceJobDefinition); } }
public void UserCanSupplyStatusDirectoryForMapReduceJob() { var mapReduceJobDefinition = new AzureHDInsightMapReduceJobDefinition { JobName = "pi estimation jobDetails", ClassName = "pi", JarFile = TestConstants.WabsProtocolSchemeName + "container@hostname/examples.jar", StatusFolder = "/myoutputfolder" }; AzureHDInsightJob startedJob = TestJobStart(mapReduceJobDefinition); Assert.AreEqual(startedJob.StatusDirectory, mapReduceJobDefinition.StatusFolder); }
public void CanStartMapReduceJob() { var mapReduceJobDefinition = new AzureHDInsightMapReduceJobDefinition { JobName = "pi estimation jobDetails", ClassName = "pi", JarFile = TestConstants.WabsProtocolSchemeName + "container@hostname/examples.jar" }; TestJobStart(mapReduceJobDefinition); }
public void CanCreateNewMapReduceJob_WithoutJobName() { var mapReduceJobDefinition = new AzureHDInsightMapReduceJobDefinition { ClassName = "pi", JarFile = TestConstants.WabsProtocolSchemeName + "container@hostname/examples.jar", StatusFolder = "/myoutputfolder" }; AzureHDInsightJob startedJob = TestJobStart(mapReduceJobDefinition); Assert.AreEqual("pi", startedJob.Name); }