/// <exception cref="System.Exception"/> private void TestReduceCommandLine(Configuration conf) { TestMapReduceChildJVM.MyMRApp app = new TestMapReduceChildJVM.MyMRApp(0, 1, true, this.GetType().FullName, true); conf.SetBoolean(MRConfig.MapreduceAppSubmissionCrossPlatform, true); Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = app.Submit(conf); app.WaitForState(job, JobState.Succeeded); app.VerifyCompleted(); long shuffleLogSize = conf.GetLong(MRJobConfig.ShuffleLogKb, 0L) * 1024L; int shuffleBackups = conf.GetInt(MRJobConfig.ShuffleLogBackups, 0); string appenderName = shuffleLogSize > 0L && shuffleBackups > 0 ? "shuffleCRLA" : "shuffleCLA"; NUnit.Framework.Assert.AreEqual("[" + MRApps.CrossPlatformify("JAVA_HOME") + "/bin/java" + " -Djava.net.preferIPv4Stack=true" + " -Dhadoop.metrics.log.level=WARN" + " -Xmx200m -Djava.io.tmpdir=" + MRApps.CrossPlatformify("PWD") + "/tmp" + " -Dlog4j.configuration=container-log4j.properties" + " -Dyarn.app.container.log.dir=<LOG_DIR>" + " -Dyarn.app.container.log.filesize=0" + " -Dhadoop.root.logger=INFO,CLA -Dhadoop.root.logfile=syslog" + " -Dyarn.app.mapreduce.shuffle.logger=INFO," + appenderName + " -Dyarn.app.mapreduce.shuffle.logfile=syslog.shuffle" + " -Dyarn.app.mapreduce.shuffle.log.filesize=" + shuffleLogSize + " -Dyarn.app.mapreduce.shuffle.log.backups=" + shuffleBackups + " org.apache.hadoop.mapred.YarnChild 127.0.0.1" + " 54321" + " attempt_0_0000_r_000000_0" + " 0" + " 1><LOG_DIR>/stdout" + " 2><LOG_DIR>/stderr ]", app.myCommandLine); NUnit.Framework.Assert.IsTrue("HADOOP_ROOT_LOGGER not set for job", app.cmdEnvironment .Contains("HADOOP_ROOT_LOGGER")); NUnit.Framework.Assert.AreEqual("INFO,console", app.cmdEnvironment["HADOOP_ROOT_LOGGER" ]); NUnit.Framework.Assert.IsTrue("HADOOP_CLIENT_OPTS not set for job", app.cmdEnvironment .Contains("HADOOP_CLIENT_OPTS")); NUnit.Framework.Assert.AreEqual(string.Empty, app.cmdEnvironment["HADOOP_CLIENT_OPTS" ]); }
public virtual void TestEnvironmentVariables() { TestMapReduceChildJVM.MyMRApp app = new TestMapReduceChildJVM.MyMRApp(1, 0, true, this.GetType().FullName, true); Configuration conf = new Configuration(); conf.Set(JobConf.MapredMapTaskEnv, "HADOOP_CLIENT_OPTS=test"); conf.SetStrings(MRJobConfig.MapLogLevel, "WARN"); conf.SetBoolean(MRConfig.MapreduceAppSubmissionCrossPlatform, false); Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = app.Submit(conf); app.WaitForState(job, JobState.Succeeded); app.VerifyCompleted(); NUnit.Framework.Assert.IsTrue("HADOOP_ROOT_LOGGER not set for job", app.cmdEnvironment .Contains("HADOOP_ROOT_LOGGER")); NUnit.Framework.Assert.AreEqual("WARN,console", app.cmdEnvironment["HADOOP_ROOT_LOGGER" ]); NUnit.Framework.Assert.IsTrue("HADOOP_CLIENT_OPTS not set for job", app.cmdEnvironment .Contains("HADOOP_CLIENT_OPTS")); NUnit.Framework.Assert.AreEqual("test", app.cmdEnvironment["HADOOP_CLIENT_OPTS"]); // Try one more. app = new TestMapReduceChildJVM.MyMRApp(1, 0, true, this.GetType().FullName, true ); conf = new Configuration(); conf.Set(JobConf.MapredMapTaskEnv, "HADOOP_ROOT_LOGGER=trace"); job = app.Submit(conf); app.WaitForState(job, JobState.Succeeded); app.VerifyCompleted(); NUnit.Framework.Assert.IsTrue("HADOOP_ROOT_LOGGER not set for job", app.cmdEnvironment .Contains("HADOOP_ROOT_LOGGER")); NUnit.Framework.Assert.AreEqual("trace", app.cmdEnvironment["HADOOP_ROOT_LOGGER"] ); }
/// <exception cref="System.Exception"/> public virtual void TestCommandLine() { TestMapReduceChildJVM.MyMRApp app = new TestMapReduceChildJVM.MyMRApp(1, 0, true, this.GetType().FullName, true); Configuration conf = new Configuration(); conf.SetBoolean(MRConfig.MapreduceAppSubmissionCrossPlatform, true); Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = app.Submit(conf); app.WaitForState(job, JobState.Succeeded); app.VerifyCompleted(); NUnit.Framework.Assert.AreEqual("[" + MRApps.CrossPlatformify("JAVA_HOME") + "/bin/java" + " -Djava.net.preferIPv4Stack=true" + " -Dhadoop.metrics.log.level=WARN" + " -Xmx200m -Djava.io.tmpdir=" + MRApps.CrossPlatformify("PWD") + "/tmp" + " -Dlog4j.configuration=container-log4j.properties" + " -Dyarn.app.container.log.dir=<LOG_DIR>" + " -Dyarn.app.container.log.filesize=0" + " -Dhadoop.root.logger=INFO,CLA -Dhadoop.root.logfile=syslog" + " org.apache.hadoop.mapred.YarnChild 127.0.0.1" + " 54321" + " attempt_0_0000_m_000000_0" + " 0" + " 1><LOG_DIR>/stdout" + " 2><LOG_DIR>/stderr ]" , app.myCommandLine); NUnit.Framework.Assert.IsTrue("HADOOP_ROOT_LOGGER not set for job", app.cmdEnvironment .Contains("HADOOP_ROOT_LOGGER")); NUnit.Framework.Assert.AreEqual("INFO,console", app.cmdEnvironment["HADOOP_ROOT_LOGGER" ]); NUnit.Framework.Assert.IsTrue("HADOOP_CLIENT_OPTS not set for job", app.cmdEnvironment .Contains("HADOOP_CLIENT_OPTS")); NUnit.Framework.Assert.AreEqual(string.Empty, app.cmdEnvironment["HADOOP_CLIENT_OPTS" ]); }
/// <exception cref="System.Exception"/> public virtual void TestCommandLineWithLog4JConifg() { TestMapReduceChildJVM.MyMRApp app = new TestMapReduceChildJVM.MyMRApp(1, 0, true, this.GetType().FullName, true); Configuration conf = new Configuration(); conf.SetBoolean(MRConfig.MapreduceAppSubmissionCrossPlatform, true); string testLogPropertieFile = "test-log4j.properties"; string testLogPropertiePath = "../" + "test-log4j.properties"; conf.Set(MRJobConfig.MapreduceJobLog4jPropertiesFile, testLogPropertiePath); Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = app.Submit(conf); app.WaitForState(job, JobState.Succeeded); app.VerifyCompleted(); NUnit.Framework.Assert.AreEqual("[" + MRApps.CrossPlatformify("JAVA_HOME") + "/bin/java" + " -Djava.net.preferIPv4Stack=true" + " -Dhadoop.metrics.log.level=WARN" + " -Xmx200m -Djava.io.tmpdir=" + MRApps.CrossPlatformify("PWD") + "/tmp" + " -Dlog4j.configuration=" + testLogPropertieFile + " -Dyarn.app.container.log.dir=<LOG_DIR>" + " -Dyarn.app.container.log.filesize=0" + " -Dhadoop.root.logger=INFO,CLA -Dhadoop.root.logfile=syslog" + " org.apache.hadoop.mapred.YarnChild 127.0.0.1" + " 54321" + " attempt_0_0000_m_000000_0" + " 0" + " 1><LOG_DIR>/stdout" + " 2><LOG_DIR>/stderr ]" , app.myCommandLine); }