Esempio n. 1
0
        /// <exception cref="System.Exception"/>
        private void TestReduceCommandLine(Configuration conf)
        {
            TestMapReduceChildJVM.MyMRApp app = new TestMapReduceChildJVM.MyMRApp(0, 1, true,
                                                                                  this.GetType().FullName, true);
            conf.SetBoolean(MRConfig.MapreduceAppSubmissionCrossPlatform, true);
            Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = app.Submit(conf);
            app.WaitForState(job, JobState.Succeeded);
            app.VerifyCompleted();
            long   shuffleLogSize = conf.GetLong(MRJobConfig.ShuffleLogKb, 0L) * 1024L;
            int    shuffleBackups = conf.GetInt(MRJobConfig.ShuffleLogBackups, 0);
            string appenderName   = shuffleLogSize > 0L && shuffleBackups > 0 ? "shuffleCRLA" :
                                    "shuffleCLA";

            NUnit.Framework.Assert.AreEqual("[" + MRApps.CrossPlatformify("JAVA_HOME") + "/bin/java"
                                            + " -Djava.net.preferIPv4Stack=true" + " -Dhadoop.metrics.log.level=WARN" + "  -Xmx200m -Djava.io.tmpdir="
                                            + MRApps.CrossPlatformify("PWD") + "/tmp" + " -Dlog4j.configuration=container-log4j.properties"
                                            + " -Dyarn.app.container.log.dir=<LOG_DIR>" + " -Dyarn.app.container.log.filesize=0"
                                            + " -Dhadoop.root.logger=INFO,CLA -Dhadoop.root.logfile=syslog" + " -Dyarn.app.mapreduce.shuffle.logger=INFO,"
                                            + appenderName + " -Dyarn.app.mapreduce.shuffle.logfile=syslog.shuffle" + " -Dyarn.app.mapreduce.shuffle.log.filesize="
                                            + shuffleLogSize + " -Dyarn.app.mapreduce.shuffle.log.backups=" + shuffleBackups
                                            + " org.apache.hadoop.mapred.YarnChild 127.0.0.1" + " 54321" + " attempt_0_0000_r_000000_0"
                                            + " 0" + " 1><LOG_DIR>/stdout" + " 2><LOG_DIR>/stderr ]", app.myCommandLine);
            NUnit.Framework.Assert.IsTrue("HADOOP_ROOT_LOGGER not set for job", app.cmdEnvironment
                                          .Contains("HADOOP_ROOT_LOGGER"));
            NUnit.Framework.Assert.AreEqual("INFO,console", app.cmdEnvironment["HADOOP_ROOT_LOGGER"
                                            ]);
            NUnit.Framework.Assert.IsTrue("HADOOP_CLIENT_OPTS not set for job", app.cmdEnvironment
                                          .Contains("HADOOP_CLIENT_OPTS"));
            NUnit.Framework.Assert.AreEqual(string.Empty, app.cmdEnvironment["HADOOP_CLIENT_OPTS"
                                            ]);
        }
Esempio n. 2
0
        public virtual void TestEnvironmentVariables()
        {
            TestMapReduceChildJVM.MyMRApp app = new TestMapReduceChildJVM.MyMRApp(1, 0, true,
                                                                                  this.GetType().FullName, true);
            Configuration conf = new Configuration();

            conf.Set(JobConf.MapredMapTaskEnv, "HADOOP_CLIENT_OPTS=test");
            conf.SetStrings(MRJobConfig.MapLogLevel, "WARN");
            conf.SetBoolean(MRConfig.MapreduceAppSubmissionCrossPlatform, false);
            Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = app.Submit(conf);
            app.WaitForState(job, JobState.Succeeded);
            app.VerifyCompleted();
            NUnit.Framework.Assert.IsTrue("HADOOP_ROOT_LOGGER not set for job", app.cmdEnvironment
                                          .Contains("HADOOP_ROOT_LOGGER"));
            NUnit.Framework.Assert.AreEqual("WARN,console", app.cmdEnvironment["HADOOP_ROOT_LOGGER"
                                            ]);
            NUnit.Framework.Assert.IsTrue("HADOOP_CLIENT_OPTS not set for job", app.cmdEnvironment
                                          .Contains("HADOOP_CLIENT_OPTS"));
            NUnit.Framework.Assert.AreEqual("test", app.cmdEnvironment["HADOOP_CLIENT_OPTS"]);
            // Try one more.
            app = new TestMapReduceChildJVM.MyMRApp(1, 0, true, this.GetType().FullName, true
                                                    );
            conf = new Configuration();
            conf.Set(JobConf.MapredMapTaskEnv, "HADOOP_ROOT_LOGGER=trace");
            job = app.Submit(conf);
            app.WaitForState(job, JobState.Succeeded);
            app.VerifyCompleted();
            NUnit.Framework.Assert.IsTrue("HADOOP_ROOT_LOGGER not set for job", app.cmdEnvironment
                                          .Contains("HADOOP_ROOT_LOGGER"));
            NUnit.Framework.Assert.AreEqual("trace", app.cmdEnvironment["HADOOP_ROOT_LOGGER"]
                                            );
        }
Esempio n. 3
0
        /// <exception cref="System.Exception"/>
        public virtual void TestCommandLine()
        {
            TestMapReduceChildJVM.MyMRApp app = new TestMapReduceChildJVM.MyMRApp(1, 0, true,
                                                                                  this.GetType().FullName, true);
            Configuration conf = new Configuration();

            conf.SetBoolean(MRConfig.MapreduceAppSubmissionCrossPlatform, true);
            Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = app.Submit(conf);
            app.WaitForState(job, JobState.Succeeded);
            app.VerifyCompleted();
            NUnit.Framework.Assert.AreEqual("[" + MRApps.CrossPlatformify("JAVA_HOME") + "/bin/java"
                                            + " -Djava.net.preferIPv4Stack=true" + " -Dhadoop.metrics.log.level=WARN" + "  -Xmx200m -Djava.io.tmpdir="
                                            + MRApps.CrossPlatformify("PWD") + "/tmp" + " -Dlog4j.configuration=container-log4j.properties"
                                            + " -Dyarn.app.container.log.dir=<LOG_DIR>" + " -Dyarn.app.container.log.filesize=0"
                                            + " -Dhadoop.root.logger=INFO,CLA -Dhadoop.root.logfile=syslog" + " org.apache.hadoop.mapred.YarnChild 127.0.0.1"
                                            + " 54321" + " attempt_0_0000_m_000000_0" + " 0" + " 1><LOG_DIR>/stdout" + " 2><LOG_DIR>/stderr ]"
                                            , app.myCommandLine);
            NUnit.Framework.Assert.IsTrue("HADOOP_ROOT_LOGGER not set for job", app.cmdEnvironment
                                          .Contains("HADOOP_ROOT_LOGGER"));
            NUnit.Framework.Assert.AreEqual("INFO,console", app.cmdEnvironment["HADOOP_ROOT_LOGGER"
                                            ]);
            NUnit.Framework.Assert.IsTrue("HADOOP_CLIENT_OPTS not set for job", app.cmdEnvironment
                                          .Contains("HADOOP_CLIENT_OPTS"));
            NUnit.Framework.Assert.AreEqual(string.Empty, app.cmdEnvironment["HADOOP_CLIENT_OPTS"
                                            ]);
        }
Esempio n. 4
0
        /// <exception cref="System.Exception"/>
        public virtual void TestCommandLineWithLog4JConifg()
        {
            TestMapReduceChildJVM.MyMRApp app = new TestMapReduceChildJVM.MyMRApp(1, 0, true,
                                                                                  this.GetType().FullName, true);
            Configuration conf = new Configuration();

            conf.SetBoolean(MRConfig.MapreduceAppSubmissionCrossPlatform, true);
            string testLogPropertieFile = "test-log4j.properties";
            string testLogPropertiePath = "../" + "test-log4j.properties";

            conf.Set(MRJobConfig.MapreduceJobLog4jPropertiesFile, testLogPropertiePath);
            Org.Apache.Hadoop.Mapreduce.V2.App.Job.Job job = app.Submit(conf);
            app.WaitForState(job, JobState.Succeeded);
            app.VerifyCompleted();
            NUnit.Framework.Assert.AreEqual("[" + MRApps.CrossPlatformify("JAVA_HOME") + "/bin/java"
                                            + " -Djava.net.preferIPv4Stack=true" + " -Dhadoop.metrics.log.level=WARN" + "  -Xmx200m -Djava.io.tmpdir="
                                            + MRApps.CrossPlatformify("PWD") + "/tmp" + " -Dlog4j.configuration=" + testLogPropertieFile
                                            + " -Dyarn.app.container.log.dir=<LOG_DIR>" + " -Dyarn.app.container.log.filesize=0"
                                            + " -Dhadoop.root.logger=INFO,CLA -Dhadoop.root.logfile=syslog" + " org.apache.hadoop.mapred.YarnChild 127.0.0.1"
                                            + " 54321" + " attempt_0_0000_m_000000_0" + " 0" + " 1><LOG_DIR>/stdout" + " 2><LOG_DIR>/stderr ]"
                                            , app.myCommandLine);
        }