Esempio n. 1
0
        public virtual void TestSetClasspathWithNoUserPrecendence()
        {
            Configuration conf = new Configuration();

            conf.SetBoolean(MRConfig.MapreduceAppSubmissionCrossPlatform, true);
            conf.SetBoolean(MRJobConfig.MapreduceJobUserClasspathFirst, false);
            IDictionary <string, string> env = new Dictionary <string, string>();

            try
            {
                MRApps.SetClasspath(env, conf);
            }
            catch (Exception)
            {
                NUnit.Framework.Assert.Fail("Got exception while setting classpath");
            }
            string env_str           = env[ApplicationConstants.Environment.Classpath.ToString()];
            string expectedClasspath = StringUtils.Join(ApplicationConstants.ClassPathSeparator
                                                        , Arrays.AsList("job.jar/job.jar", "job.jar/classes/", "job.jar/lib/*", ApplicationConstants.Environment
                                                                        .Pwd.$$() + "/*"));

            NUnit.Framework.Assert.IsTrue("MAPREDUCE_JOB_USER_CLASSPATH_FIRST false, and job.jar is not in"
                                          + " the classpath!", env_str.Contains(expectedClasspath));
            NUnit.Framework.Assert.IsFalse("MAPREDUCE_JOB_USER_CLASSPATH_FIRST false, but taking effect!"
                                           , env_str.StartsWith(expectedClasspath));
        }
Esempio n. 2
0
        /// <exception cref="System.IO.IOException"/>
        public virtual void TestSetClasspath()
        {
            Configuration conf = new Configuration();

            conf.SetBoolean(MRConfig.MapreduceAppSubmissionCrossPlatform, true);
            Job job = Job.GetInstance(conf);
            IDictionary <string, string> environment = new Dictionary <string, string>();

            MRApps.SetClasspath(environment, job.GetConfiguration());
            NUnit.Framework.Assert.IsTrue(environment[ApplicationConstants.Environment.Classpath
                                                      .ToString()].StartsWith(ApplicationConstants.Environment.Pwd.$$() + ApplicationConstants
                                                                              .ClassPathSeparator));
            string yarnAppClasspath = job.GetConfiguration().Get(YarnConfiguration.YarnApplicationClasspath
                                                                 , StringUtils.Join(",", YarnConfiguration.DefaultYarnCrossPlatformApplicationClasspath
                                                                                    ));

            if (yarnAppClasspath != null)
            {
                yarnAppClasspath = yarnAppClasspath.ReplaceAll(",\\s*", ApplicationConstants.ClassPathSeparator
                                                               ).Trim();
            }
            NUnit.Framework.Assert.IsTrue(environment[ApplicationConstants.Environment.Classpath
                                                      .ToString()].Contains(yarnAppClasspath));
            string mrAppClasspath = job.GetConfiguration().Get(MRJobConfig.MapreduceApplicationClasspath
                                                               , MRJobConfig.DefaultMapreduceCrossPlatformApplicationClasspath);

            if (mrAppClasspath != null)
            {
                mrAppClasspath = mrAppClasspath.ReplaceAll(",\\s*", ApplicationConstants.ClassPathSeparator
                                                           ).Trim();
            }
            NUnit.Framework.Assert.IsTrue(environment[ApplicationConstants.Environment.Classpath
                                                      .ToString()].Contains(mrAppClasspath));
        }
Esempio n. 3
0
        /// <exception cref="System.IO.IOException"/>
        public virtual void TestSetClasspathWithArchives()
        {
            FilePath         testTGZ = new FilePath(testWorkDir, "test.tgz");
            FileOutputStream @out    = new FileOutputStream(testTGZ);

            @out.Write(0);
            @out.Close();
            Configuration conf = new Configuration();

            conf.SetBoolean(MRConfig.MapreduceAppSubmissionCrossPlatform, true);
            Job job = Job.GetInstance(conf);

            conf = job.GetConfiguration();
            string testTGZQualifiedPath = FileSystem.GetLocal(conf).MakeQualified(new Path(testTGZ
                                                                                           .GetAbsolutePath())).ToString();

            conf.Set(MRJobConfig.ClasspathArchives, testTGZQualifiedPath);
            conf.Set(MRJobConfig.CacheArchives, testTGZQualifiedPath + "#testTGZ");
            IDictionary <string, string> environment = new Dictionary <string, string>();

            MRApps.SetClasspath(environment, conf);
            NUnit.Framework.Assert.IsTrue(environment[ApplicationConstants.Environment.Classpath
                                                      .ToString()].StartsWith(ApplicationConstants.Environment.Pwd.$$() + ApplicationConstants
                                                                              .ClassPathSeparator));
            NUnit.Framework.Assert.IsTrue(environment[ApplicationConstants.Environment.HadoopClasspath
                                                      .ToString()].StartsWith(ApplicationConstants.Environment.Pwd.$$() + ApplicationConstants
                                                                              .ClassPathSeparator));
            string confClasspath = job.GetConfiguration().Get(YarnConfiguration.YarnApplicationClasspath
                                                              , StringUtils.Join(",", YarnConfiguration.DefaultYarnCrossPlatformApplicationClasspath
                                                                                 ));

            if (confClasspath != null)
            {
                confClasspath = confClasspath.ReplaceAll(",\\s*", ApplicationConstants.ClassPathSeparator
                                                         ).Trim();
            }
            Log.Info("CLASSPATH: " + environment[ApplicationConstants.Environment.Classpath.ToString
                                                     ()]);
            Log.Info("confClasspath: " + confClasspath);
            NUnit.Framework.Assert.IsTrue(environment[ApplicationConstants.Environment.Classpath
                                                      .ToString()].Contains(confClasspath));
            Log.Info("HADOOP_CLASSPATH: " + environment[ApplicationConstants.Environment.HadoopClasspath
                                                        .ToString()]);
            NUnit.Framework.Assert.IsTrue(environment[ApplicationConstants.Environment.Classpath
                                                      .ToString()].Contains("testTGZ"));
            NUnit.Framework.Assert.IsTrue(environment[ApplicationConstants.Environment.HadoopClasspath
                                                      .ToString()].Contains("testTGZ"));
        }
Esempio n. 4
0
        /// <exception cref="System.IO.IOException"/>
        public virtual void TestSetClasspathWithFramework()
        {
            string        FrameworkName = "some-framework-name";
            string        FrameworkPath = "some-framework-path#" + FrameworkName;
            Configuration conf          = new Configuration();

            conf.SetBoolean(MRConfig.MapreduceAppSubmissionCrossPlatform, true);
            conf.Set(MRJobConfig.MapreduceApplicationFrameworkPath, FrameworkPath);
            IDictionary <string, string> env = new Dictionary <string, string>();

            try
            {
                MRApps.SetClasspath(env, conf);
                NUnit.Framework.Assert.Fail("Failed to catch framework path set without classpath change"
                                            );
            }
            catch (ArgumentException e)
            {
                NUnit.Framework.Assert.IsTrue("Unexpected IllegalArgumentException", e.Message.Contains
                                                  ("Could not locate MapReduce framework name '" + FrameworkName + "'"));
            }
            env.Clear();
            string FrameworkClasspath = FrameworkName + "/*.jar";

            conf.Set(MRJobConfig.MapreduceApplicationClasspath, FrameworkClasspath);
            MRApps.SetClasspath(env, conf);
            string stdClasspath = StringUtils.Join(ApplicationConstants.ClassPathSeparator, Arrays
                                                   .AsList("job.jar/job.jar", "job.jar/classes/", "job.jar/lib/*", ApplicationConstants.Environment
                                                           .Pwd.$$() + "/*"));
            string expectedClasspath = StringUtils.Join(ApplicationConstants.ClassPathSeparator
                                                        , Arrays.AsList(ApplicationConstants.Environment.Pwd.$$(), FrameworkClasspath, stdClasspath
                                                                        ));

            NUnit.Framework.Assert.AreEqual("Incorrect classpath with framework and no user precedence"
                                            , expectedClasspath, env[ApplicationConstants.Environment.Classpath.ToString()]);
            env.Clear();
            conf.SetBoolean(MRJobConfig.MapreduceJobUserClasspathFirst, true);
            MRApps.SetClasspath(env, conf);
            expectedClasspath = StringUtils.Join(ApplicationConstants.ClassPathSeparator, Arrays
                                                 .AsList(ApplicationConstants.Environment.Pwd.$$(), stdClasspath, FrameworkClasspath
                                                         ));
            NUnit.Framework.Assert.AreEqual("Incorrect classpath with framework and user precedence"
                                            , expectedClasspath, env[ApplicationConstants.Environment.Classpath.ToString()]);
        }
Esempio n. 5
0
        /// <exception cref="System.IO.IOException"/>
        public virtual void TestSetClasspathWithJobClassloader()
        {
            Configuration conf = new Configuration();

            conf.SetBoolean(MRConfig.MapreduceAppSubmissionCrossPlatform, true);
            conf.SetBoolean(MRJobConfig.MapreduceJobClassloader, true);
            IDictionary <string, string> env = new Dictionary <string, string>();

            MRApps.SetClasspath(env, conf);
            string cp    = env[ApplicationConstants.Environment.Classpath.ToString()];
            string appCp = env[ApplicationConstants.Environment.AppClasspath.ToString()];

            NUnit.Framework.Assert.IsFalse("MAPREDUCE_JOB_CLASSLOADER true, but job.jar is in the"
                                           + " classpath!", cp.Contains("jar" + ApplicationConstants.ClassPathSeparator +
                                                                        "job"));
            NUnit.Framework.Assert.IsFalse("MAPREDUCE_JOB_CLASSLOADER true, but PWD is in the classpath!"
                                           , cp.Contains("PWD"));
            string expectedAppClasspath = StringUtils.Join(ApplicationConstants.ClassPathSeparator
                                                           , Arrays.AsList(ApplicationConstants.Environment.Pwd.$$(), "job.jar/job.jar", "job.jar/classes/"
                                                                           , "job.jar/lib/*", ApplicationConstants.Environment.Pwd.$$() + "/*"));

            NUnit.Framework.Assert.AreEqual("MAPREDUCE_JOB_CLASSLOADER true, but job.jar is not in the app"
                                            + " classpath!", expectedAppClasspath, appCp);
        }