/// <exception cref="System.IO.IOException"/>
public virtual void TestSetClasspath()
{
Configuration conf = new Configuration();
conf.SetBoolean(MRConfig.MapreduceAppSubmissionCrossPlatform, true);
Job job = Job.GetInstance(conf);
IDictionary <string, string> environment = new Dictionary <string, string>();
MRApps.SetClasspath(environment, job.GetConfiguration());
NUnit.Framework.Assert.IsTrue(environment[ApplicationConstants.Environment.Classpath
.ToString()].StartsWith(ApplicationConstants.Environment.Pwd.$$() + ApplicationConstants
.ClassPathSeparator));
string yarnAppClasspath = job.GetConfiguration().Get(YarnConfiguration.YarnApplicationClasspath
, StringUtils.Join(",", YarnConfiguration.DefaultYarnCrossPlatformApplicationClasspath
));
if (yarnAppClasspath != null)
{
yarnAppClasspath = yarnAppClasspath.ReplaceAll(",\\s*", ApplicationConstants.ClassPathSeparator
).Trim();
}
NUnit.Framework.Assert.IsTrue(environment[ApplicationConstants.Environment.Classpath
.ToString()].Contains(yarnAppClasspath));
string mrAppClasspath = job.GetConfiguration().Get(MRJobConfig.MapreduceApplicationClasspath
, MRJobConfig.DefaultMapreduceCrossPlatformApplicationClasspath);
if (mrAppClasspath != null)
{
mrAppClasspath = mrAppClasspath.ReplaceAll(",\\s*", ApplicationConstants.ClassPathSeparator
).Trim();
}
NUnit.Framework.Assert.IsTrue(environment[ApplicationConstants.Environment.Classpath
.ToString()].Contains(mrAppClasspath));
}
public static string GetJobFile(Configuration conf, string user, JobID jobId)
{
Path jobFile = new Path(MRApps.GetStagingAreaDir(conf, user), jobId.ToString() +
Path.Separator + MRJobConfig.JobConfFile);
return(jobFile.ToString());
}
public virtual void TestSetClasspathWithNoUserPrecendence()
{
Configuration conf = new Configuration();
conf.SetBoolean(MRConfig.MapreduceAppSubmissionCrossPlatform, true);
conf.SetBoolean(MRJobConfig.MapreduceJobUserClasspathFirst, false);
IDictionary <string, string> env = new Dictionary <string, string>();
try
{
MRApps.SetClasspath(env, conf);
}
catch (Exception)
{
NUnit.Framework.Assert.Fail("Got exception while setting classpath");
}
string env_str = env[ApplicationConstants.Environment.Classpath.ToString()];
string expectedClasspath = StringUtils.Join(ApplicationConstants.ClassPathSeparator
, Arrays.AsList("job.jar/job.jar", "job.jar/classes/", "job.jar/lib/*", ApplicationConstants.Environment
.Pwd.$$() + "/*"));
NUnit.Framework.Assert.IsTrue("MAPREDUCE_JOB_USER_CLASSPATH_FIRST false, and job.jar is not in"
+ " the classpath!", env_str.Contains(expectedClasspath));
NUnit.Framework.Assert.IsFalse("MAPREDUCE_JOB_USER_CLASSPATH_FIRST false, but taking effect!"
, env_str.StartsWith(expectedClasspath));
}
// TODO: Remove duplicates.
/// <exception cref="System.IO.IOException"/>
public static void SetClasspath(IDictionary <string, string> environment, Configuration
conf)
{
bool userClassesTakesPrecedence = conf.GetBoolean(MRJobConfig.MapreduceJobUserClasspathFirst
, false);
string classpathEnvVar = conf.GetBoolean(MRJobConfig.MapreduceJobClassloader, false
) ? ApplicationConstants.Environment.AppClasspath.ToString() : ApplicationConstants.Environment
.Classpath.ToString();
string hadoopClasspathEnvVar = ApplicationConstants.Environment.HadoopClasspath.ToString
();
MRApps.AddToEnvironment(environment, classpathEnvVar, CrossPlatformifyMREnv(conf,
ApplicationConstants.Environment.Pwd), conf);
MRApps.AddToEnvironment(environment, hadoopClasspathEnvVar, CrossPlatformifyMREnv
(conf, ApplicationConstants.Environment.Pwd), conf);
if (!userClassesTakesPrecedence)
{
MRApps.SetMRFrameworkClasspath(environment, conf);
}
AddClasspathToEnv(environment, classpathEnvVar, conf);
AddClasspathToEnv(environment, hadoopClasspathEnvVar, conf);
if (userClassesTakesPrecedence)
{
MRApps.SetMRFrameworkClasspath(environment, conf);
}
}
/// <exception cref="System.Exception"/>
public virtual void TestSetupDistributedCacheConflictsFiles()
{
Configuration conf = new Configuration();
conf.SetClass("fs.mockfs.impl", typeof(TestMRApps.MockFileSystem), typeof(FileSystem
));
URI mockUri = URI.Create("mockfs://mock/");
FileSystem mockFs = ((FilterFileSystem)FileSystem.Get(mockUri, conf)).GetRawFileSystem
();
URI file = new URI("mockfs://mock/tmp/something.zip#something");
Path filePath = new Path(file);
URI file2 = new URI("mockfs://mock/tmp/something.txt#something");
Path file2Path = new Path(file2);
Org.Mockito.Mockito.When(mockFs.ResolvePath(filePath)).ThenReturn(filePath);
Org.Mockito.Mockito.When(mockFs.ResolvePath(file2Path)).ThenReturn(file2Path);
DistributedCache.AddCacheFile(file, conf);
DistributedCache.AddCacheFile(file2, conf);
conf.Set(MRJobConfig.CacheFileTimestamps, "10,11");
conf.Set(MRJobConfig.CacheFilesSizes, "10,11");
conf.Set(MRJobConfig.CacheFileVisibilities, "true,true");
IDictionary <string, LocalResource> localResources = new Dictionary <string, LocalResource
>();
MRApps.SetupDistributedCache(conf, localResources);
NUnit.Framework.Assert.AreEqual(1, localResources.Count);
LocalResource lr = localResources["something"];
//First one wins
NUnit.Framework.Assert.IsNotNull(lr);
NUnit.Framework.Assert.AreEqual(10l, lr.GetSize());
NUnit.Framework.Assert.AreEqual(10l, lr.GetTimestamp());
NUnit.Framework.Assert.AreEqual(LocalResourceType.File, lr.GetType());
}
public virtual void TestJobIDtoString()
{
JobId jid = RecordFactoryProvider.GetRecordFactory(null).NewRecordInstance <JobId>
();
jid.SetAppId(ApplicationId.NewInstance(0, 0));
NUnit.Framework.Assert.AreEqual("job_0_0000", MRApps.ToString(jid));
}
public static Path GetStartJobCommitFile(Configuration conf, string user, JobId jobId
)
{
Path startCommitFile = new Path(MRApps.GetStagingAreaDir(conf, user), jobId.ToString
() + Path.Separator + "COMMIT_STARTED");
return(startCommitFile);
}
public static Path GetEndJobCommitFailureFile(Configuration conf, string user, JobId
jobId)
{
Path endCommitFile = new Path(MRApps.GetStagingAreaDir(conf, user), jobId.ToString
() + Path.Separator + "COMMIT_FAIL");
return(endCommitFile);
}
public virtual void TestToJobID()
{
JobId jid = MRApps.ToJobID("job_1_1");
NUnit.Framework.Assert.AreEqual(1, jid.GetAppId().GetClusterTimestamp());
NUnit.Framework.Assert.AreEqual(1, jid.GetAppId().GetId());
NUnit.Framework.Assert.AreEqual(1, jid.GetId());
}
/// <exception cref="System.IO.IOException"/>
public virtual void TestSetupDistributedCacheEmpty()
{
Configuration conf = new Configuration();
IDictionary <string, LocalResource> localResources = new Dictionary <string, LocalResource
>();
MRApps.SetupDistributedCache(conf, localResources);
NUnit.Framework.Assert.IsTrue("Empty Config did not produce an empty list of resources"
, localResources.IsEmpty());
}
public virtual void TestToTaskAttemptID()
{
TaskAttemptId taid = MRApps.ToTaskAttemptID("attempt_0_1_m_2_3");
NUnit.Framework.Assert.AreEqual(0, taid.GetTaskId().GetJobId().GetAppId().GetClusterTimestamp
());
NUnit.Framework.Assert.AreEqual(1, taid.GetTaskId().GetJobId().GetAppId().GetId()
);
NUnit.Framework.Assert.AreEqual(1, taid.GetTaskId().GetJobId().GetId());
NUnit.Framework.Assert.AreEqual(2, taid.GetTaskId().GetId());
NUnit.Framework.Assert.AreEqual(3, taid.GetId());
}
public virtual void TestToTaskID()
{
TaskId tid = MRApps.ToTaskID("task_1_2_r_3");
NUnit.Framework.Assert.AreEqual(1, tid.GetJobId().GetAppId().GetClusterTimestamp(
));
NUnit.Framework.Assert.AreEqual(2, tid.GetJobId().GetAppId().GetId());
NUnit.Framework.Assert.AreEqual(2, tid.GetJobId().GetId());
NUnit.Framework.Assert.AreEqual(TaskType.Reduce, tid.GetTaskType());
NUnit.Framework.Assert.AreEqual(3, tid.GetId());
tid = MRApps.ToTaskID("task_1_2_m_3");
NUnit.Framework.Assert.AreEqual(TaskType.Map, tid.GetTaskType());
}
public virtual void TestGetJobFileWithUser()
{
Configuration conf = new Configuration();
conf.Set(MRJobConfig.MrAmStagingDir, "/my/path/to/staging");
string jobFile = MRApps.GetJobFile(conf, "dummy-user", new JobID("dummy-job", 12345
));
NUnit.Framework.Assert.IsNotNull("getJobFile results in null.", jobFile);
NUnit.Framework.Assert.AreEqual("jobFile with specified user is not as expected."
, "/my/path/to/staging/dummy-user/.staging/job_dummy-job_12345/job.xml", jobFile
);
}
//TODO_get.set
public virtual void TestTaskAttemptIDtoString()
{
TaskAttemptId taid = RecordFactoryProvider.GetRecordFactory(null).NewRecordInstance
<TaskAttemptId>();
taid.SetTaskId(RecordFactoryProvider.GetRecordFactory(null).NewRecordInstance <TaskId
>());
taid.GetTaskId().SetTaskType(TaskType.Map);
taid.GetTaskId().SetJobId(RecordFactoryProvider.GetRecordFactory(null).NewRecordInstance
<JobId>());
taid.GetTaskId().GetJobId().SetAppId(ApplicationId.NewInstance(0, 0));
NUnit.Framework.Assert.AreEqual("attempt_0_0000_m_000000_0", MRApps.ToString(taid
));
}
/// <exception cref="System.IO.IOException"/>
public virtual void TestSetClasspathWithArchives()
{
FilePath testTGZ = new FilePath(testWorkDir, "test.tgz");
FileOutputStream @out = new FileOutputStream(testTGZ);
@out.Write(0);
@out.Close();
Configuration conf = new Configuration();
conf.SetBoolean(MRConfig.MapreduceAppSubmissionCrossPlatform, true);
Job job = Job.GetInstance(conf);
conf = job.GetConfiguration();
string testTGZQualifiedPath = FileSystem.GetLocal(conf).MakeQualified(new Path(testTGZ
.GetAbsolutePath())).ToString();
conf.Set(MRJobConfig.ClasspathArchives, testTGZQualifiedPath);
conf.Set(MRJobConfig.CacheArchives, testTGZQualifiedPath + "#testTGZ");
IDictionary <string, string> environment = new Dictionary <string, string>();
MRApps.SetClasspath(environment, conf);
NUnit.Framework.Assert.IsTrue(environment[ApplicationConstants.Environment.Classpath
.ToString()].StartsWith(ApplicationConstants.Environment.Pwd.$$() + ApplicationConstants
.ClassPathSeparator));
NUnit.Framework.Assert.IsTrue(environment[ApplicationConstants.Environment.HadoopClasspath
.ToString()].StartsWith(ApplicationConstants.Environment.Pwd.$$() + ApplicationConstants
.ClassPathSeparator));
string confClasspath = job.GetConfiguration().Get(YarnConfiguration.YarnApplicationClasspath
, StringUtils.Join(",", YarnConfiguration.DefaultYarnCrossPlatformApplicationClasspath
));
if (confClasspath != null)
{
confClasspath = confClasspath.ReplaceAll(",\\s*", ApplicationConstants.ClassPathSeparator
).Trim();
}
Log.Info("CLASSPATH: " + environment[ApplicationConstants.Environment.Classpath.ToString
()]);
Log.Info("confClasspath: " + confClasspath);
NUnit.Framework.Assert.IsTrue(environment[ApplicationConstants.Environment.Classpath
.ToString()].Contains(confClasspath));
Log.Info("HADOOP_CLASSPATH: " + environment[ApplicationConstants.Environment.HadoopClasspath
.ToString()]);
NUnit.Framework.Assert.IsTrue(environment[ApplicationConstants.Environment.Classpath
.ToString()].Contains("testTGZ"));
NUnit.Framework.Assert.IsTrue(environment[ApplicationConstants.Environment.HadoopClasspath
.ToString()].Contains("testTGZ"));
}
/// <exception cref="System.IO.IOException"/>
public static void AddClasspathToEnv(IDictionary <string, string> environment, string
classpathEnvVar, Configuration conf)
{
MRApps.AddToEnvironment(environment, classpathEnvVar, MRJobConfig.JobJar + Path.Separator
+ MRJobConfig.JobJar, conf);
MRApps.AddToEnvironment(environment, classpathEnvVar, MRJobConfig.JobJar + Path.Separator
+ "classes" + Path.Separator, conf);
MRApps.AddToEnvironment(environment, classpathEnvVar, MRJobConfig.JobJar + Path.Separator
+ "lib" + Path.Separator + "*", conf);
MRApps.AddToEnvironment(environment, classpathEnvVar, CrossPlatformifyMREnv(conf,
ApplicationConstants.Environment.Pwd) + Path.Separator + "*", conf);
// a * in the classpath will only find a .jar, so we need to filter out
// all .jars and add everything else
AddToClasspathIfNotJar(DistributedCache.GetFileClassPaths(conf), DistributedCache
.GetCacheFiles(conf), conf, environment, classpathEnvVar);
AddToClasspathIfNotJar(DistributedCache.GetArchiveClassPaths(conf), DistributedCache
.GetCacheArchives(conf), conf, environment, classpathEnvVar);
}
/// <exception cref="System.IO.IOException"/>
private static void SetMRFrameworkClasspath(IDictionary <string, string> environment
, Configuration conf)
{
// Propagate the system classpath when using the mini cluster
if (conf.GetBoolean(YarnConfiguration.IsMiniYarnCluster, false))
{
MRApps.AddToEnvironment(environment, ApplicationConstants.Environment.Classpath.ToString
(), Runtime.GetProperty("java.class.path"), conf);
}
bool crossPlatform = conf.GetBoolean(MRConfig.MapreduceAppSubmissionCrossPlatform
, MRConfig.DefaultMapreduceAppSubmissionCrossPlatform);
// if the framework is specified then only use the MR classpath
string frameworkName = GetMRFrameworkName(conf);
if (frameworkName == null)
{
// Add standard Hadoop classes
foreach (string c in conf.GetStrings(YarnConfiguration.YarnApplicationClasspath,
crossPlatform ? YarnConfiguration.DefaultYarnCrossPlatformApplicationClasspath :
YarnConfiguration.DefaultYarnApplicationClasspath))
{
MRApps.AddToEnvironment(environment, ApplicationConstants.Environment.Classpath.ToString
(), c.Trim(), conf);
}
}
bool foundFrameworkInClasspath = (frameworkName == null);
foreach (string c_1 in conf.GetStrings(MRJobConfig.MapreduceApplicationClasspath,
crossPlatform ? StringUtils.GetStrings(MRJobConfig.DefaultMapreduceCrossPlatformApplicationClasspath
) : StringUtils.GetStrings(MRJobConfig.DefaultMapreduceApplicationClasspath)))
{
MRApps.AddToEnvironment(environment, ApplicationConstants.Environment.Classpath.ToString
(), c_1.Trim(), conf);
if (!foundFrameworkInClasspath)
{
foundFrameworkInClasspath = c_1.Contains(frameworkName);
}
}
if (!foundFrameworkInClasspath)
{
throw new ArgumentException("Could not locate MapReduce framework name '" + frameworkName
+ "' in " + MRJobConfig.MapreduceApplicationClasspath);
}
}
/// <exception cref="System.IO.IOException"/>
public virtual void TestSetClasspathWithFramework()
{
string FrameworkName = "some-framework-name";
string FrameworkPath = "some-framework-path#" + FrameworkName;
Configuration conf = new Configuration();
conf.SetBoolean(MRConfig.MapreduceAppSubmissionCrossPlatform, true);
conf.Set(MRJobConfig.MapreduceApplicationFrameworkPath, FrameworkPath);
IDictionary <string, string> env = new Dictionary <string, string>();
try
{
MRApps.SetClasspath(env, conf);
NUnit.Framework.Assert.Fail("Failed to catch framework path set without classpath change"
);
}
catch (ArgumentException e)
{
NUnit.Framework.Assert.IsTrue("Unexpected IllegalArgumentException", e.Message.Contains
("Could not locate MapReduce framework name '" + FrameworkName + "'"));
}
env.Clear();
string FrameworkClasspath = FrameworkName + "/*.jar";
conf.Set(MRJobConfig.MapreduceApplicationClasspath, FrameworkClasspath);
MRApps.SetClasspath(env, conf);
string stdClasspath = StringUtils.Join(ApplicationConstants.ClassPathSeparator, Arrays
.AsList("job.jar/job.jar", "job.jar/classes/", "job.jar/lib/*", ApplicationConstants.Environment
.Pwd.$$() + "/*"));
string expectedClasspath = StringUtils.Join(ApplicationConstants.ClassPathSeparator
, Arrays.AsList(ApplicationConstants.Environment.Pwd.$$(), FrameworkClasspath, stdClasspath
));
NUnit.Framework.Assert.AreEqual("Incorrect classpath with framework and no user precedence"
, expectedClasspath, env[ApplicationConstants.Environment.Classpath.ToString()]);
env.Clear();
conf.SetBoolean(MRJobConfig.MapreduceJobUserClasspathFirst, true);
MRApps.SetClasspath(env, conf);
expectedClasspath = StringUtils.Join(ApplicationConstants.ClassPathSeparator, Arrays
.AsList(ApplicationConstants.Environment.Pwd.$$(), stdClasspath, FrameworkClasspath
));
NUnit.Framework.Assert.AreEqual("Incorrect classpath with framework and user precedence"
, expectedClasspath, env[ApplicationConstants.Environment.Classpath.ToString()]);
}
//TODO_get.set
public virtual void TestTaskIDtoString()
{
TaskId tid = RecordFactoryProvider.GetRecordFactory(null).NewRecordInstance <TaskId
>();
tid.SetJobId(RecordFactoryProvider.GetRecordFactory(null).NewRecordInstance <JobId
>());
tid.GetJobId().SetAppId(ApplicationId.NewInstance(0, 0));
tid.SetTaskType(TaskType.Map);
TaskType type = tid.GetTaskType();
System.Console.Error.WriteLine(type);
type = TaskType.Reduce;
System.Console.Error.WriteLine(type);
System.Console.Error.WriteLine(tid.GetTaskType());
NUnit.Framework.Assert.AreEqual("task_0_0000_m_000000", MRApps.ToString(tid));
tid.SetTaskType(TaskType.Reduce);
NUnit.Framework.Assert.AreEqual("task_0_0000_r_000000", MRApps.ToString(tid));
}
/// <summary>Add the paths to the classpath if they are not jars</summary>
/// <param name="paths">the paths to add to the classpath</param>
/// <param name="withLinks">the corresponding paths that may have a link name in them
/// </param>
/// <param name="conf">used to resolve the paths</param>
/// <param name="environment">the environment to update CLASSPATH in</param>
/// <exception cref="System.IO.IOException">if there is an error resolving any of the paths.
/// </exception>
private static void AddToClasspathIfNotJar(Path[] paths, URI[] withLinks, Configuration
conf, IDictionary <string, string> environment, string classpathEnvVar)
{
if (paths != null)
{
Dictionary <Path, string> linkLookup = new Dictionary <Path, string>();
if (withLinks != null)
{
foreach (URI u in withLinks)
{
Path p = new Path(u);
FileSystem remoteFS = p.GetFileSystem(conf);
p = remoteFS.ResolvePath(p.MakeQualified(remoteFS.GetUri(), remoteFS.GetWorkingDirectory
()));
string name = (null == u.GetFragment()) ? p.GetName() : u.GetFragment();
if (!StringUtils.ToLowerCase(name).EndsWith(".jar"))
{
linkLookup[p] = name;
}
}
}
foreach (Path p_1 in paths)
{
FileSystem remoteFS = p_1.GetFileSystem(conf);
p_1 = remoteFS.ResolvePath(p_1.MakeQualified(remoteFS.GetUri(), remoteFS.GetWorkingDirectory
()));
string name = linkLookup[p_1];
if (name == null)
{
name = p_1.GetName();
}
if (!StringUtils.ToLowerCase(name).EndsWith(".jar"))
{
MRApps.AddToEnvironment(environment, classpathEnvVar, CrossPlatformifyMREnv(conf,
ApplicationConstants.Environment.Pwd) + Path.Separator + name, conf);
}
}
}
}
/// <exception cref="System.IO.IOException"/>
public virtual void TestSetClasspathWithJobClassloader()
{
Configuration conf = new Configuration();
conf.SetBoolean(MRConfig.MapreduceAppSubmissionCrossPlatform, true);
conf.SetBoolean(MRJobConfig.MapreduceJobClassloader, true);
IDictionary <string, string> env = new Dictionary <string, string>();
MRApps.SetClasspath(env, conf);
string cp = env[ApplicationConstants.Environment.Classpath.ToString()];
string appCp = env[ApplicationConstants.Environment.AppClasspath.ToString()];
NUnit.Framework.Assert.IsFalse("MAPREDUCE_JOB_CLASSLOADER true, but job.jar is in the"
+ " classpath!", cp.Contains("jar" + ApplicationConstants.ClassPathSeparator +
"job"));
NUnit.Framework.Assert.IsFalse("MAPREDUCE_JOB_CLASSLOADER true, but PWD is in the classpath!"
, cp.Contains("PWD"));
string expectedAppClasspath = StringUtils.Join(ApplicationConstants.ClassPathSeparator
, Arrays.AsList(ApplicationConstants.Environment.Pwd.$$(), "job.jar/job.jar", "job.jar/classes/"
, "job.jar/lib/*", ApplicationConstants.Environment.Pwd.$$() + "/*"));
NUnit.Framework.Assert.AreEqual("MAPREDUCE_JOB_CLASSLOADER true, but job.jar is not in the app"
+ " classpath!", expectedAppClasspath, appCp);
}
public virtual void TestTaskAttemptIDShort()
{
MRApps.ToTaskAttemptID("attempt_0_0_0_m_0");
}
// tests against some proto.id and not a job.id field
public virtual void TestJobIDShort()
{
MRApps.ToJobID("job_0_0_0");
}
public virtual void TestTaskIDShort()
{
MRApps.ToTaskID("task_0_0000_m");
}
public virtual void TestTaskIDBadType()
{
MRApps.ToTaskID("task_0_0000_x_000000");
}