public virtual void TestMapRedExecutionEnv()
{
// test if the env variable can be set
try
{
// Application environment
IDictionary <string, string> environment = new Dictionary <string, string>();
string setupHadoopHomeCommand = Shell.Windows ? "HADOOP_COMMON_HOME=C:\\fake\\PATH\\to\\hadoop\\common\\home"
: "HADOOP_COMMON_HOME=/fake/path/to/hadoop/common/home";
MRApps.SetEnvFromInputString(environment, setupHadoopHomeCommand, conf);
// Add the env variables passed by the admin
MRApps.SetEnvFromInputString(environment, conf.Get(MRJobConfig.MapredAdminUserEnv
, MRJobConfig.DefaultMapredAdminUserEnv), conf);
string executionPaths = environment[Shell.Windows ? "PATH" : "LD_LIBRARY_PATH"];
string toFind = Shell.Windows ? "C:\\fake\\PATH\\to\\hadoop\\common\\home\\bin" :
"/fake/path/to/hadoop/common/home/lib/native";
// Ensure execution PATH/LD_LIBRARY_PATH set up pointing to hadoop lib
NUnit.Framework.Assert.IsTrue("execution path does not include the hadoop lib location "
+ toFind, executionPaths.Contains(toFind));
}
catch (Exception e)
{
Sharpen.Runtime.PrintStackTrace(e);
NUnit.Framework.Assert.Fail("Exception in testing execution environment for MapReduce task"
);
TearDown();
}
// now launch a mapreduce job to ensure that the child
// also gets the configured setting for hadoop lib
try
{
JobConf conf = new JobConf(mr.GetConfig());
// initialize input, output directories
Path inDir = new Path("input");
Path outDir = new Path("output");
string input = "The input";
// set config to use the ExecutionEnvCheckMapClass map class
Configure(conf, inDir, outDir, input, typeof(TestMiniMRChildTask.ExecutionEnvCheckMapClass
), typeof(IdentityReducer));
LaunchTest(conf, inDir, outDir, input);
}
catch (Exception e)
{
Sharpen.Runtime.PrintStackTrace(e);
NUnit.Framework.Assert.Fail("Exception in testing propagation of env setting to child task"
);
TearDown();
}
}
public virtual void TestSocketFactory()
{
// Create a standard mini-cluster
Configuration sconf = new Configuration();
MiniDFSCluster cluster = new MiniDFSCluster.Builder(sconf).NumDataNodes(1).Build(
);
int nameNodePort = cluster.GetNameNodePort();
// Get a reference to its DFS directly
FileSystem fs = cluster.GetFileSystem();
NUnit.Framework.Assert.IsTrue(fs is DistributedFileSystem);
DistributedFileSystem directDfs = (DistributedFileSystem)fs;
Configuration cconf = GetCustomSocketConfigs(nameNodePort);
fs = FileSystem.Get(cconf);
NUnit.Framework.Assert.IsTrue(fs is DistributedFileSystem);
DistributedFileSystem dfs = (DistributedFileSystem)fs;
JobClient client = null;
MiniMRYarnCluster miniMRYarnCluster = null;
try
{
// This will test RPC to the NameNode only.
// could we test Client-DataNode connections?
Path filePath = new Path("/dir");
NUnit.Framework.Assert.IsFalse(directDfs.Exists(filePath));
NUnit.Framework.Assert.IsFalse(dfs.Exists(filePath));
directDfs.Mkdirs(filePath);
NUnit.Framework.Assert.IsTrue(directDfs.Exists(filePath));
NUnit.Framework.Assert.IsTrue(dfs.Exists(filePath));
// This will test RPC to a Resource Manager
fs = FileSystem.Get(sconf);
JobConf jobConf = new JobConf();
FileSystem.SetDefaultUri(jobConf, fs.GetUri().ToString());
miniMRYarnCluster = InitAndStartMiniMRYarnCluster(jobConf);
JobConf jconf = new JobConf(miniMRYarnCluster.GetConfig());
jconf.Set("hadoop.rpc.socket.factory.class.default", "org.apache.hadoop.ipc.DummySocketFactory"
);
jconf.Set(MRConfig.FrameworkName, MRConfig.YarnFrameworkName);
string rmAddress = jconf.Get("yarn.resourcemanager.address");
string[] split = rmAddress.Split(":");
jconf.Set("yarn.resourcemanager.address", split[0] + ':' + (System.Convert.ToInt32
(split[1]) + 10));
client = new JobClient(jconf);
JobStatus[] jobs = client.JobsToComplete();
NUnit.Framework.Assert.IsTrue(jobs.Length == 0);
}
finally
{
CloseClient(client);
CloseDfs(dfs);
CloseDfs(directDfs);
StopMiniMRYarnCluster(miniMRYarnCluster);
ShutdownDFSCluster(cluster);
}
}
public virtual void TestBinaryTokenFile()
{
Configuration conf = mrCluster.GetConfig();
// provide namenodes names for the job to get the delegation tokens for
string nnUri = dfsCluster.GetURI(0).ToString();
conf.Set(MRJobConfig.JobNamenodes, nnUri + "," + nnUri);
// using argument to pass the file name
string[] args = new string[] { "-m", "1", "-r", "1", "-mt", "1", "-rt", "1" };
int res = -1;
try
{
res = ToolRunner.Run(conf, new TestBinaryTokenFile.MySleepJob(this), args);
}
catch (Exception e)
{
System.Console.Out.WriteLine("Job failed with " + e.GetLocalizedMessage());
Sharpen.Runtime.PrintStackTrace(e, System.Console.Out);
NUnit.Framework.Assert.Fail("Job failed");
}
NUnit.Framework.Assert.AreEqual("dist job res is not 0:", 0, res);
}
public virtual Configuration GetConfig()
{
return(miniMRYarnCluster.GetConfig());
}