/// <exception cref="System.IO.IOException"/>
public virtual void TestJobWithDFS()
{
string namenode = null;
MiniDFSCluster dfs = null;
MiniMRCluster mr = null;
FileSystem fileSys = null;
try
{
int taskTrackers = 4;
int jobTrackerPort = 60050;
Configuration conf = new Configuration();
dfs = new MiniDFSCluster.Builder(conf).Build();
fileSys = dfs.GetFileSystem();
namenode = fileSys.GetUri().ToString();
mr = new MiniMRCluster(taskTrackers, namenode, 2);
JobConf jobConf = new JobConf();
bool result;
result = LaunchJob(fileSys.GetUri(), jobConf, 3, 1);
NUnit.Framework.Assert.IsTrue(result);
}
finally
{
if (dfs != null)
{
dfs.Shutdown();
}
if (mr != null)
{
mr.Shutdown();
}
}
}
public virtual void TestExternalWritable()
{
string namenode = null;
MiniDFSCluster dfs = null;
MiniMRCluster mr = null;
FileSystem fileSys = null;
try
{
int taskTrackers = 4;
Configuration conf = new Configuration();
dfs = new MiniDFSCluster.Builder(conf).Build();
fileSys = dfs.GetFileSystem();
namenode = fileSys.GetUri().ToString();
mr = new MiniMRCluster(taskTrackers, namenode, 3);
JobConf jobConf = mr.CreateJobConf();
string result;
result = LaunchExternal(fileSys.GetUri(), jobConf, "Dennis was here!\nDennis again!"
, 3, 1);
NUnit.Framework.Assert.AreEqual("Dennis again!\t1\nDennis was here!\t1\n", result
);
}
finally
{
if (dfs != null)
{
dfs.Shutdown();
}
if (mr != null)
{
mr.Shutdown();
}
}
}
public virtual void TestClassPath()
{
string namenode = null;
MiniDFSCluster dfs = null;
MiniMRCluster mr = null;
FileSystem fileSys = null;
try
{
int taskTrackers = 4;
int jobTrackerPort = 60050;
Configuration conf = new Configuration();
dfs = new MiniDFSCluster.Builder(conf).Build();
fileSys = dfs.GetFileSystem();
namenode = fileSys.GetUri().ToString();
mr = new MiniMRCluster(taskTrackers, namenode, 3);
JobConf jobConf = mr.CreateJobConf();
string result;
result = LaunchWordCount(fileSys.GetUri(), jobConf, "The quick brown fox\nhas many silly\n"
+ "red fox sox\n", 3, 1);
NUnit.Framework.Assert.AreEqual("The\t1\nbrown\t1\nfox\t2\nhas\t1\nmany\t1\n" + "quick\t1\nred\t1\nsilly\t1\nsox\t1\n"
, result);
}
finally
{
if (dfs != null)
{
dfs.Shutdown();
}
if (mr != null)
{
mr.Shutdown();
}
}
}
/// <exception cref="System.Exception"/>
private void TestCachingAtLevel(int level)
{
string namenode = null;
MiniDFSCluster dfs = null;
MiniMRCluster mr = null;
FileSystem fileSys = null;
string testName = "TestMultiLevelCaching";
try
{
int taskTrackers = 1;
// generate the racks
// use rack1 for data node
string rack1 = GetRack(0, level);
// use rack2 for task tracker
string rack2 = GetRack(1, level);
Configuration conf = new Configuration();
// Run a datanode on host1 under /a/b/c/..../d1/e1/f1
dfs = new MiniDFSCluster.Builder(conf).Racks(new string[] { rack1 }).Hosts(new string
[] { "host1.com" }).Build();
dfs.WaitActive();
fileSys = dfs.GetFileSystem();
if (!fileSys.Mkdirs(inDir))
{
throw new IOException("Mkdirs failed to create " + inDir.ToString());
}
UtilsForTests.WriteFile(dfs.GetNameNode(), conf, new Path(inDir + "/file"), (short
)1);
namenode = (dfs.GetFileSystem()).GetUri().GetHost() + ":" + (dfs.GetFileSystem())
.GetUri().GetPort();
// Run a job with the (only)tasktracker on host2 under diff topology
// e.g /a/b/c/..../d2/e2/f2.
JobConf jc = new JobConf();
// cache-level = level (unshared levels) + 1(topmost shared node i.e /a)
// + 1 (for host)
jc.SetInt(JTConfig.JtTaskcacheLevels, level + 2);
mr = new MiniMRCluster(taskTrackers, namenode, 1, new string[] { rack2 }, new string
[] { "host2.com" }, jc);
/* The job is configured with 1 map for one (non-splittable) file.
* Since the datanode is running under different subtree, there is no
* node-level data locality but there should be topological locality.
*/
LaunchJobAndTestCounters(testName, mr, fileSys, inDir, outputPath, 1, 1, 0, 0);
mr.Shutdown();
}
finally
{
if (null != fileSys)
{
// inDir, outputPath only exist if fileSys is valid.
fileSys.Delete(inDir, true);
fileSys.Delete(outputPath, true);
}
if (dfs != null)
{
dfs.Shutdown();
}
}
}
/// <exception cref="System.IO.IOException"/>
public virtual void TestWithDFS()
{
MiniDFSCluster dfs = null;
MiniMRCluster mr = null;
FileSystem fileSys = null;
try
{
int taskTrackers = 4;
JobConf conf = new JobConf();
conf.Set(JTConfig.JtSystemDir, "/tmp/custom/mapred/system");
dfs = new MiniDFSCluster.Builder(conf).NumDataNodes(4).Build();
fileSys = dfs.GetFileSystem();
mr = new MiniMRCluster(taskTrackers, fileSys.GetUri().ToString(), 1, null, null,
conf);
RunWordCount(mr, mr.CreateJobConf(), conf.Get("mapred.system.dir"));
}
finally
{
if (dfs != null)
{
dfs.Shutdown();
}
if (mr != null)
{
mr.Shutdown();
}
}
}
/// <summary>Stops the cluster within a testcase.</summary>
/// <remarks>
/// Stops the cluster within a testcase.
/// <p/>
/// Note that the cluster is already started when the testcase method
/// is invoked. This method is useful if as part of the testcase the
/// cluster has to be shutdown.
/// <p/>
/// If the cluster is already stopped this method does nothing.
/// </remarks>
/// <exception cref="System.Exception">if the cluster could not be stopped</exception>
protected internal virtual void StopCluster()
{
if (mrCluster != null)
{
mrCluster.Shutdown();
mrCluster = null;
}
if (dfsCluster != null)
{
dfsCluster.Shutdown();
dfsCluster = null;
}
}
/// <exception cref="System.IO.IOException"/>
public virtual void TestBringUp()
{
MiniMRCluster mr = null;
try
{
mr = new MiniMRCluster(1, "local", 1);
}
finally
{
if (mr != null)
{
mr.Shutdown();
}
}
}
/// <exception cref="System.IO.IOException"/>
internal static void RunWordCount(MiniMRCluster mr, JobConf jobConf, string sysDir
)
{
Log.Info("runWordCount");
// Run a word count example
// Keeping tasks that match this pattern
TestJobSysDirWithDFS.TestResult result;
Path inDir = new Path("./wc/input");
Path outDir = new Path("./wc/output");
result = LaunchWordCount(jobConf, inDir, outDir, "The quick brown fox\nhas many silly\n"
+ "red fox sox\n", 3, 1, sysDir);
NUnit.Framework.Assert.AreEqual("The\t1\nbrown\t1\nfox\t2\nhas\t1\nmany\t1\n" + "quick\t1\nred\t1\nsilly\t1\nsox\t1\n"
, result.output);
// Checking if the Job ran successfully in spite of different system dir config
// between Job Client & Job Tracker
NUnit.Framework.Assert.IsTrue(result.job.IsSuccessful());
}
public virtual void SetUp()
{
dfs = new MiniDFSCluster.Builder(conf).NumDataNodes(4).Build();
fs = DfsUgi.DoAs(new _PrivilegedExceptionAction_80(this));
// Home directories for users
Mkdir(fs, "/user", "nobody", "nogroup", (short)0x3ff);
Mkdir(fs, "/user/alice", "alice", "nogroup", (short)0x1ed);
Mkdir(fs, "/user/bob", "bob", "nogroup", (short)0x1ed);
// staging directory root with sticky bit
UserGroupInformation MrUgi = UserGroupInformation.GetLoginUser();
Mkdir(fs, "/staging", MrUgi.GetShortUserName(), "nogroup", (short)0x3ff);
JobConf mrConf = new JobConf();
mrConf.Set(JTConfig.JtStagingAreaRoot, "/staging");
mr = new MiniMRCluster(0, 0, 4, dfs.GetFileSystem().GetUri().ToString(), 1, null,
null, MrUgi, mrConf);
}
/// <exception cref="System.IO.IOException"/>
public virtual void TestWithDFS()
{
MiniMRCluster mr = null;
MiniDFSCluster dfs = null;
FileSystem fileSys = null;
try
{
JobConf conf = new JobConf();
dfs = new MiniDFSCluster.Builder(conf).Build();
fileSys = dfs.GetFileSystem();
mr = new MiniMRCluster(2, fileSys.GetUri().ToString(), 4);
MRCaching.SetupCache("/cachedir", fileSys);
// run the wordcount example with caching
MRCaching.TestResult ret = MRCaching.LaunchMRCache("/testing/wc/input", "/testing/wc/output"
, "/cachedir", mr.CreateJobConf(), "The quick brown fox\nhas many silly\n" + "red fox sox\n"
);
NUnit.Framework.Assert.IsTrue("Archives not matching", ret.isOutputOk);
// launch MR cache with symlinks
ret = MRCaching.LaunchMRCache("/testing/wc/input", "/testing/wc/output", "/cachedir"
, mr.CreateJobConf(), "The quick brown fox\nhas many silly\n" + "red fox sox\n");
NUnit.Framework.Assert.IsTrue("Archives not matching", ret.isOutputOk);
}
finally
{
if (fileSys != null)
{
fileSys.Close();
}
if (dfs != null)
{
dfs.Shutdown();
}
if (mr != null)
{
mr.Shutdown();
}
}
}
/// <summary>
/// Creates Hadoop instance based on constructor configuration before
/// a test case is run.
/// </summary>
/// <exception cref="System.Exception"/>
protected override void SetUp()
{
base.SetUp();
if (localFS)
{
fileSystem = FileSystem.GetLocal(new JobConf());
}
else
{
dfsCluster = new MiniDFSCluster.Builder(new JobConf()).NumDataNodes(dataNodes).Build
();
fileSystem = dfsCluster.GetFileSystem();
}
if (localMR)
{
}
else
{
//noinspection deprecation
mrCluster = new MiniMRCluster(taskTrackers, fileSystem.GetUri().ToString(), 1);
}
}
/// <summary>Launches a MR job and tests the job counters against the expected values.
/// </summary>
/// <param name="testName">The name for the job</param>
/// <param name="mr">The MR cluster</param>
/// <param name="fileSys">The FileSystem</param>
/// <param name="in">Input path</param>
/// <param name="out">Output path</param>
/// <param name="numMaps">Number of maps</param>
/// <param name="otherLocalMaps">Expected value of other local maps</param>
/// <param name="datalocalMaps">Expected value of data(node) local maps</param>
/// <param name="racklocalMaps">Expected value of rack local maps</param>
/// <exception cref="System.IO.IOException"/>
internal static void LaunchJobAndTestCounters(string jobName, MiniMRCluster mr, FileSystem
fileSys, Path @in, Path @out, int numMaps, int otherLocalMaps, int dataLocalMaps
, int rackLocalMaps)
{
JobConf jobConf = mr.CreateJobConf();
if (fileSys.Exists(@out))
{
fileSys.Delete(@out, true);
}
RunningJob job = LaunchJob(jobConf, @in, @out, numMaps, jobName);
Counters counters = job.GetCounters();
NUnit.Framework.Assert.AreEqual("Number of local maps", counters.GetCounter(JobCounter
.OtherLocalMaps), otherLocalMaps);
NUnit.Framework.Assert.AreEqual("Number of Data-local maps", counters.GetCounter(
JobCounter.DataLocalMaps), dataLocalMaps);
NUnit.Framework.Assert.AreEqual("Number of Rack-local maps", counters.GetCounter(
JobCounter.RackLocalMaps), rackLocalMaps);
mr.WaitUntilIdle();
mr.Shutdown();
}
/// <summary>Starts the cluster within a testcase.</summary>
/// <remarks>
/// Starts the cluster within a testcase.
/// <p/>
/// Note that the cluster is already started when the testcase method
/// is invoked. This method is useful if as part of the testcase the
/// cluster has to be shutdown and restarted again.
/// <p/>
/// If the cluster is already running this method does nothing.
/// </remarks>
/// <param name="reformatDFS">indicates if DFS has to be reformated</param>
/// <param name="props">configuration properties to inject to the mini cluster</param>
/// <exception cref="System.Exception">if the cluster could not be started</exception>
protected internal virtual void StartCluster(bool reformatDFS, Properties props)
{
lock (this)
{
if (dfsCluster == null)
{
JobConf conf = new JobConf();
if (props != null)
{
foreach (DictionaryEntry entry in props)
{
conf.Set((string)entry.Key, (string)entry.Value);
}
}
dfsCluster = new MiniDFSCluster.Builder(conf).NumDataNodes(2).Format(reformatDFS)
.Racks(null).Build();
ClusterMapReduceTestCase.ConfigurableMiniMRCluster.SetConfiguration(props);
//noinspection deprecation
mrCluster = new ClusterMapReduceTestCase.ConfigurableMiniMRCluster(2, GetFileSystem
().GetUri().ToString(), 1, conf);
}
}
}
public static void SetUp()
{
JobConf conf = new JobConf();
fileSys = FileSystem.Get(conf);
fileSys.Delete(new Path(TestRootDir), true);
conf.Set("mapred.job.tracker.handler.count", "1");
conf.Set("mapred.job.tracker", "127.0.0.1:0");
conf.Set("mapred.job.tracker.http.address", "127.0.0.1:0");
conf.Set("mapred.task.tracker.http.address", "127.0.0.1:0");
conf.Set(JHAdminConfig.MrHistoryIntermediateDoneDir, TestRootDir + "/intermediate"
);
conf.Set(FileOutputCommitter.SuccessfulJobOutputDirMarker, "true");
mr = new MiniMRCluster(1, "file:///", 1, null, null, conf);
inDir = new Path(TestRootDir, "test-input");
string input = "The quick brown fox\n" + "has many silly\n" + "red fox sox\n";
DataOutputStream file = fileSys.Create(new Path(inDir, "part-" + 0));
file.WriteBytes(input);
file.Close();
emptyInDir = new Path(TestRootDir, "empty-input");
fileSys.Mkdirs(emptyInDir);
}
public virtual void TestHeapUsageCounter()
{
JobConf conf = new JobConf();
// create a local filesystem handle
FileSystem fileSystem = FileSystem.GetLocal(conf);
// define test root directories
Path rootDir = new Path(Runtime.GetProperty("test.build.data", "/tmp"));
Path testRootDir = new Path(rootDir, "testHeapUsageCounter");
// cleanup the test root directory
fileSystem.Delete(testRootDir, true);
// set the current working directory
fileSystem.SetWorkingDirectory(testRootDir);
fileSystem.DeleteOnExit(testRootDir);
// create a mini cluster using the local file system
MiniMRCluster mrCluster = new MiniMRCluster(1, fileSystem.GetUri().ToString(), 1);
try
{
conf = mrCluster.CreateJobConf();
JobClient jobClient = new JobClient(conf);
// define job input
Path inDir = new Path(testRootDir, "in");
// create input data
CreateWordsFile(inDir, conf);
// configure and run a low memory job which will run without loading the
// jvm's heap
RunningJob lowMemJob = RunHeapUsageTestJob(conf, testRootDir, "-Xms32m -Xmx1G", 0
, 0, fileSystem, jobClient, inDir);
JobID lowMemJobID = lowMemJob.GetID();
long lowMemJobMapHeapUsage = GetTaskCounterUsage(jobClient, lowMemJobID, 1, 0, TaskType
.Map);
System.Console.Out.WriteLine("Job1 (low memory job) map task heap usage: " + lowMemJobMapHeapUsage
);
long lowMemJobReduceHeapUsage = GetTaskCounterUsage(jobClient, lowMemJobID, 1, 0,
TaskType.Reduce);
System.Console.Out.WriteLine("Job1 (low memory job) reduce task heap usage: " + lowMemJobReduceHeapUsage
);
// configure and run a high memory job which will load the jvm's heap
RunningJob highMemJob = RunHeapUsageTestJob(conf, testRootDir, "-Xms32m -Xmx1G",
lowMemJobMapHeapUsage + 256 * 1024 * 1024, lowMemJobReduceHeapUsage + 256 * 1024
* 1024, fileSystem, jobClient, inDir);
JobID highMemJobID = highMemJob.GetID();
long highMemJobMapHeapUsage = GetTaskCounterUsage(jobClient, highMemJobID, 1, 0,
TaskType.Map);
System.Console.Out.WriteLine("Job2 (high memory job) map task heap usage: " + highMemJobMapHeapUsage
);
long highMemJobReduceHeapUsage = GetTaskCounterUsage(jobClient, highMemJobID, 1,
0, TaskType.Reduce);
System.Console.Out.WriteLine("Job2 (high memory job) reduce task heap usage: " +
highMemJobReduceHeapUsage);
NUnit.Framework.Assert.IsTrue("Incorrect map heap usage reported by the map task"
, lowMemJobMapHeapUsage < highMemJobMapHeapUsage);
NUnit.Framework.Assert.IsTrue("Incorrect reduce heap usage reported by the reduce task"
, lowMemJobReduceHeapUsage < highMemJobReduceHeapUsage);
}
finally
{
// shutdown the mr cluster
mrCluster.Shutdown();
try
{
fileSystem.Delete(testRootDir, true);
}
catch (IOException)
{
}
}
}
// Input output paths for this..
// these are all dummy and does not test
// much in map reduce except for the command line
// params
/// <exception cref="System.Exception"/>
public virtual void TestJobShell()
{
MiniDFSCluster dfs = null;
MiniMRCluster mr = null;
FileSystem fs = null;
Path testFile = new Path(input, "testfile");
try
{
Configuration conf = new Configuration();
//start the mini mr and dfs cluster.
dfs = new MiniDFSCluster.Builder(conf).NumDataNodes(2).Build();
fs = dfs.GetFileSystem();
FSDataOutputStream stream = fs.Create(testFile);
stream.Write(Sharpen.Runtime.GetBytesForString("teststring"));
stream.Close();
mr = new MiniMRCluster(2, fs.GetUri().ToString(), 1);
FilePath thisbuildDir = new FilePath(buildDir, "jobCommand");
NUnit.Framework.Assert.IsTrue("create build dir", thisbuildDir.Mkdirs());
FilePath f = new FilePath(thisbuildDir, "files_tmp");
FileOutputStream fstream = new FileOutputStream(f);
fstream.Write(Sharpen.Runtime.GetBytesForString("somestrings"));
fstream.Close();
FilePath f1 = new FilePath(thisbuildDir, "files_tmp1");
fstream = new FileOutputStream(f1);
fstream.Write(Sharpen.Runtime.GetBytesForString("somestrings"));
fstream.Close();
// copy files to dfs
Path cachePath = new Path("/cacheDir");
if (!fs.Mkdirs(cachePath))
{
throw new IOException("Mkdirs failed to create " + cachePath.ToString());
}
Path localCachePath = new Path(Runtime.GetProperty("test.cache.data"));
Path txtPath = new Path(localCachePath, new Path("test.txt"));
Path jarPath = new Path(localCachePath, new Path("test.jar"));
Path zipPath = new Path(localCachePath, new Path("test.zip"));
Path tarPath = new Path(localCachePath, new Path("test.tar"));
Path tgzPath = new Path(localCachePath, new Path("test.tgz"));
fs.CopyFromLocalFile(txtPath, cachePath);
fs.CopyFromLocalFile(jarPath, cachePath);
fs.CopyFromLocalFile(zipPath, cachePath);
// construct options for -files
string[] files = new string[3];
files[0] = f.ToString();
files[1] = f1.ToString() + "#localfilelink";
files[2] = fs.GetUri().Resolve(cachePath + "/test.txt#dfsfilelink").ToString();
// construct options for -libjars
string[] libjars = new string[2];
libjars[0] = "build/test/mapred/testjar/testjob.jar";
libjars[1] = fs.GetUri().Resolve(cachePath + "/test.jar").ToString();
// construct options for archives
string[] archives = new string[3];
archives[0] = tgzPath.ToString();
archives[1] = tarPath + "#tarlink";
archives[2] = fs.GetUri().Resolve(cachePath + "/test.zip#ziplink").ToString();
string[] args = new string[10];
args[0] = "-files";
args[1] = StringUtils.ArrayToString(files);
args[2] = "-libjars";
// the testjob.jar as a temporary jar file
// rather than creating its own
args[3] = StringUtils.ArrayToString(libjars);
args[4] = "-archives";
args[5] = StringUtils.ArrayToString(archives);
args[6] = "-D";
args[7] = "mapred.output.committer.class=testjar.CustomOutputCommitter";
args[8] = input.ToString();
args[9] = output.ToString();
JobConf jobConf = mr.CreateJobConf();
//before running the job, verify that libjar is not in client classpath
NUnit.Framework.Assert.IsTrue("libjar not in client classpath", LoadLibJar(jobConf
) == null);
int ret = ToolRunner.Run(jobConf, new ExternalMapReduce(), args);
//after running the job, verify that libjar is in the client classpath
NUnit.Framework.Assert.IsTrue("libjar added to client classpath", LoadLibJar(jobConf
) != null);
NUnit.Framework.Assert.IsTrue("not failed ", ret != -1);
f.Delete();
thisbuildDir.Delete();
}
finally
{
if (dfs != null)
{
dfs.Shutdown();
}
if (mr != null)
{
mr.Shutdown();
}
}
}
internal TaskTrackerRunner(MiniMRCluster _enclosing)
{
this._enclosing = _enclosing;
}
/// <exception cref="System.Exception"/>
public virtual void TestLazyOutput()
{
MiniDFSCluster dfs = null;
MiniMRCluster mr = null;
FileSystem fileSys = null;
try
{
Configuration conf = new Configuration();
// Start the mini-MR and mini-DFS clusters
dfs = new MiniDFSCluster.Builder(conf).NumDataNodes(NumHadoopSlaves).Build();
fileSys = dfs.GetFileSystem();
mr = new MiniMRCluster(NumHadoopSlaves, fileSys.GetUri().ToString(), 1);
int numReducers = 2;
int numMappers = NumHadoopSlaves * NumMapsPerNode;
CreateInput(fileSys, numMappers);
Path output1 = new Path("/testlazy/output1");
// Test 1.
RunTestLazyOutput(mr.CreateJobConf(), output1, numReducers, true);
Path[] fileList = FileUtil.Stat2Paths(fileSys.ListStatus(output1, new Utils.OutputFileUtils.OutputFilesFilter
()));
for (int i = 0; i < fileList.Length; ++i)
{
System.Console.Out.WriteLine("Test1 File list[" + i + "]" + ": " + fileList[i]);
}
NUnit.Framework.Assert.IsTrue(fileList.Length == (numReducers - 1));
// Test 2. 0 Reducers, maps directly write to the output files
Path output2 = new Path("/testlazy/output2");
RunTestLazyOutput(mr.CreateJobConf(), output2, 0, true);
fileList = FileUtil.Stat2Paths(fileSys.ListStatus(output2, new Utils.OutputFileUtils.OutputFilesFilter
()));
for (int i_1 = 0; i_1 < fileList.Length; ++i_1)
{
System.Console.Out.WriteLine("Test2 File list[" + i_1 + "]" + ": " + fileList[i_1
]);
}
NUnit.Framework.Assert.IsTrue(fileList.Length == numMappers - 1);
// Test 3. 0 Reducers, but flag is turned off
Path output3 = new Path("/testlazy/output3");
RunTestLazyOutput(mr.CreateJobConf(), output3, 0, false);
fileList = FileUtil.Stat2Paths(fileSys.ListStatus(output3, new Utils.OutputFileUtils.OutputFilesFilter
()));
for (int i_2 = 0; i_2 < fileList.Length; ++i_2)
{
System.Console.Out.WriteLine("Test3 File list[" + i_2 + "]" + ": " + fileList[i_2
]);
}
NUnit.Framework.Assert.IsTrue(fileList.Length == numMappers);
}
finally
{
if (dfs != null)
{
dfs.Shutdown();
}
if (mr != null)
{
mr.Shutdown();
}
}
}
