/// <exception cref="System.Exception"/>
protected override void TearDown()
{
if (mrCluster != null)
{
mrCluster.Shutdown();
}
if (dfsCluster != null)
{
dfsCluster.Shutdown();
}
base.TearDown();
}
/// <exception cref="System.IO.IOException"/>
public virtual void TestPipes()
{
if (Runtime.GetProperty("compile.c++") == null)
{
Log.Info("compile.c++ is not defined, so skipping TestPipes");
return;
}
MiniDFSCluster dfs = null;
MiniMRCluster mr = null;
Path inputPath = new Path("testing/in");
Path outputPath = new Path("testing/out");
try
{
int numSlaves = 2;
Configuration conf = new Configuration();
dfs = new MiniDFSCluster.Builder(conf).NumDataNodes(numSlaves).Build();
mr = new MiniMRCluster(numSlaves, dfs.GetFileSystem().GetUri().ToString(), 1);
WriteInputFile(dfs.GetFileSystem(), inputPath);
RunProgram(mr, dfs, wordCountSimple, inputPath, outputPath, 3, 2, twoSplitOutput,
null);
Cleanup(dfs.GetFileSystem(), outputPath);
RunProgram(mr, dfs, wordCountSimple, inputPath, outputPath, 3, 0, noSortOutput, null
);
Cleanup(dfs.GetFileSystem(), outputPath);
RunProgram(mr, dfs, wordCountPart, inputPath, outputPath, 3, 2, fixedPartitionOutput
, null);
RunNonPipedProgram(mr, dfs, wordCountNoPipes, null);
mr.WaitUntilIdle();
}
finally
{
mr.Shutdown();
dfs.Shutdown();
}
}
/// <exception cref="System.Exception"/>
public virtual void TestLazyOutput()
{
MiniDFSCluster dfs = null;
MiniMRCluster mr = null;
FileSystem fileSys = null;
try
{
Configuration conf = new Configuration();
// Start the mini-MR and mini-DFS clusters
dfs = new MiniDFSCluster.Builder(conf).NumDataNodes(NumHadoopSlaves).Build();
fileSys = dfs.GetFileSystem();
mr = new MiniMRCluster(NumHadoopSlaves, fileSys.GetUri().ToString(), 1);
int numReducers = 2;
int numMappers = NumHadoopSlaves * NumMapsPerNode;
CreateInput(fileSys, numMappers);
Path output1 = new Path("/testlazy/output1");
// Test 1.
RunTestLazyOutput(mr.CreateJobConf(), output1, numReducers, true);
Path[] fileList = FileUtil.Stat2Paths(fileSys.ListStatus(output1, new Utils.OutputFileUtils.OutputFilesFilter
()));
for (int i = 0; i < fileList.Length; ++i)
{
System.Console.Out.WriteLine("Test1 File list[" + i + "]" + ": " + fileList[i]);
}
NUnit.Framework.Assert.IsTrue(fileList.Length == (numReducers - 1));
// Test 2. 0 Reducers, maps directly write to the output files
Path output2 = new Path("/testlazy/output2");
RunTestLazyOutput(mr.CreateJobConf(), output2, 0, true);
fileList = FileUtil.Stat2Paths(fileSys.ListStatus(output2, new Utils.OutputFileUtils.OutputFilesFilter
()));
for (int i_1 = 0; i_1 < fileList.Length; ++i_1)
{
System.Console.Out.WriteLine("Test2 File list[" + i_1 + "]" + ": " + fileList[i_1
]);
}
NUnit.Framework.Assert.IsTrue(fileList.Length == numMappers - 1);
// Test 3. 0 Reducers, but flag is turned off
Path output3 = new Path("/testlazy/output3");
RunTestLazyOutput(mr.CreateJobConf(), output3, 0, false);
fileList = FileUtil.Stat2Paths(fileSys.ListStatus(output3, new Utils.OutputFileUtils.OutputFilesFilter
()));
for (int i_2 = 0; i_2 < fileList.Length; ++i_2)
{
System.Console.Out.WriteLine("Test3 File list[" + i_2 + "]" + ": " + fileList[i_2
]);
}
NUnit.Framework.Assert.IsTrue(fileList.Length == numMappers);
}
finally
{
if (dfs != null)
{
dfs.Shutdown();
}
if (mr != null)
{
mr.Shutdown();
}
}
}