public static void Setup()
{
Configuration conf = new Configuration();
Path TestRootDir = new Path(Runtime.GetProperty("test.build.data", "/tmp"));
testdir = new Path(TestRootDir, "TestMiniMRClientCluster");
inDir = new Path(testdir, "in");
outDir = new Path(testdir, "out");
FileSystem fs = FileSystem.GetLocal(conf);
if (fs.Exists(testdir) && !fs.Delete(testdir, true))
{
throw new IOException("Could not delete " + testdir);
}
if (!fs.Mkdirs(inDir))
{
throw new IOException("Mkdirs failed to create " + inDir);
}
for (int i = 0; i < inFiles.Length; i++)
{
inFiles[i] = new Path(inDir, "part_" + i);
CreateFile(inFiles[i], conf);
}
// create the mini cluster to be used for the tests
mrCluster = MiniMRClientClusterFactory.Create(typeof(TestMiniMRClientCluster.InternalClass
), 1, new Configuration());
}
// Number of input files is same as the number of mappers.
// Number of reducers.
// Number of lines per input file.
// Where MR job's input will reside.
// Where output goes.
/// <exception cref="System.Exception"/>
public virtual void TestMerge()
{
MiniDFSCluster dfsCluster = null;
MiniMRClientCluster mrCluster = null;
FileSystem fileSystem = null;
try
{
Configuration conf = new Configuration();
// Start the mini-MR and mini-DFS clusters
dfsCluster = new MiniDFSCluster.Builder(conf).NumDataNodes(NumHadoopDataNodes).Build
();
fileSystem = dfsCluster.GetFileSystem();
mrCluster = MiniMRClientClusterFactory.Create(this.GetType(), NumHadoopDataNodes,
conf);
// Generate input.
CreateInput(fileSystem);
// Run the test.
RunMergeTest(new JobConf(mrCluster.GetConfig()), fileSystem);
}
finally
{
if (dfsCluster != null)
{
dfsCluster.Shutdown();
}
if (mrCluster != null)
{
mrCluster.Stop();
}
}
}
/// <exception cref="System.Exception"/>
public virtual void DoEncryptionTest(int numMappers, int numReducers, int numNodes
, int numLines, bool isUber)
{
MiniDFSCluster dfsCluster = null;
MiniMRClientCluster mrCluster = null;
FileSystem fileSystem = null;
try
{
Configuration conf = new Configuration();
// Start the mini-MR and mini-DFS clusters
dfsCluster = new MiniDFSCluster.Builder(conf).NumDataNodes(numNodes).Build();
fileSystem = dfsCluster.GetFileSystem();
mrCluster = MiniMRClientClusterFactory.Create(this.GetType(), numNodes, conf);
// Generate input.
CreateInput(fileSystem, numMappers, numLines);
// Run the test.
RunMergeTest(new JobConf(mrCluster.GetConfig()), fileSystem, numMappers, numReducers
, numLines, isUber);
}
finally
{
if (dfsCluster != null)
{
dfsCluster.Shutdown();
}
if (mrCluster != null)
{
mrCluster.Stop();
}
}
}
/// <exception cref="System.IO.IOException"/>
private MiniMRClientCluster CreateMiniClusterWithCapacityScheduler()
{
Configuration conf = new Configuration();
// Expected queue names depending on Capacity Scheduler queue naming
conf.SetClass(YarnConfiguration.RmScheduler, typeof(CapacityScheduler), typeof(CapacityScheduler
));
return(MiniMRClientClusterFactory.Create(this.GetType(), 2, conf));
}
/// <exception cref="System.IO.IOException"/>
public MiniMRCluster(int jobTrackerPort, int taskTrackerPort, int numTaskTrackers
, string namenode, int numDir, string[] racks, string[] hosts, UserGroupInformation
ugi, JobConf conf, int numTrackerToExclude, Clock clock)
{
if (conf == null)
{
conf = new JobConf();
}
FileSystem.SetDefaultUri(conf, namenode);
string identifier = this.GetType().Name + "_" + Sharpen.Extensions.ToString(new Random
().Next(int.MaxValue));
mrClientCluster = MiniMRClientClusterFactory.Create(this.GetType(), identifier, numTaskTrackers
, conf);
}