// Number of input files is same as the number of mappers.
// Number of reducers.
// Number of lines per input file.
// Where MR job's input will reside.
// Where output goes.
/// <exception cref="System.Exception"/>
public virtual void TestMerge()
{
MiniDFSCluster dfsCluster = null;
MiniMRClientCluster mrCluster = null;
FileSystem fileSystem = null;
try
{
Configuration conf = new Configuration();
// Start the mini-MR and mini-DFS clusters
dfsCluster = new MiniDFSCluster.Builder(conf).NumDataNodes(NumHadoopDataNodes).Build
();
fileSystem = dfsCluster.GetFileSystem();
mrCluster = MiniMRClientClusterFactory.Create(this.GetType(), NumHadoopDataNodes,
conf);
// Generate input.
CreateInput(fileSystem);
// Run the test.
RunMergeTest(new JobConf(mrCluster.GetConfig()), fileSystem);
}
finally
{
if (dfsCluster != null)
{
dfsCluster.Shutdown();
}
if (mrCluster != null)
{
mrCluster.Stop();
}
}
}
public static void Setup()
{
Configuration conf = new Configuration();
Path TestRootDir = new Path(Runtime.GetProperty("test.build.data", "/tmp"));
testdir = new Path(TestRootDir, "TestMiniMRClientCluster");
inDir = new Path(testdir, "in");
outDir = new Path(testdir, "out");
FileSystem fs = FileSystem.GetLocal(conf);
if (fs.Exists(testdir) && !fs.Delete(testdir, true))
{
throw new IOException("Could not delete " + testdir);
}
if (!fs.Mkdirs(inDir))
{
throw new IOException("Mkdirs failed to create " + inDir);
}
for (int i = 0; i < inFiles.Length; i++)
{
inFiles[i] = new Path(inDir, "part_" + i);
CreateFile(inFiles[i], conf);
}
// create the mini cluster to be used for the tests
mrCluster = MiniMRClientClusterFactory.Create(typeof(TestMiniMRClientCluster.InternalClass
), 1, new Configuration());
}
/// <exception cref="System.Exception"/>
public virtual void DoEncryptionTest(int numMappers, int numReducers, int numNodes
, int numLines, bool isUber)
{
MiniDFSCluster dfsCluster = null;
MiniMRClientCluster mrCluster = null;
FileSystem fileSystem = null;
try
{
Configuration conf = new Configuration();
// Start the mini-MR and mini-DFS clusters
dfsCluster = new MiniDFSCluster.Builder(conf).NumDataNodes(numNodes).Build();
fileSystem = dfsCluster.GetFileSystem();
mrCluster = MiniMRClientClusterFactory.Create(this.GetType(), numNodes, conf);
// Generate input.
CreateInput(fileSystem, numMappers, numLines);
// Run the test.
RunMergeTest(new JobConf(mrCluster.GetConfig()), fileSystem, numMappers, numReducers
, numLines, isUber);
}
finally
{
if (dfsCluster != null)
{
dfsCluster.Shutdown();
}
if (mrCluster != null)
{
mrCluster.Stop();
}
}
}
/// <exception cref="System.Exception"/>
private void StartCluster(Configuration conf)
{
if (Runtime.GetProperty("hadoop.log.dir") == null)
{
Runtime.SetProperty("hadoop.log.dir", "target/test-dir");
}
conf.Set("dfs.block.access.token.enable", "false");
conf.Set("dfs.permissions", "true");
conf.Set("hadoop.security.authentication", "simple");
string cp = conf.Get(YarnConfiguration.YarnApplicationClasspath, StringUtils.Join
(",", YarnConfiguration.DefaultYarnCrossPlatformApplicationClasspath)) + FilePath
.pathSeparator + classpathDir;
conf.Set(YarnConfiguration.YarnApplicationClasspath, cp);
dfsCluster = new MiniDFSCluster.Builder(conf).Build();
FileSystem fileSystem = dfsCluster.GetFileSystem();
fileSystem.Mkdirs(new Path("/tmp"));
fileSystem.Mkdirs(new Path("/user"));
fileSystem.Mkdirs(new Path("/hadoop/mapred/system"));
fileSystem.SetPermission(new Path("/tmp"), FsPermission.ValueOf("-rwxrwxrwx"));
fileSystem.SetPermission(new Path("/user"), FsPermission.ValueOf("-rwxrwxrwx"));
fileSystem.SetPermission(new Path("/hadoop/mapred/system"), FsPermission.ValueOf(
"-rwx------"));
FileSystem.SetDefaultUri(conf, fileSystem.GetUri());
mrCluster = MiniMRClientClusterFactory.Create(this.GetType(), 1, conf);
// so the minicluster conf is avail to the containers.
TextWriter writer = new FileWriter(classpathDir + "/core-site.xml");
mrCluster.GetConfig().WriteXml(writer);
writer.Close();
}
public virtual void Setup()
{
Configuration conf = new YarnConfiguration();
cluster = MiniMRClientClusterFactory.Create(this.GetType(), 2, conf);
cluster.Start();
}
public virtual void Cleanup()
{
if (cluster != null)
{
cluster.Stop();
cluster = null;
}
}
// nothing
/// <summary>Starts DFS and MR clusters, as specified in member-variable options.</summary>
/// <remarks>
/// Starts DFS and MR clusters, as specified in member-variable options. Also
/// writes out configuration and details, if requested.
/// </remarks>
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.IO.FileNotFoundException"/>
/// <exception cref="Sharpen.URISyntaxException"/>
public virtual void Start()
{
if (!noDFS)
{
dfs = new MiniDFSCluster.Builder(conf).NameNodePort(nnPort).NumDataNodes(numDataNodes
).StartupOption(dfsOpts).Build();
Log.Info("Started MiniDFSCluster -- namenode on port " + dfs.GetNameNodePort());
}
if (!noMR)
{
if (fs == null && dfs != null)
{
fs = dfs.GetFileSystem().GetUri().ToString();
}
else
{
if (fs == null)
{
fs = "file:///tmp/minimr-" + Runtime.NanoTime();
}
}
FileSystem.SetDefaultUri(conf, new URI(fs));
// Instruct the minicluster to use fixed ports, so user will know which
// ports to use when communicating with the cluster.
conf.SetBoolean(YarnConfiguration.YarnMiniclusterFixedPorts, true);
conf.SetBoolean(JHAdminConfig.MrHistoryMiniclusterFixedPorts, true);
conf.Set(YarnConfiguration.RmAddress, MiniYARNCluster.GetHostname() + ":" + this.
rmPort);
conf.Set(JHAdminConfig.MrHistoryAddress, MiniYARNCluster.GetHostname() + ":" + this
.jhsPort);
mr = MiniMRClientClusterFactory.Create(this.GetType(), numNodeManagers, conf);
Log.Info("Started MiniMRCluster");
}
if (writeConfig != null)
{
FileOutputStream fos = new FileOutputStream(new FilePath(writeConfig));
conf.WriteXml(fos);
fos.Close();
}
if (writeDetails != null)
{
IDictionary <string, object> map = new SortedDictionary <string, object>();
if (dfs != null)
{
map["namenode_port"] = dfs.GetNameNodePort();
}
if (mr != null)
{
map["resourcemanager_port"] = mr.GetConfig().Get(YarnConfiguration.RmAddress).Split
(":")[1];
}
FileWriter fw = new FileWriter(new FilePath(writeDetails));
fw.Write(new JSON().ToJSON(map));
fw.Close();
}
}
public static void SetUp()
{
Runtime.SetProperty("hadoop.log.dir", "logs");
Configuration conf = new Configuration();
dfsCluster = new MiniDFSCluster.Builder(conf).NumDataNodes(numSlaves).Build();
jConf = new JobConf(conf);
FileSystem.SetDefaultUri(conf, dfsCluster.GetFileSystem().GetUri().ToString());
mrCluster = MiniMRClientClusterFactory.Create(typeof(TestMRCredentials), 1, jConf
);
CreateKeysAsJson("keys.json");
}
public static void TearDown()
{
if (mrCluster != null)
{
mrCluster.Stop();
}
mrCluster = null;
if (dfsCluster != null)
{
dfsCluster.Shutdown();
}
dfsCluster = null;
new FilePath("keys.json").Delete();
}
/// <exception cref="System.IO.IOException"/>
public MiniMRCluster(int jobTrackerPort, int taskTrackerPort, int numTaskTrackers
, string namenode, int numDir, string[] racks, string[] hosts, UserGroupInformation
ugi, JobConf conf, int numTrackerToExclude, Clock clock)
{
if (conf == null)
{
conf = new JobConf();
}
FileSystem.SetDefaultUri(conf, namenode);
string identifier = this.GetType().Name + "_" + Sharpen.Extensions.ToString(new Random
().Next(int.MaxValue));
mrClientCluster = MiniMRClientClusterFactory.Create(this.GetType(), identifier, numTaskTrackers
, conf);
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
/// <exception cref="System.TypeLoadException"/>
public virtual void TestGetJobStatus()
{
MiniMRClientCluster mr = null;
FileSystem fileSys = null;
try
{
mr = CreateMiniClusterWithCapacityScheduler();
JobConf job = new JobConf(mr.GetConfig());
fileSys = FileSystem.Get(job);
fileSys.Delete(testDir, true);
FSDataOutputStream @out = fileSys.Create(inFile, true);
@out.WriteBytes("This is a test file");
@out.Close();
FileInputFormat.SetInputPaths(job, inFile);
FileOutputFormat.SetOutputPath(job, outDir);
job.SetInputFormat(typeof(TextInputFormat));
job.SetOutputFormat(typeof(TextOutputFormat));
job.SetMapperClass(typeof(IdentityMapper));
job.SetReducerClass(typeof(IdentityReducer));
job.SetNumReduceTasks(0);
JobClient client = new JobClient(mr.GetConfig());
RunningJob rj = client.SubmitJob(job);
JobID jobId = rj.GetID();
// The following asserts read JobStatus twice and ensure the returned
// JobStatus objects correspond to the same Job.
NUnit.Framework.Assert.AreEqual("Expected matching JobIDs", jobId, ((JobID)client
.GetJob(jobId).GetJobStatus().GetJobID()));
NUnit.Framework.Assert.AreEqual("Expected matching startTimes", rj.GetJobStatus()
.GetStartTime(), client.GetJob(jobId).GetJobStatus().GetStartTime());
}
finally
{
if (fileSys != null)
{
fileSys.Delete(testDir, true);
}
if (mr != null)
{
mr.Stop();
}
}
}
/// <summary>test JobConf</summary>
/// <exception cref="System.Exception"/>
public virtual void TestNetworkedJob()
{
// mock creation
MiniMRClientCluster mr = null;
FileSystem fileSys = null;
try
{
mr = CreateMiniClusterWithCapacityScheduler();
JobConf job = new JobConf(mr.GetConfig());
fileSys = FileSystem.Get(job);
fileSys.Delete(testDir, true);
FSDataOutputStream @out = fileSys.Create(inFile, true);
@out.WriteBytes("This is a test file");
@out.Close();
FileInputFormat.SetInputPaths(job, inFile);
FileOutputFormat.SetOutputPath(job, outDir);
job.SetInputFormat(typeof(TextInputFormat));
job.SetOutputFormat(typeof(TextOutputFormat));
job.SetMapperClass(typeof(IdentityMapper));
job.SetReducerClass(typeof(IdentityReducer));
job.SetNumReduceTasks(0);
JobClient client = new JobClient(mr.GetConfig());
RunningJob rj = client.SubmitJob(job);
JobID jobId = rj.GetID();
JobClient.NetworkedJob runningJob = (JobClient.NetworkedJob)client.GetJob(jobId);
runningJob.SetJobPriority(JobPriority.High.ToString());
// test getters
NUnit.Framework.Assert.IsTrue(runningJob.GetConfiguration().ToString().EndsWith("0001/job.xml"
));
NUnit.Framework.Assert.AreEqual(runningJob.GetID(), jobId);
NUnit.Framework.Assert.AreEqual(runningJob.GetJobID(), jobId.ToString());
NUnit.Framework.Assert.AreEqual(runningJob.GetJobName(), "N/A");
NUnit.Framework.Assert.IsTrue(runningJob.GetJobFile().EndsWith(".staging/" + runningJob
.GetJobID() + "/job.xml"));
NUnit.Framework.Assert.IsTrue(runningJob.GetTrackingURL().Length > 0);
NUnit.Framework.Assert.IsTrue(runningJob.MapProgress() == 0.0f);
NUnit.Framework.Assert.IsTrue(runningJob.ReduceProgress() == 0.0f);
NUnit.Framework.Assert.IsTrue(runningJob.CleanupProgress() == 0.0f);
NUnit.Framework.Assert.IsTrue(runningJob.SetupProgress() == 0.0f);
TaskCompletionEvent[] tce = runningJob.GetTaskCompletionEvents(0);
NUnit.Framework.Assert.AreEqual(tce.Length, 0);
NUnit.Framework.Assert.AreEqual(runningJob.GetHistoryUrl(), string.Empty);
NUnit.Framework.Assert.IsFalse(runningJob.IsRetired());
NUnit.Framework.Assert.AreEqual(runningJob.GetFailureInfo(), string.Empty);
NUnit.Framework.Assert.AreEqual(runningJob.GetJobStatus().GetJobName(), "N/A");
NUnit.Framework.Assert.AreEqual(client.GetMapTaskReports(jobId).Length, 0);
try
{
client.GetSetupTaskReports(jobId);
}
catch (YarnRuntimeException e)
{
NUnit.Framework.Assert.AreEqual(e.Message, "Unrecognized task type: JOB_SETUP");
}
try
{
client.GetCleanupTaskReports(jobId);
}
catch (YarnRuntimeException e)
{
NUnit.Framework.Assert.AreEqual(e.Message, "Unrecognized task type: JOB_CLEANUP");
}
NUnit.Framework.Assert.AreEqual(client.GetReduceTaskReports(jobId).Length, 0);
// test ClusterStatus
ClusterStatus status = client.GetClusterStatus(true);
NUnit.Framework.Assert.AreEqual(status.GetActiveTrackerNames().Count, 2);
// it method does not implemented and always return empty array or null;
NUnit.Framework.Assert.AreEqual(status.GetBlacklistedTrackers(), 0);
NUnit.Framework.Assert.AreEqual(status.GetBlacklistedTrackerNames().Count, 0);
NUnit.Framework.Assert.AreEqual(status.GetBlackListedTrackersInfo().Count, 0);
NUnit.Framework.Assert.AreEqual(status.GetJobTrackerStatus(), Cluster.JobTrackerStatus
.Running);
NUnit.Framework.Assert.AreEqual(status.GetMapTasks(), 1);
NUnit.Framework.Assert.AreEqual(status.GetMaxMapTasks(), 20);
NUnit.Framework.Assert.AreEqual(status.GetMaxReduceTasks(), 4);
NUnit.Framework.Assert.AreEqual(status.GetNumExcludedNodes(), 0);
NUnit.Framework.Assert.AreEqual(status.GetReduceTasks(), 1);
NUnit.Framework.Assert.AreEqual(status.GetTaskTrackers(), 2);
NUnit.Framework.Assert.AreEqual(status.GetTTExpiryInterval(), 0);
NUnit.Framework.Assert.AreEqual(status.GetJobTrackerStatus(), Cluster.JobTrackerStatus
.Running);
NUnit.Framework.Assert.AreEqual(status.GetGraylistedTrackers(), 0);
// test read and write
ByteArrayOutputStream dataOut = new ByteArrayOutputStream();
status.Write(new DataOutputStream(dataOut));
ClusterStatus status2 = new ClusterStatus();
status2.ReadFields(new DataInputStream(new ByteArrayInputStream(dataOut.ToByteArray
())));
NUnit.Framework.Assert.AreEqual(status.GetActiveTrackerNames(), status2.GetActiveTrackerNames
());
NUnit.Framework.Assert.AreEqual(status.GetBlackListedTrackersInfo(), status2.GetBlackListedTrackersInfo
());
NUnit.Framework.Assert.AreEqual(status.GetMapTasks(), status2.GetMapTasks());
try
{
}
catch (RuntimeException e)
{
NUnit.Framework.Assert.IsTrue(e.Message.EndsWith("not found on CLASSPATH"));
}
// test taskStatusfilter
JobClient.SetTaskOutputFilter(job, JobClient.TaskStatusFilter.All);
NUnit.Framework.Assert.AreEqual(JobClient.GetTaskOutputFilter(job), JobClient.TaskStatusFilter
.All);
// runningJob.setJobPriority(JobPriority.HIGH.name());
// test default map
NUnit.Framework.Assert.AreEqual(client.GetDefaultMaps(), 20);
NUnit.Framework.Assert.AreEqual(client.GetDefaultReduces(), 4);
NUnit.Framework.Assert.AreEqual(client.GetSystemDir().GetName(), "jobSubmitDir");
// test queue information
JobQueueInfo[] rootQueueInfo = client.GetRootQueues();
NUnit.Framework.Assert.AreEqual(rootQueueInfo.Length, 1);
NUnit.Framework.Assert.AreEqual(rootQueueInfo[0].GetQueueName(), "default");
JobQueueInfo[] qinfo = client.GetQueues();
NUnit.Framework.Assert.AreEqual(qinfo.Length, 1);
NUnit.Framework.Assert.AreEqual(qinfo[0].GetQueueName(), "default");
NUnit.Framework.Assert.AreEqual(client.GetChildQueues("default").Length, 0);
NUnit.Framework.Assert.AreEqual(client.GetJobsFromQueue("default").Length, 1);
NUnit.Framework.Assert.IsTrue(client.GetJobsFromQueue("default")[0].GetJobFile().
EndsWith("/job.xml"));
JobQueueInfo qi = client.GetQueueInfo("default");
NUnit.Framework.Assert.AreEqual(qi.GetQueueName(), "default");
NUnit.Framework.Assert.AreEqual(qi.GetQueueState(), "running");
QueueAclsInfo[] aai = client.GetQueueAclsForCurrentUser();
NUnit.Framework.Assert.AreEqual(aai.Length, 2);
NUnit.Framework.Assert.AreEqual(aai[0].GetQueueName(), "root");
NUnit.Framework.Assert.AreEqual(aai[1].GetQueueName(), "default");
// test token
Org.Apache.Hadoop.Security.Token.Token <DelegationTokenIdentifier> token = client.
GetDelegationToken(new Text(UserGroupInformation.GetCurrentUser().GetShortUserName
()));
NUnit.Framework.Assert.AreEqual(token.GetKind().ToString(), "RM_DELEGATION_TOKEN"
);
// test JobClient
// The following asserts read JobStatus twice and ensure the returned
// JobStatus objects correspond to the same Job.
NUnit.Framework.Assert.AreEqual("Expected matching JobIDs", jobId, ((JobID)client
.GetJob(jobId).GetJobStatus().GetJobID()));
NUnit.Framework.Assert.AreEqual("Expected matching startTimes", rj.GetJobStatus()
.GetStartTime(), client.GetJob(jobId).GetJobStatus().GetStartTime());
}
finally
{
if (fileSys != null)
{
fileSys.Delete(testDir, true);
}
if (mr != null)
{
mr.Stop();
}
}
}
/// <summary>test run from command line JobQueueClient</summary>
/// <exception cref="System.Exception"/>
public virtual void TestJobQueueClient()
{
MiniMRClientCluster mr = null;
FileSystem fileSys = null;
TextWriter oldOut = System.Console.Out;
try
{
mr = CreateMiniClusterWithCapacityScheduler();
JobConf job = new JobConf(mr.GetConfig());
fileSys = FileSystem.Get(job);
fileSys.Delete(testDir, true);
FSDataOutputStream @out = fileSys.Create(inFile, true);
@out.WriteBytes("This is a test file");
@out.Close();
FileInputFormat.SetInputPaths(job, inFile);
FileOutputFormat.SetOutputPath(job, outDir);
job.SetInputFormat(typeof(TextInputFormat));
job.SetOutputFormat(typeof(TextOutputFormat));
job.SetMapperClass(typeof(IdentityMapper));
job.SetReducerClass(typeof(IdentityReducer));
job.SetNumReduceTasks(0);
JobClient client = new JobClient(mr.GetConfig());
client.SubmitJob(job);
JobQueueClient jobClient = new JobQueueClient(job);
ByteArrayOutputStream bytes = new ByteArrayOutputStream();
Runtime.SetOut(new TextWriter(bytes));
string[] arg = new string[] { "-list" };
jobClient.Run(arg);
NUnit.Framework.Assert.IsTrue(bytes.ToString().Contains("Queue Name : default"));
NUnit.Framework.Assert.IsTrue(bytes.ToString().Contains("Queue State : running"));
bytes = new ByteArrayOutputStream();
Runtime.SetOut(new TextWriter(bytes));
string[] arg1 = new string[] { "-showacls" };
jobClient.Run(arg1);
NUnit.Framework.Assert.IsTrue(bytes.ToString().Contains("Queue acls for user :"******"root ADMINISTER_QUEUE,SUBMIT_APPLICATIONS"
));
NUnit.Framework.Assert.IsTrue(bytes.ToString().Contains("default ADMINISTER_QUEUE,SUBMIT_APPLICATIONS"
));
// test for info and default queue
bytes = new ByteArrayOutputStream();
Runtime.SetOut(new TextWriter(bytes));
string[] arg2 = new string[] { "-info", "default" };
jobClient.Run(arg2);
NUnit.Framework.Assert.IsTrue(bytes.ToString().Contains("Queue Name : default"));
NUnit.Framework.Assert.IsTrue(bytes.ToString().Contains("Queue State : running"));
NUnit.Framework.Assert.IsTrue(bytes.ToString().Contains("Scheduling Info"));
// test for info , default queue and jobs
bytes = new ByteArrayOutputStream();
Runtime.SetOut(new TextWriter(bytes));
string[] arg3 = new string[] { "-info", "default", "-showJobs" };
jobClient.Run(arg3);
NUnit.Framework.Assert.IsTrue(bytes.ToString().Contains("Queue Name : default"));
NUnit.Framework.Assert.IsTrue(bytes.ToString().Contains("Queue State : running"));
NUnit.Framework.Assert.IsTrue(bytes.ToString().Contains("Scheduling Info"));
NUnit.Framework.Assert.IsTrue(bytes.ToString().Contains("job_1"));
string[] arg4 = new string[] { };
jobClient.Run(arg4);
}
finally
{
Runtime.SetOut(oldOut);
if (fileSys != null)
{
fileSys.Delete(testDir, true);
}
if (mr != null)
{
mr.Stop();
}
}
}
