/// <exception cref="System.IO.IOException"/>
public static MiniMRClientCluster Create(Type caller, string identifier, int noOfNMs
, Configuration conf)
{
if (conf == null)
{
conf = new Configuration();
}
FileSystem fs = FileSystem.Get(conf);
Path testRootDir = new Path("target", identifier + "-tmpDir").MakeQualified(fs);
Path appJar = new Path(testRootDir, "MRAppJar.jar");
// Copy MRAppJar and make it private.
Path appMasterJar = new Path(MiniMRYarnCluster.Appjar);
fs.CopyFromLocalFile(appMasterJar, appJar);
fs.SetPermission(appJar, new FsPermission("744"));
Job job = Job.GetInstance(conf);
job.AddFileToClassPath(appJar);
Path callerJar = new Path(JarFinder.GetJar(caller));
Path remoteCallerJar = new Path(testRootDir, callerJar.GetName());
fs.CopyFromLocalFile(callerJar, remoteCallerJar);
fs.SetPermission(remoteCallerJar, new FsPermission("744"));
job.AddFileToClassPath(remoteCallerJar);
MiniMRYarnCluster miniMRYarnCluster = new MiniMRYarnCluster(identifier, noOfNMs);
job.GetConfiguration().Set("minimrclientcluster.caller.name", identifier);
job.GetConfiguration().SetInt("minimrclientcluster.nodemanagers.number", noOfNMs);
miniMRYarnCluster.Init(job.GetConfiguration());
miniMRYarnCluster.Start();
return(new MiniMRYarnClusterAdapter(miniMRYarnCluster));
}
public static void TearDown()
{
// close file system and shut down dfs and mapred cluster
try
{
if (fileSys != null)
{
fileSys.Close();
}
if (dfs != null)
{
dfs.Shutdown();
}
if (mr != null)
{
mr.Stop();
mr = null;
}
}
catch (IOException ioe)
{
Log.Info("IO exception in closing file system)");
Sharpen.Runtime.PrintStackTrace(ioe);
}
}
private MiniMRYarnCluster InitAndStartMiniMRYarnCluster(JobConf jobConf)
{
MiniMRYarnCluster miniMRYarnCluster;
miniMRYarnCluster = new MiniMRYarnCluster(this.GetType().FullName, 1);
miniMRYarnCluster.Init(jobConf);
miniMRYarnCluster.Start();
return(miniMRYarnCluster);
}
public virtual void TestSocketFactory()
{
// Create a standard mini-cluster
Configuration sconf = new Configuration();
MiniDFSCluster cluster = new MiniDFSCluster.Builder(sconf).NumDataNodes(1).Build(
);
int nameNodePort = cluster.GetNameNodePort();
// Get a reference to its DFS directly
FileSystem fs = cluster.GetFileSystem();
NUnit.Framework.Assert.IsTrue(fs is DistributedFileSystem);
DistributedFileSystem directDfs = (DistributedFileSystem)fs;
Configuration cconf = GetCustomSocketConfigs(nameNodePort);
fs = FileSystem.Get(cconf);
NUnit.Framework.Assert.IsTrue(fs is DistributedFileSystem);
DistributedFileSystem dfs = (DistributedFileSystem)fs;
JobClient client = null;
MiniMRYarnCluster miniMRYarnCluster = null;
try
{
// This will test RPC to the NameNode only.
// could we test Client-DataNode connections?
Path filePath = new Path("/dir");
NUnit.Framework.Assert.IsFalse(directDfs.Exists(filePath));
NUnit.Framework.Assert.IsFalse(dfs.Exists(filePath));
directDfs.Mkdirs(filePath);
NUnit.Framework.Assert.IsTrue(directDfs.Exists(filePath));
NUnit.Framework.Assert.IsTrue(dfs.Exists(filePath));
// This will test RPC to a Resource Manager
fs = FileSystem.Get(sconf);
JobConf jobConf = new JobConf();
FileSystem.SetDefaultUri(jobConf, fs.GetUri().ToString());
miniMRYarnCluster = InitAndStartMiniMRYarnCluster(jobConf);
JobConf jconf = new JobConf(miniMRYarnCluster.GetConfig());
jconf.Set("hadoop.rpc.socket.factory.class.default", "org.apache.hadoop.ipc.DummySocketFactory"
);
jconf.Set(MRConfig.FrameworkName, MRConfig.YarnFrameworkName);
string rmAddress = jconf.Get("yarn.resourcemanager.address");
string[] split = rmAddress.Split(":");
jconf.Set("yarn.resourcemanager.address", split[0] + ':' + (System.Convert.ToInt32
(split[1]) + 10));
client = new JobClient(jconf);
JobStatus[] jobs = client.JobsToComplete();
NUnit.Framework.Assert.IsTrue(jobs.Length == 0);
}
finally
{
CloseClient(client);
CloseDfs(dfs);
CloseDfs(directDfs);
StopMiniMRYarnCluster(miniMRYarnCluster);
ShutdownDFSCluster(cluster);
}
}
public virtual void TestMRTimelineEventHandling()
{
Configuration conf = new YarnConfiguration();
conf.SetBoolean(YarnConfiguration.TimelineServiceEnabled, true);
conf.SetBoolean(MRJobConfig.MapreduceJobEmitTimelineData, true);
MiniMRYarnCluster cluster = null;
try
{
cluster = new MiniMRYarnCluster(typeof(TestJobHistoryEventHandler).Name, 1);
cluster.Init(conf);
cluster.Start();
conf.Set(YarnConfiguration.TimelineServiceWebappAddress, MiniYARNCluster.GetHostname
() + ":" + cluster.GetApplicationHistoryServer().GetPort());
TimelineStore ts = cluster.GetApplicationHistoryServer().GetTimelineStore();
Path inDir = new Path("input");
Path outDir = new Path("output");
RunningJob job = UtilsForTests.RunJobSucceed(new JobConf(conf), inDir, outDir);
NUnit.Framework.Assert.AreEqual(JobStatus.Succeeded, job.GetJobStatus().GetState(
).GetValue());
TimelineEntities entities = ts.GetEntities("MAPREDUCE_JOB", null, null, null, null
, null, null, null, null, null);
NUnit.Framework.Assert.AreEqual(1, entities.GetEntities().Count);
TimelineEntity tEntity = entities.GetEntities()[0];
NUnit.Framework.Assert.AreEqual(job.GetID().ToString(), tEntity.GetEntityId());
NUnit.Framework.Assert.AreEqual("MAPREDUCE_JOB", tEntity.GetEntityType());
NUnit.Framework.Assert.AreEqual(EventType.AmStarted.ToString(), tEntity.GetEvents
()[tEntity.GetEvents().Count - 1].GetEventType());
NUnit.Framework.Assert.AreEqual(EventType.JobFinished.ToString(), tEntity.GetEvents
()[0].GetEventType());
job = UtilsForTests.RunJobFail(new JobConf(conf), inDir, outDir);
NUnit.Framework.Assert.AreEqual(JobStatus.Failed, job.GetJobStatus().GetState().GetValue
());
entities = ts.GetEntities("MAPREDUCE_JOB", null, null, null, null, null, null, null
, null, null);
NUnit.Framework.Assert.AreEqual(2, entities.GetEntities().Count);
tEntity = entities.GetEntities()[0];
NUnit.Framework.Assert.AreEqual(job.GetID().ToString(), tEntity.GetEntityId());
NUnit.Framework.Assert.AreEqual("MAPREDUCE_JOB", tEntity.GetEntityType());
NUnit.Framework.Assert.AreEqual(EventType.AmStarted.ToString(), tEntity.GetEvents
()[tEntity.GetEvents().Count - 1].GetEventType());
NUnit.Framework.Assert.AreEqual(EventType.JobFailed.ToString(), tEntity.GetEvents
()[0].GetEventType());
}
finally
{
if (cluster != null)
{
cluster.Stop();
}
}
}
public static void TearDown()
{
if (mrCluster != null)
{
mrCluster.Stop();
mrCluster = null;
}
if (dfsCluster != null)
{
dfsCluster.Shutdown();
dfsCluster = null;
}
}
private void StopMiniMRYarnCluster(MiniMRYarnCluster miniMRYarnCluster)
{
try
{
if (miniMRYarnCluster != null)
{
miniMRYarnCluster.Stop();
}
}
catch (Exception ignored)
{
// nothing we can do
Sharpen.Runtime.PrintStackTrace(ignored);
}
}
public virtual void TestTimelineServiceStartInMiniCluster()
{
Configuration conf = new YarnConfiguration();
/*
* Timeline service should not start if the config is set to false
* Regardless to the value of MAPREDUCE_JOB_EMIT_TIMELINE_DATA
*/
conf.SetBoolean(YarnConfiguration.TimelineServiceEnabled, false);
conf.SetBoolean(MRJobConfig.MapreduceJobEmitTimelineData, true);
MiniMRYarnCluster cluster = null;
try
{
cluster = new MiniMRYarnCluster(typeof(TestJobHistoryEventHandler).Name, 1);
cluster.Init(conf);
cluster.Start();
//verify that the timeline service is not started.
NUnit.Framework.Assert.IsNull("Timeline Service should not have been started", cluster
.GetApplicationHistoryServer());
}
finally
{
if (cluster != null)
{
cluster.Stop();
}
}
conf.SetBoolean(YarnConfiguration.TimelineServiceEnabled, false);
conf.SetBoolean(MRJobConfig.MapreduceJobEmitTimelineData, false);
cluster = null;
try
{
cluster = new MiniMRYarnCluster(typeof(TestJobHistoryEventHandler).Name, 1);
cluster.Init(conf);
cluster.Start();
//verify that the timeline service is not started.
NUnit.Framework.Assert.IsNull("Timeline Service should not have been started", cluster
.GetApplicationHistoryServer());
}
finally
{
if (cluster != null)
{
cluster.Stop();
}
}
}
public virtual void Restart()
{
if (!miniMRYarnCluster.GetServiceState().Equals(Service.STATE.Started))
{
Log.Warn("Cannot restart the mini cluster, start it first");
return;
}
Configuration oldConf = new Configuration(GetConfig());
string callerName = oldConf.Get("minimrclientcluster.caller.name", this.GetType()
.FullName);
int noOfNMs = oldConf.GetInt("minimrclientcluster.nodemanagers.number", 1);
oldConf.SetBoolean(YarnConfiguration.YarnMiniclusterFixedPorts, true);
oldConf.SetBoolean(JHAdminConfig.MrHistoryMiniclusterFixedPorts, true);
Stop();
miniMRYarnCluster = new MiniMRYarnCluster(callerName, noOfNMs);
miniMRYarnCluster.Init(oldConf);
miniMRYarnCluster.Start();
}
public static void SetUp()
{
Configuration conf = new Configuration();
conf.Set(MRConfig.FrameworkName, MRConfig.YarnFrameworkName);
conf.Set(YarnConfiguration.RmPrincipal, "jt_id/" + SecurityUtil.HostnamePattern +
"@APACHE.ORG");
MiniDFSCluster.Builder builder = new MiniDFSCluster.Builder(conf);
builder.CheckExitOnShutdown(true);
builder.NumDataNodes(numSlaves);
builder.Format(true);
builder.Racks(null);
dfsCluster = builder.Build();
mrCluster = new MiniMRYarnCluster(typeof(TestBinaryTokenFile).FullName, noOfNMs);
mrCluster.Init(conf);
mrCluster.Start();
NameNodeAdapter.GetDtSecretManager(dfsCluster.GetNamesystem()).StartThreads();
FileSystem fs = dfsCluster.GetFileSystem();
p1 = new Path("file1");
p1 = fs.MakeQualified(p1);
}
public static void Setup()
{
// create configuration, dfs, file system and mapred cluster
dfs = new MiniDFSCluster.Builder(conf).Build();
fileSys = dfs.GetFileSystem();
if (!(new FilePath(MiniMRYarnCluster.Appjar)).Exists())
{
Log.Info("MRAppJar " + MiniMRYarnCluster.Appjar + " not found. Not running test."
);
return;
}
if (mr == null)
{
mr = new MiniMRYarnCluster(typeof(TestMiniMRChildTask).FullName);
Configuration conf = new Configuration();
mr.Init(conf);
mr.Start();
}
// Copy MRAppJar and make it private. TODO: FIXME. This is a hack to
// workaround the absent public discache.
localFs.CopyFromLocalFile(new Path(MiniMRYarnCluster.Appjar), AppJar);
localFs.SetPermission(AppJar, new FsPermission("700"));
}
public MiniMRYarnClusterAdapter(MiniMRYarnCluster miniMRYarnCluster)
{
this.miniMRYarnCluster = miniMRYarnCluster;
}
