/// <exception cref="System.IO.IOException"/>
public static MiniMRClientCluster Create(Type caller, string identifier, int noOfNMs
, Configuration conf)
{
if (conf == null)
{
conf = new Configuration();
}
FileSystem fs = FileSystem.Get(conf);
Path testRootDir = new Path("target", identifier + "-tmpDir").MakeQualified(fs);
Path appJar = new Path(testRootDir, "MRAppJar.jar");
// Copy MRAppJar and make it private.
Path appMasterJar = new Path(MiniMRYarnCluster.Appjar);
fs.CopyFromLocalFile(appMasterJar, appJar);
fs.SetPermission(appJar, new FsPermission("744"));
Job job = Job.GetInstance(conf);
job.AddFileToClassPath(appJar);
Path callerJar = new Path(JarFinder.GetJar(caller));
Path remoteCallerJar = new Path(testRootDir, callerJar.GetName());
fs.CopyFromLocalFile(callerJar, remoteCallerJar);
fs.SetPermission(remoteCallerJar, new FsPermission("744"));
job.AddFileToClassPath(remoteCallerJar);
MiniMRYarnCluster miniMRYarnCluster = new MiniMRYarnCluster(identifier, noOfNMs);
job.GetConfiguration().Set("minimrclientcluster.caller.name", identifier);
job.GetConfiguration().SetInt("minimrclientcluster.nodemanagers.number", noOfNMs);
miniMRYarnCluster.Init(job.GetConfiguration());
miniMRYarnCluster.Start();
return(new MiniMRYarnClusterAdapter(miniMRYarnCluster));
}
private MiniMRYarnCluster InitAndStartMiniMRYarnCluster(JobConf jobConf)
{
MiniMRYarnCluster miniMRYarnCluster;
miniMRYarnCluster = new MiniMRYarnCluster(this.GetType().FullName, 1);
miniMRYarnCluster.Init(jobConf);
miniMRYarnCluster.Start();
return(miniMRYarnCluster);
}
public virtual void TestMRTimelineEventHandling()
{
Configuration conf = new YarnConfiguration();
conf.SetBoolean(YarnConfiguration.TimelineServiceEnabled, true);
conf.SetBoolean(MRJobConfig.MapreduceJobEmitTimelineData, true);
MiniMRYarnCluster cluster = null;
try
{
cluster = new MiniMRYarnCluster(typeof(TestJobHistoryEventHandler).Name, 1);
cluster.Init(conf);
cluster.Start();
conf.Set(YarnConfiguration.TimelineServiceWebappAddress, MiniYARNCluster.GetHostname
() + ":" + cluster.GetApplicationHistoryServer().GetPort());
TimelineStore ts = cluster.GetApplicationHistoryServer().GetTimelineStore();
Path inDir = new Path("input");
Path outDir = new Path("output");
RunningJob job = UtilsForTests.RunJobSucceed(new JobConf(conf), inDir, outDir);
NUnit.Framework.Assert.AreEqual(JobStatus.Succeeded, job.GetJobStatus().GetState(
).GetValue());
TimelineEntities entities = ts.GetEntities("MAPREDUCE_JOB", null, null, null, null
, null, null, null, null, null);
NUnit.Framework.Assert.AreEqual(1, entities.GetEntities().Count);
TimelineEntity tEntity = entities.GetEntities()[0];
NUnit.Framework.Assert.AreEqual(job.GetID().ToString(), tEntity.GetEntityId());
NUnit.Framework.Assert.AreEqual("MAPREDUCE_JOB", tEntity.GetEntityType());
NUnit.Framework.Assert.AreEqual(EventType.AmStarted.ToString(), tEntity.GetEvents
()[tEntity.GetEvents().Count - 1].GetEventType());
NUnit.Framework.Assert.AreEqual(EventType.JobFinished.ToString(), tEntity.GetEvents
()[0].GetEventType());
job = UtilsForTests.RunJobFail(new JobConf(conf), inDir, outDir);
NUnit.Framework.Assert.AreEqual(JobStatus.Failed, job.GetJobStatus().GetState().GetValue
());
entities = ts.GetEntities("MAPREDUCE_JOB", null, null, null, null, null, null, null
, null, null);
NUnit.Framework.Assert.AreEqual(2, entities.GetEntities().Count);
tEntity = entities.GetEntities()[0];
NUnit.Framework.Assert.AreEqual(job.GetID().ToString(), tEntity.GetEntityId());
NUnit.Framework.Assert.AreEqual("MAPREDUCE_JOB", tEntity.GetEntityType());
NUnit.Framework.Assert.AreEqual(EventType.AmStarted.ToString(), tEntity.GetEvents
()[tEntity.GetEvents().Count - 1].GetEventType());
NUnit.Framework.Assert.AreEqual(EventType.JobFailed.ToString(), tEntity.GetEvents
()[0].GetEventType());
}
finally
{
if (cluster != null)
{
cluster.Stop();
}
}
}
public virtual void TestTimelineServiceStartInMiniCluster()
{
Configuration conf = new YarnConfiguration();
/*
* Timeline service should not start if the config is set to false
* Regardless to the value of MAPREDUCE_JOB_EMIT_TIMELINE_DATA
*/
conf.SetBoolean(YarnConfiguration.TimelineServiceEnabled, false);
conf.SetBoolean(MRJobConfig.MapreduceJobEmitTimelineData, true);
MiniMRYarnCluster cluster = null;
try
{
cluster = new MiniMRYarnCluster(typeof(TestJobHistoryEventHandler).Name, 1);
cluster.Init(conf);
cluster.Start();
//verify that the timeline service is not started.
NUnit.Framework.Assert.IsNull("Timeline Service should not have been started", cluster
.GetApplicationHistoryServer());
}
finally
{
if (cluster != null)
{
cluster.Stop();
}
}
conf.SetBoolean(YarnConfiguration.TimelineServiceEnabled, false);
conf.SetBoolean(MRJobConfig.MapreduceJobEmitTimelineData, false);
cluster = null;
try
{
cluster = new MiniMRYarnCluster(typeof(TestJobHistoryEventHandler).Name, 1);
cluster.Init(conf);
cluster.Start();
//verify that the timeline service is not started.
NUnit.Framework.Assert.IsNull("Timeline Service should not have been started", cluster
.GetApplicationHistoryServer());
}
finally
{
if (cluster != null)
{
cluster.Stop();
}
}
}
public virtual void Restart()
{
if (!miniMRYarnCluster.GetServiceState().Equals(Service.STATE.Started))
{
Log.Warn("Cannot restart the mini cluster, start it first");
return;
}
Configuration oldConf = new Configuration(GetConfig());
string callerName = oldConf.Get("minimrclientcluster.caller.name", this.GetType()
.FullName);
int noOfNMs = oldConf.GetInt("minimrclientcluster.nodemanagers.number", 1);
oldConf.SetBoolean(YarnConfiguration.YarnMiniclusterFixedPorts, true);
oldConf.SetBoolean(JHAdminConfig.MrHistoryMiniclusterFixedPorts, true);
Stop();
miniMRYarnCluster = new MiniMRYarnCluster(callerName, noOfNMs);
miniMRYarnCluster.Init(oldConf);
miniMRYarnCluster.Start();
}
public static void SetUp()
{
Configuration conf = new Configuration();
conf.Set(MRConfig.FrameworkName, MRConfig.YarnFrameworkName);
conf.Set(YarnConfiguration.RmPrincipal, "jt_id/" + SecurityUtil.HostnamePattern +
"@APACHE.ORG");
MiniDFSCluster.Builder builder = new MiniDFSCluster.Builder(conf);
builder.CheckExitOnShutdown(true);
builder.NumDataNodes(numSlaves);
builder.Format(true);
builder.Racks(null);
dfsCluster = builder.Build();
mrCluster = new MiniMRYarnCluster(typeof(TestBinaryTokenFile).FullName, noOfNMs);
mrCluster.Init(conf);
mrCluster.Start();
NameNodeAdapter.GetDtSecretManager(dfsCluster.GetNamesystem()).StartThreads();
FileSystem fs = dfsCluster.GetFileSystem();
p1 = new Path("file1");
p1 = fs.MakeQualified(p1);
}
public static void Setup()
{
// create configuration, dfs, file system and mapred cluster
dfs = new MiniDFSCluster.Builder(conf).Build();
fileSys = dfs.GetFileSystem();
if (!(new FilePath(MiniMRYarnCluster.Appjar)).Exists())
{
Log.Info("MRAppJar " + MiniMRYarnCluster.Appjar + " not found. Not running test."
);
return;
}
if (mr == null)
{
mr = new MiniMRYarnCluster(typeof(TestMiniMRChildTask).FullName);
Configuration conf = new Configuration();
mr.Init(conf);
mr.Start();
}
// Copy MRAppJar and make it private. TODO: FIXME. This is a hack to
// workaround the absent public discache.
localFs.CopyFromLocalFile(new Path(MiniMRYarnCluster.Appjar), AppJar);
localFs.SetPermission(AppJar, new FsPermission("700"));
}
public virtual void Start()
{
miniMRYarnCluster.Start();
}