public static void Setup()
{
try
{
dfsCluster = new MiniDFSCluster.Builder(conf).NumDataNodes(2).Format(true).Racks(
null).Build();
remoteFs = dfsCluster.GetFileSystem();
}
catch (IOException io)
{
throw new RuntimeException("problem starting mini dfs cluster", io);
}
if (!(new FilePath(MiniMRYarnCluster.Appjar)).Exists())
{
Log.Info("MRAppJar " + MiniMRYarnCluster.Appjar + " not found. Not running test."
);
return;
}
if (mrCluster == null)
{
mrCluster = new MiniMRYarnCluster(typeof(TestMRJobs).FullName, NumNodeMgrs);
Configuration conf = new Configuration();
conf.Set("fs.defaultFS", remoteFs.GetUri().ToString());
// use HDFS
conf.Set(MRJobConfig.MrAmStagingDir, "/apps_staging_dir");
mrCluster.Init(conf);
mrCluster.Start();
}
// Copy MRAppJar and make it private. TODO: FIXME. This is a hack to
// workaround the absent public discache.
localFs.CopyFromLocalFile(new Path(MiniMRYarnCluster.Appjar), AppJar);
localFs.SetPermission(AppJar, new FsPermission("700"));
}
public static void TearDown()
{
if (mrCluster != null)
{
mrCluster.Stop();
mrCluster = null;
}
}
public static void TearDown()
{
if (mrCluster != null)
{
mrCluster.Stop();
mrCluster = null;
}
if (dfsCluster != null)
{
dfsCluster.Shutdown();
dfsCluster = null;
}
}
public static void Setup()
{
if (!(new FilePath(MiniMRYarnCluster.Appjar)).Exists())
{
Log.Info("MRAppJar " + MiniMRYarnCluster.Appjar + " not found. Not running test."
);
return;
}
if (mrCluster == null)
{
mrCluster = new MiniMRYarnCluster(typeof(TestMRJobsWithProfiler).FullName);
mrCluster.Init(Conf);
mrCluster.Start();
}
// Copy MRAppJar and make it private. TODO: FIXME. This is a hack to
// workaround the absent public discache.
localFs.CopyFromLocalFile(new Path(MiniMRYarnCluster.Appjar), AppJar);
localFs.SetPermission(AppJar, new FsPermission("700"));
}
public static void Setup()
{
if (!(new FilePath(MiniMRYarnCluster.Appjar)).Exists())
{
Log.Info("MRAppJar " + MiniMRYarnCluster.Appjar + " not found. Not running test."
);
return;
}
if (mrCluster == null)
{
mrCluster = new MiniMRYarnCluster(typeof(TestSpeculativeExecution).FullName, 4);
Configuration conf = new Configuration();
mrCluster.Init(conf);
mrCluster.Start();
}
// workaround the absent public distcache.
localFs.CopyFromLocalFile(new Path(MiniMRYarnCluster.Appjar), AppJar);
localFs.SetPermission(AppJar, new FsPermission("700"));
}
public static void Setup()
{
if (!(new FilePath(MiniMRYarnCluster.Appjar)).Exists())
{
Log.Info("MRAppJar " + MiniMRYarnCluster.Appjar + " not found. Not running test."
);
return;
}
if (mrCluster == null)
{
mrCluster = new MiniMRYarnCluster(typeof(Org.Apache.Hadoop.Mapreduce.V2.TestRMNMInfo
).FullName, Numnodemanagers);
Configuration conf = new Configuration();
mrCluster.Init(conf);
mrCluster.Start();
}
// workaround the absent public distcache.
localFs.CopyFromLocalFile(new Path(MiniMRYarnCluster.Appjar), AppJar);
localFs.SetPermission(AppJar, new FsPermission("700"));
}
public static void Setup()
{
if (!(new FilePath(MiniMRYarnCluster.Appjar)).Exists())
{
Log.Info("MRAppJar " + MiniMRYarnCluster.Appjar + " not found. Not running test."
);
return;
}
if (mrCluster == null)
{
mrCluster = new MiniMRYarnCluster(typeof(TestMROldApiJobs).FullName);
mrCluster.Init(new Configuration());
mrCluster.Start();
}
// TestMRJobs is for testing non-uberized operation only; see TestUberAM
// for corresponding uberized tests.
mrCluster.GetConfig().SetBoolean(MRJobConfig.JobUbertaskEnable, false);
// Copy MRAppJar and make it private. TODO: FIXME. This is a hack to
// workaround the absent public discache.
localFs.CopyFromLocalFile(new Path(MiniMRYarnCluster.Appjar), TestMRJobs.AppJar);
localFs.SetPermission(TestMRJobs.AppJar, new FsPermission("700"));
}
/// <exception cref="System.Exception"/>
protected override void ServiceStart()
{
lock (this)
{
try
{
if (!this.GetConfig().GetBoolean(JHAdminConfig.MrHistoryMiniclusterFixedPorts, JHAdminConfig
.DefaultMrHistoryMiniclusterFixedPorts))
{
string hostname = MiniYARNCluster.GetHostname();
// pick free random ports.
this.GetConfig().Set(JHAdminConfig.MrHistoryAddress, hostname + ":0");
MRWebAppUtil.SetJHSWebappURLWithoutScheme(this.GetConfig(), hostname + ":0");
this.GetConfig().Set(JHAdminConfig.JhsAdminAddress, hostname + ":0");
}
this._enclosing.historyServer = new JobHistoryServer();
this._enclosing.historyServer.Init(this.GetConfig());
new _Thread_212(this).Start();
while (!this.jhsStarted)
{
MiniMRYarnCluster.Log.Info("Waiting for HistoryServer to start...");
Sharpen.Thread.Sleep(1500);
}
//TODO Add a timeout. State.STOPPED check ?
if (this._enclosing.historyServer.GetServiceState() != Service.STATE.Started)
{
throw new IOException("HistoryServer failed to start");
}
base.ServiceStart();
}
catch (Exception t)
{
throw new YarnRuntimeException(t);
}
//need to do this because historyServer.init creates a new Configuration
this.GetConfig().Set(JHAdminConfig.MrHistoryAddress, this._enclosing.historyServer
.GetConfig().Get(JHAdminConfig.MrHistoryAddress));
MRWebAppUtil.SetJHSWebappURLWithoutScheme(this.GetConfig(), MRWebAppUtil.GetJHSWebappURLWithoutScheme
(this._enclosing.historyServer.GetConfig()));
MiniMRYarnCluster.Log.Info("MiniMRYARN ResourceManager address: " + this.GetConfig
().Get(YarnConfiguration.RmAddress));
MiniMRYarnCluster.Log.Info("MiniMRYARN ResourceManager web address: " + WebAppUtils
.GetRMWebAppURLWithoutScheme(this.GetConfig()));
MiniMRYarnCluster.Log.Info("MiniMRYARN HistoryServer address: " + this.GetConfig(
).Get(JHAdminConfig.MrHistoryAddress));
MiniMRYarnCluster.Log.Info("MiniMRYARN HistoryServer web address: " + MiniMRYarnCluster
.GetResolvedMRHistoryWebAppURLWithoutScheme(this.GetConfig(), MRWebAppUtil.GetJHSHttpPolicy
() == HttpConfig.Policy.HttpsOnly));
}
}
public JobHistoryServerWrapper(MiniMRYarnCluster _enclosing)
: base(typeof(MiniMRYarnCluster.JobHistoryServerWrapper).FullName)
{
this._enclosing = _enclosing;
}
