public static void Setup() { try { dfsCluster = new MiniDFSCluster.Builder(conf).NumDataNodes(2).Format(true).Racks( null).Build(); remoteFs = dfsCluster.GetFileSystem(); } catch (IOException io) { throw new RuntimeException("problem starting mini dfs cluster", io); } if (!(new FilePath(MiniMRYarnCluster.Appjar)).Exists()) { Log.Info("MRAppJar " + MiniMRYarnCluster.Appjar + " not found. Not running test." ); return; } if (mrCluster == null) { mrCluster = new MiniMRYarnCluster(typeof(TestMRJobs).FullName, NumNodeMgrs); Configuration conf = new Configuration(); conf.Set("fs.defaultFS", remoteFs.GetUri().ToString()); // use HDFS conf.Set(MRJobConfig.MrAmStagingDir, "/apps_staging_dir"); mrCluster.Init(conf); mrCluster.Start(); } // Copy MRAppJar and make it private. TODO: FIXME. This is a hack to // workaround the absent public discache. localFs.CopyFromLocalFile(new Path(MiniMRYarnCluster.Appjar), AppJar); localFs.SetPermission(AppJar, new FsPermission("700")); }
public static void TearDown() { if (mrCluster != null) { mrCluster.Stop(); mrCluster = null; } }
public static void TearDown() { if (mrCluster != null) { mrCluster.Stop(); mrCluster = null; } if (dfsCluster != null) { dfsCluster.Shutdown(); dfsCluster = null; } }
public static void Setup() { if (!(new FilePath(MiniMRYarnCluster.Appjar)).Exists()) { Log.Info("MRAppJar " + MiniMRYarnCluster.Appjar + " not found. Not running test." ); return; } if (mrCluster == null) { mrCluster = new MiniMRYarnCluster(typeof(TestMRJobsWithProfiler).FullName); mrCluster.Init(Conf); mrCluster.Start(); } // Copy MRAppJar and make it private. TODO: FIXME. This is a hack to // workaround the absent public discache. localFs.CopyFromLocalFile(new Path(MiniMRYarnCluster.Appjar), AppJar); localFs.SetPermission(AppJar, new FsPermission("700")); }
public static void Setup() { if (!(new FilePath(MiniMRYarnCluster.Appjar)).Exists()) { Log.Info("MRAppJar " + MiniMRYarnCluster.Appjar + " not found. Not running test." ); return; } if (mrCluster == null) { mrCluster = new MiniMRYarnCluster(typeof(TestSpeculativeExecution).FullName, 4); Configuration conf = new Configuration(); mrCluster.Init(conf); mrCluster.Start(); } // workaround the absent public distcache. localFs.CopyFromLocalFile(new Path(MiniMRYarnCluster.Appjar), AppJar); localFs.SetPermission(AppJar, new FsPermission("700")); }
public static void Setup() { if (!(new FilePath(MiniMRYarnCluster.Appjar)).Exists()) { Log.Info("MRAppJar " + MiniMRYarnCluster.Appjar + " not found. Not running test." ); return; } if (mrCluster == null) { mrCluster = new MiniMRYarnCluster(typeof(Org.Apache.Hadoop.Mapreduce.V2.TestRMNMInfo ).FullName, Numnodemanagers); Configuration conf = new Configuration(); mrCluster.Init(conf); mrCluster.Start(); } // workaround the absent public distcache. localFs.CopyFromLocalFile(new Path(MiniMRYarnCluster.Appjar), AppJar); localFs.SetPermission(AppJar, new FsPermission("700")); }
public static void Setup() { if (!(new FilePath(MiniMRYarnCluster.Appjar)).Exists()) { Log.Info("MRAppJar " + MiniMRYarnCluster.Appjar + " not found. Not running test." ); return; } if (mrCluster == null) { mrCluster = new MiniMRYarnCluster(typeof(TestMROldApiJobs).FullName); mrCluster.Init(new Configuration()); mrCluster.Start(); } // TestMRJobs is for testing non-uberized operation only; see TestUberAM // for corresponding uberized tests. mrCluster.GetConfig().SetBoolean(MRJobConfig.JobUbertaskEnable, false); // Copy MRAppJar and make it private. TODO: FIXME. This is a hack to // workaround the absent public discache. localFs.CopyFromLocalFile(new Path(MiniMRYarnCluster.Appjar), TestMRJobs.AppJar); localFs.SetPermission(TestMRJobs.AppJar, new FsPermission("700")); }
/// <exception cref="System.Exception"/> protected override void ServiceStart() { lock (this) { try { if (!this.GetConfig().GetBoolean(JHAdminConfig.MrHistoryMiniclusterFixedPorts, JHAdminConfig .DefaultMrHistoryMiniclusterFixedPorts)) { string hostname = MiniYARNCluster.GetHostname(); // pick free random ports. this.GetConfig().Set(JHAdminConfig.MrHistoryAddress, hostname + ":0"); MRWebAppUtil.SetJHSWebappURLWithoutScheme(this.GetConfig(), hostname + ":0"); this.GetConfig().Set(JHAdminConfig.JhsAdminAddress, hostname + ":0"); } this._enclosing.historyServer = new JobHistoryServer(); this._enclosing.historyServer.Init(this.GetConfig()); new _Thread_212(this).Start(); while (!this.jhsStarted) { MiniMRYarnCluster.Log.Info("Waiting for HistoryServer to start..."); Sharpen.Thread.Sleep(1500); } //TODO Add a timeout. State.STOPPED check ? if (this._enclosing.historyServer.GetServiceState() != Service.STATE.Started) { throw new IOException("HistoryServer failed to start"); } base.ServiceStart(); } catch (Exception t) { throw new YarnRuntimeException(t); } //need to do this because historyServer.init creates a new Configuration this.GetConfig().Set(JHAdminConfig.MrHistoryAddress, this._enclosing.historyServer .GetConfig().Get(JHAdminConfig.MrHistoryAddress)); MRWebAppUtil.SetJHSWebappURLWithoutScheme(this.GetConfig(), MRWebAppUtil.GetJHSWebappURLWithoutScheme (this._enclosing.historyServer.GetConfig())); MiniMRYarnCluster.Log.Info("MiniMRYARN ResourceManager address: " + this.GetConfig ().Get(YarnConfiguration.RmAddress)); MiniMRYarnCluster.Log.Info("MiniMRYARN ResourceManager web address: " + WebAppUtils .GetRMWebAppURLWithoutScheme(this.GetConfig())); MiniMRYarnCluster.Log.Info("MiniMRYARN HistoryServer address: " + this.GetConfig( ).Get(JHAdminConfig.MrHistoryAddress)); MiniMRYarnCluster.Log.Info("MiniMRYARN HistoryServer web address: " + MiniMRYarnCluster .GetResolvedMRHistoryWebAppURLWithoutScheme(this.GetConfig(), MRWebAppUtil.GetJHSHttpPolicy () == HttpConfig.Policy.HttpsOnly)); } }
public JobHistoryServerWrapper(MiniMRYarnCluster _enclosing) : base(typeof(MiniMRYarnCluster.JobHistoryServerWrapper).FullName) { this._enclosing = _enclosing; }