/// <exception cref="System.Exception"/> private void StartCluster(Configuration conf) { if (Runtime.GetProperty("hadoop.log.dir") == null) { Runtime.SetProperty("hadoop.log.dir", "target/test-dir"); } conf.Set("dfs.block.access.token.enable", "false"); conf.Set("dfs.permissions", "true"); conf.Set("hadoop.security.authentication", "simple"); string cp = conf.Get(YarnConfiguration.YarnApplicationClasspath, StringUtils.Join (",", YarnConfiguration.DefaultYarnCrossPlatformApplicationClasspath)) + FilePath .pathSeparator + classpathDir; conf.Set(YarnConfiguration.YarnApplicationClasspath, cp); dfsCluster = new MiniDFSCluster.Builder(conf).Build(); FileSystem fileSystem = dfsCluster.GetFileSystem(); fileSystem.Mkdirs(new Path("/tmp")); fileSystem.Mkdirs(new Path("/user")); fileSystem.Mkdirs(new Path("/hadoop/mapred/system")); fileSystem.SetPermission(new Path("/tmp"), FsPermission.ValueOf("-rwxrwxrwx")); fileSystem.SetPermission(new Path("/user"), FsPermission.ValueOf("-rwxrwxrwx")); fileSystem.SetPermission(new Path("/hadoop/mapred/system"), FsPermission.ValueOf( "-rwx------")); FileSystem.SetDefaultUri(conf, fileSystem.GetUri()); mrCluster = MiniMRClientClusterFactory.Create(this.GetType(), 1, conf); // so the minicluster conf is avail to the containers. TextWriter writer = new FileWriter(classpathDir + "/core-site.xml"); mrCluster.GetConfig().WriteXml(writer); writer.Close(); }
public virtual void Setup() { Configuration conf = new YarnConfiguration(); cluster = MiniMRClientClusterFactory.Create(this.GetType(), 2, conf); cluster.Start(); }
// nothing /// <summary>Starts DFS and MR clusters, as specified in member-variable options.</summary> /// <remarks> /// Starts DFS and MR clusters, as specified in member-variable options. Also /// writes out configuration and details, if requested. /// </remarks> /// <exception cref="System.IO.IOException"/> /// <exception cref="System.IO.FileNotFoundException"/> /// <exception cref="Sharpen.URISyntaxException"/> public virtual void Start() { if (!noDFS) { dfs = new MiniDFSCluster.Builder(conf).NameNodePort(nnPort).NumDataNodes(numDataNodes ).StartupOption(dfsOpts).Build(); Log.Info("Started MiniDFSCluster -- namenode on port " + dfs.GetNameNodePort()); } if (!noMR) { if (fs == null && dfs != null) { fs = dfs.GetFileSystem().GetUri().ToString(); } else { if (fs == null) { fs = "file:///tmp/minimr-" + Runtime.NanoTime(); } } FileSystem.SetDefaultUri(conf, new URI(fs)); // Instruct the minicluster to use fixed ports, so user will know which // ports to use when communicating with the cluster. conf.SetBoolean(YarnConfiguration.YarnMiniclusterFixedPorts, true); conf.SetBoolean(JHAdminConfig.MrHistoryMiniclusterFixedPorts, true); conf.Set(YarnConfiguration.RmAddress, MiniYARNCluster.GetHostname() + ":" + this. rmPort); conf.Set(JHAdminConfig.MrHistoryAddress, MiniYARNCluster.GetHostname() + ":" + this .jhsPort); mr = MiniMRClientClusterFactory.Create(this.GetType(), numNodeManagers, conf); Log.Info("Started MiniMRCluster"); } if (writeConfig != null) { FileOutputStream fos = new FileOutputStream(new FilePath(writeConfig)); conf.WriteXml(fos); fos.Close(); } if (writeDetails != null) { IDictionary <string, object> map = new SortedDictionary <string, object>(); if (dfs != null) { map["namenode_port"] = dfs.GetNameNodePort(); } if (mr != null) { map["resourcemanager_port"] = mr.GetConfig().Get(YarnConfiguration.RmAddress).Split (":")[1]; } FileWriter fw = new FileWriter(new FilePath(writeDetails)); fw.Write(new JSON().ToJSON(map)); fw.Close(); } }
public static void SetUp() { Runtime.SetProperty("hadoop.log.dir", "logs"); Configuration conf = new Configuration(); dfsCluster = new MiniDFSCluster.Builder(conf).NumDataNodes(numSlaves).Build(); jConf = new JobConf(conf); FileSystem.SetDefaultUri(conf, dfsCluster.GetFileSystem().GetUri().ToString()); mrCluster = MiniMRClientClusterFactory.Create(typeof(TestMRCredentials), 1, jConf ); CreateKeysAsJson("keys.json"); }