/// <exception cref="System.Exception"/>
private void StartCluster(Configuration conf)
{
if (Runtime.GetProperty("hadoop.log.dir") == null)
{
Runtime.SetProperty("hadoop.log.dir", "target/test-dir");
}
conf.Set("dfs.block.access.token.enable", "false");
conf.Set("dfs.permissions", "true");
conf.Set("hadoop.security.authentication", "simple");
string cp = conf.Get(YarnConfiguration.YarnApplicationClasspath, StringUtils.Join
(",", YarnConfiguration.DefaultYarnCrossPlatformApplicationClasspath)) + FilePath
.pathSeparator + classpathDir;
conf.Set(YarnConfiguration.YarnApplicationClasspath, cp);
dfsCluster = new MiniDFSCluster.Builder(conf).Build();
FileSystem fileSystem = dfsCluster.GetFileSystem();
fileSystem.Mkdirs(new Path("/tmp"));
fileSystem.Mkdirs(new Path("/user"));
fileSystem.Mkdirs(new Path("/hadoop/mapred/system"));
fileSystem.SetPermission(new Path("/tmp"), FsPermission.ValueOf("-rwxrwxrwx"));
fileSystem.SetPermission(new Path("/user"), FsPermission.ValueOf("-rwxrwxrwx"));
fileSystem.SetPermission(new Path("/hadoop/mapred/system"), FsPermission.ValueOf(
"-rwx------"));
FileSystem.SetDefaultUri(conf, fileSystem.GetUri());
mrCluster = MiniMRClientClusterFactory.Create(this.GetType(), 1, conf);
// so the minicluster conf is avail to the containers.
TextWriter writer = new FileWriter(classpathDir + "/core-site.xml");
mrCluster.GetConfig().WriteXml(writer);
writer.Close();
}
public virtual void Setup()
{
Configuration conf = new YarnConfiguration();
cluster = MiniMRClientClusterFactory.Create(this.GetType(), 2, conf);
cluster.Start();
}
// nothing
/// <summary>Starts DFS and MR clusters, as specified in member-variable options.</summary>
/// <remarks>
/// Starts DFS and MR clusters, as specified in member-variable options. Also
/// writes out configuration and details, if requested.
/// </remarks>
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.IO.FileNotFoundException"/>
/// <exception cref="Sharpen.URISyntaxException"/>
public virtual void Start()
{
if (!noDFS)
{
dfs = new MiniDFSCluster.Builder(conf).NameNodePort(nnPort).NumDataNodes(numDataNodes
).StartupOption(dfsOpts).Build();
Log.Info("Started MiniDFSCluster -- namenode on port " + dfs.GetNameNodePort());
}
if (!noMR)
{
if (fs == null && dfs != null)
{
fs = dfs.GetFileSystem().GetUri().ToString();
}
else
{
if (fs == null)
{
fs = "file:///tmp/minimr-" + Runtime.NanoTime();
}
}
FileSystem.SetDefaultUri(conf, new URI(fs));
// Instruct the minicluster to use fixed ports, so user will know which
// ports to use when communicating with the cluster.
conf.SetBoolean(YarnConfiguration.YarnMiniclusterFixedPorts, true);
conf.SetBoolean(JHAdminConfig.MrHistoryMiniclusterFixedPorts, true);
conf.Set(YarnConfiguration.RmAddress, MiniYARNCluster.GetHostname() + ":" + this.
rmPort);
conf.Set(JHAdminConfig.MrHistoryAddress, MiniYARNCluster.GetHostname() + ":" + this
.jhsPort);
mr = MiniMRClientClusterFactory.Create(this.GetType(), numNodeManagers, conf);
Log.Info("Started MiniMRCluster");
}
if (writeConfig != null)
{
FileOutputStream fos = new FileOutputStream(new FilePath(writeConfig));
conf.WriteXml(fos);
fos.Close();
}
if (writeDetails != null)
{
IDictionary <string, object> map = new SortedDictionary <string, object>();
if (dfs != null)
{
map["namenode_port"] = dfs.GetNameNodePort();
}
if (mr != null)
{
map["resourcemanager_port"] = mr.GetConfig().Get(YarnConfiguration.RmAddress).Split
(":")[1];
}
FileWriter fw = new FileWriter(new FilePath(writeDetails));
fw.Write(new JSON().ToJSON(map));
fw.Close();
}
}
public static void SetUp()
{
Runtime.SetProperty("hadoop.log.dir", "logs");
Configuration conf = new Configuration();
dfsCluster = new MiniDFSCluster.Builder(conf).NumDataNodes(numSlaves).Build();
jConf = new JobConf(conf);
FileSystem.SetDefaultUri(conf, dfsCluster.GetFileSystem().GetUri().ToString());
mrCluster = MiniMRClientClusterFactory.Create(typeof(TestMRCredentials), 1, jConf
);
CreateKeysAsJson("keys.json");
}