public ClusterClient Client(DryadLinqContext context)
{
if (_clusterClient == null)
{
_clusterClient = new NativeYarnClient(HeadNode, HdfsPort, LauncherPort);
}
return(_clusterClient);
}
/// <summary>
/// Create a cluster containing just the local machine.
/// </summary>
/// <param name="conf">Configuration for the local machine.</param>
public WebHdfsClusterStatus(ClusterConfiguration conf)
: base(conf)
{
if (!(conf is WebHdfsClusterConfiguration))
{
throw new ArgumentException("Expected a WebHdfsClusterConfiguration, got a " + conf.GetType());
}
this.config = conf as WebHdfsClusterConfiguration;
this.yarnClient = new NativeYarnClient(this.config.StatusNode, this.config.StatusNodePort, new HdfsClient(this.config.UserName));
}
/// <summary>
/// Make a new cluster object representing a YARN cluster with default ports
/// </summary>
/// <param name="headNode">The computer where the YarnLauncher is running</param>
public DryadLinqYarnCluster(string headNode)
{
HeadNode = headNode;
LauncherPort = 8471;
NameNode = headNode;
HdfsPort = 9000;
WebHdfsPort = 50070;
_dfsClient = null;
_clusterClient = null;
}
/// <summary>
/// Create a cluster containing just the local machine.
/// </summary>
/// <param name="conf">Configuration for the local machine.</param>
public HdfsClusterStatus(ClusterConfiguration conf)
: base(conf)
{
if (!(conf is HdfsClusterConfiguration))
{
throw new ArgumentException("Expected an HdfsClusterConfiguration, got a " + conf.GetType());
}
this.config = conf as HdfsClusterConfiguration;
// make a fake call to initialize the cluster on the foreground thread
// HDFS does not work if initialized on the background thread.
Uri uri = DfsFile.UriFromPath(this.config.JobsFolderUri, "");
this.config.DfsClient.IsFileExists(uri); // ignore result
this.yarnClient = new NativeYarnClient(this.config.StatusNode, this.config.StatusNodePort, new HdfsClient(this.config.UserName));
}
/// <summary>
/// Create a cluster containing just the local machine.
/// </summary>
/// <param name="conf">Configuration for the local machine.</param>
public HdfsClusterStatus(ClusterConfiguration conf)
: base(conf)
{
if (!(conf is HdfsClusterConfiguration))
throw new ArgumentException("Expected an HdfsClusterConfiguration, got a " + conf.GetType());
this.config = conf as HdfsClusterConfiguration;
// make a fake call to initialize the cluster on the foreground thread
// HDFS does not work if initialized on the background thread.
Uri uri = DfsFile.UriFromPath(this.config.JobsFolderUri, "");
this.config.DfsClient.IsFileExists(uri); // ignore result
this.yarnClient = new NativeYarnClient(this.config.StatusNode, this.config.StatusNodePort, new HdfsClient(this.config.UserName));
}
/// <summary>
/// Create a cluster containing just the local machine.
/// </summary>
/// <param name="conf">Configuration for the local machine.</param>
public WebHdfsClusterStatus(ClusterConfiguration conf)
: base(conf)
{
if (!(conf is WebHdfsClusterConfiguration))
throw new ArgumentException("Expected a WebHdfsClusterConfiguration, got a " + conf.GetType());
this.config = conf as WebHdfsClusterConfiguration;
this.yarnClient = new NativeYarnClient(this.config.StatusNode, this.config.StatusNodePort, new HdfsClient(this.config.UserName));
}
public string Launch(XDocument config)
{
return(NativeYarnClient.LaunchJob(config, this.jarFile, this.yarnDirectory, this.logger));
}
