public ClusterClient Client(DryadLinqContext context) { if (_clusterClient == null) { _clusterClient = new NativeYarnClient(HeadNode, HdfsPort, LauncherPort); } return(_clusterClient); }
/// <summary> /// Create a cluster containing just the local machine. /// </summary> /// <param name="conf">Configuration for the local machine.</param> public WebHdfsClusterStatus(ClusterConfiguration conf) : base(conf) { if (!(conf is WebHdfsClusterConfiguration)) { throw new ArgumentException("Expected a WebHdfsClusterConfiguration, got a " + conf.GetType()); } this.config = conf as WebHdfsClusterConfiguration; this.yarnClient = new NativeYarnClient(this.config.StatusNode, this.config.StatusNodePort, new HdfsClient(this.config.UserName)); }
/// <summary> /// Make a new cluster object representing a YARN cluster with default ports /// </summary> /// <param name="headNode">The computer where the YarnLauncher is running</param> public DryadLinqYarnCluster(string headNode) { HeadNode = headNode; LauncherPort = 8471; NameNode = headNode; HdfsPort = 9000; WebHdfsPort = 50070; _dfsClient = null; _clusterClient = null; }
/// <summary> /// Create a cluster containing just the local machine. /// </summary> /// <param name="conf">Configuration for the local machine.</param> public HdfsClusterStatus(ClusterConfiguration conf) : base(conf) { if (!(conf is HdfsClusterConfiguration)) { throw new ArgumentException("Expected an HdfsClusterConfiguration, got a " + conf.GetType()); } this.config = conf as HdfsClusterConfiguration; // make a fake call to initialize the cluster on the foreground thread // HDFS does not work if initialized on the background thread. Uri uri = DfsFile.UriFromPath(this.config.JobsFolderUri, ""); this.config.DfsClient.IsFileExists(uri); // ignore result this.yarnClient = new NativeYarnClient(this.config.StatusNode, this.config.StatusNodePort, new HdfsClient(this.config.UserName)); }
/// <summary> /// Create a cluster containing just the local machine. /// </summary> /// <param name="conf">Configuration for the local machine.</param> public HdfsClusterStatus(ClusterConfiguration conf) : base(conf) { if (!(conf is HdfsClusterConfiguration)) throw new ArgumentException("Expected an HdfsClusterConfiguration, got a " + conf.GetType()); this.config = conf as HdfsClusterConfiguration; // make a fake call to initialize the cluster on the foreground thread // HDFS does not work if initialized on the background thread. Uri uri = DfsFile.UriFromPath(this.config.JobsFolderUri, ""); this.config.DfsClient.IsFileExists(uri); // ignore result this.yarnClient = new NativeYarnClient(this.config.StatusNode, this.config.StatusNodePort, new HdfsClient(this.config.UserName)); }
/// <summary> /// Create a cluster containing just the local machine. /// </summary> /// <param name="conf">Configuration for the local machine.</param> public WebHdfsClusterStatus(ClusterConfiguration conf) : base(conf) { if (!(conf is WebHdfsClusterConfiguration)) throw new ArgumentException("Expected a WebHdfsClusterConfiguration, got a " + conf.GetType()); this.config = conf as WebHdfsClusterConfiguration; this.yarnClient = new NativeYarnClient(this.config.StatusNode, this.config.StatusNodePort, new HdfsClient(this.config.UserName)); }
public string Launch(XDocument config) { return(NativeYarnClient.LaunchJob(config, this.jarFile, this.yarnDirectory, this.logger)); }