public virtual void TestShortCircuitRenewCancel() { IPEndPoint addr = NetUtils.CreateSocketAddr(Sharpen.Runtime.GetLocalHost().GetHostName (), 123, null); CheckShortCircuitRenewCancel(addr, addr, true); }
/* * Replace block */ /// <exception cref="System.IO.IOException"/> /// <exception cref="System.Net.Sockets.SocketException"/> private bool ReplaceBlock(ExtendedBlock block, DatanodeInfo source, DatanodeInfo sourceProxy, DatanodeInfo destination, StorageType targetStorageType) { Socket sock = new Socket(); try { sock.Connect(NetUtils.CreateSocketAddr(destination.GetXferAddr()), HdfsServerConstants .ReadTimeout); sock.SetKeepAlive(true); // sendRequest DataOutputStream @out = new DataOutputStream(sock.GetOutputStream()); new Sender(@out).ReplaceBlock(block, targetStorageType, BlockTokenSecretManager.DummyToken , source.GetDatanodeUuid(), sourceProxy); @out.Flush(); // receiveResponse DataInputStream reply = new DataInputStream(sock.GetInputStream()); DataTransferProtos.BlockOpResponseProto proto = DataTransferProtos.BlockOpResponseProto .ParseDelimitedFrom(reply); while (proto.GetStatus() == DataTransferProtos.Status.InProgress) { proto = DataTransferProtos.BlockOpResponseProto.ParseDelimitedFrom(reply); } return(proto.GetStatus() == DataTransferProtos.Status.Success); } finally { sock.Close(); } }
public virtual void TestGetMasterAddress() { YarnConfiguration conf = new YarnConfiguration(); // Default is yarn framework string masterHostname = Master.GetMasterAddress(conf).GetHostName(); // no address set so should default to default rm address IPEndPoint rmAddr = NetUtils.CreateSocketAddr(YarnConfiguration.DefaultRmAddress); NUnit.Framework.Assert.AreEqual(masterHostname, rmAddr.GetHostName()); // Trying invalid master address for classic conf.Set(MRConfig.FrameworkName, MRConfig.ClassicFrameworkName); conf.Set(MRConfig.MasterAddress, "local:invalid"); // should throw an exception for invalid value try { Master.GetMasterAddress(conf); NUnit.Framework.Assert.Fail("Should not reach here as there is a bad master address" ); } catch (Exception) { } // Expected // Change master address to a valid value conf.Set(MRConfig.MasterAddress, "bar.com:8042"); masterHostname = Master.GetMasterAddress(conf).GetHostName(); NUnit.Framework.Assert.AreEqual(masterHostname, "bar.com"); // change framework to yarn conf.Set(MRConfig.FrameworkName, MRConfig.YarnFrameworkName); conf.Set(YarnConfiguration.RmAddress, "foo1.com:8192"); masterHostname = Master.GetMasterAddress(conf).GetHostName(); NUnit.Framework.Assert.AreEqual(masterHostname, "foo1.com"); }
/// <exception cref="Sharpen.UnknownHostException"/> public static string GetApplicationWebURLOnJHSWithoutScheme(Configuration conf, ApplicationId appId) { //construct the history url for job string addr = GetJHSWebappURLWithoutScheme(conf); IEnumerator <string> it = AddrSplitter.Split(addr).GetEnumerator(); it.Next(); // ignore the bind host string port = it.Next(); // Use hs address to figure out the host for webapp addr = conf.Get(JHAdminConfig.MrHistoryAddress, JHAdminConfig.DefaultMrHistoryAddress ); string host = AddrSplitter.Split(addr).GetEnumerator().Next(); string hsAddress = Joiner.Join(host, ":", port); IPEndPoint address = NetUtils.CreateSocketAddr(hsAddress, GetDefaultJHSWebappPort (), GetDefaultJHSWebappURLWithoutScheme()); StringBuilder sb = new StringBuilder(); if (address.Address.IsAnyLocalAddress() || address.Address.IsLoopbackAddress()) { sb.Append(Sharpen.Runtime.GetLocalHost().ToString()); } else { sb.Append(address.GetHostName()); } sb.Append(":").Append(address.Port); sb.Append("/jobhistory/job/"); JobID jobId = TypeConverter.FromYarn(appId); sb.Append(jobId.ToString()); return(sb.ToString()); }
/// <exception cref="System.IO.IOException"/> internal EditLogBackupOutputStream(NamenodeRegistration bnReg, JournalInfo journalInfo ) : base() { // RPC proxy to backup node // backup node registration // active node registration // serialized output sent to backup node // backup node // active name-node this.bnRegistration = bnReg; this.journalInfo = journalInfo; IPEndPoint bnAddress = NetUtils.CreateSocketAddr(bnRegistration.GetAddress()); try { this.backupNode = NameNodeProxies.CreateNonHAProxy <JournalProtocol>(new HdfsConfiguration (), bnAddress, UserGroupInformation.GetCurrentUser(), true).GetProxy(); } catch (IOException e) { Storage.Log.Error("Error connecting to: " + bnAddress, e); throw; } this.doubleBuf = new EditsDoubleBuffer(DefaultBufferSize); this.@out = new DataOutputBuffer(DefaultBufferSize); }
/// <exception cref="System.IO.IOException"/> private static ApplicationClientProtocol GetRmClient <_T0>(Org.Apache.Hadoop.Security.Token.Token <_T0> token, Configuration conf) where _T0 : TokenIdentifier { string[] services = token.GetService().ToString().Split(","); foreach (string service in services) { IPEndPoint addr = NetUtils.CreateSocketAddr(service); if (localSecretManager != null) { // return null if it's our token if (localServiceAddress.Address.IsAnyLocalAddress()) { if (NetUtils.IsLocalAddress(addr.Address) && addr.Port == localServiceAddress.Port) { return(null); } } else { if (addr.Equals(localServiceAddress)) { return(null); } } } } return(ClientRMProxy.CreateRMProxy <ApplicationClientProtocol>(conf)); }
/// <exception cref="System.IO.IOException"/> internal static ClientDatanodeProtocolPB CreateClientDatanodeProtocolProxy(DatanodeID datanodeid, Configuration conf, int socketTimeout, bool connectToDnViaHostname, LocatedBlock locatedBlock) { string dnAddr = datanodeid.GetIpcAddr(connectToDnViaHostname); IPEndPoint addr = NetUtils.CreateSocketAddr(dnAddr); if (Log.IsDebugEnabled()) { Log.Debug("Connecting to datanode " + dnAddr + " addr=" + addr); } // Since we're creating a new UserGroupInformation here, we know that no // future RPC proxies will be able to re-use the same connection. And // usages of this proxy tend to be one-off calls. // // This is a temporary fix: callers should really achieve this by using // RPC.stopProxy() on the resulting object, but this is currently not // working in trunk. See the discussion on HDFS-1965. Configuration confWithNoIpcIdle = new Configuration(conf); confWithNoIpcIdle.SetInt(CommonConfigurationKeysPublic.IpcClientConnectionMaxidletimeKey , 0); UserGroupInformation ticket = UserGroupInformation.CreateRemoteUser(locatedBlock. GetBlock().GetLocalBlock().ToString()); ticket.AddToken(locatedBlock.GetBlockToken()); return(CreateClientDatanodeProtocolProxy(addr, ticket, confWithNoIpcIdle, NetUtils .GetDefaultSocketFactory(conf), socketTimeout)); }
private RemoteBlockReader(string file, string bpid, long blockId, DataInputStream @in, DataChecksum checksum, bool verifyChecksum, long startOffset, long firstChunkOffset , long bytesToRead, Peer peer, DatanodeID datanodeID, PeerCache peerCache) : base(new Path("/" + Block.BlockFilePrefix + blockId + ":" + bpid + ":of:" + file ), 1, verifyChecksum, checksum.GetChecksumSize() > 0 ? checksum : null, checksum .GetBytesPerChecksum(), checksum.GetChecksumSize()) { // Path is used only for printing block and file information in debug /*too non path-like?*/ this.isLocal = DFSClient.IsLocalAddress(NetUtils.CreateSocketAddr(datanodeID.GetXferAddr ())); this.peer = peer; this.datanodeID = datanodeID; this.@in = @in; this.checksum = checksum; this.startOffset = Math.Max(startOffset, 0); this.blockId = blockId; // The total number of bytes that we need to transfer from the DN is // the amount that the user wants (bytesToRead), plus the padding at // the beginning in order to chunk-align. Note that the DN may elect // to send more than this amount if the read starts/ends mid-chunk. this.bytesNeededToFinish = bytesToRead + (startOffset - firstChunkOffset); this.firstChunkOffset = firstChunkOffset; lastChunkOffset = firstChunkOffset; lastChunkLen = -1; bytesPerChecksum = this.checksum.GetBytesPerChecksum(); checksumSize = this.checksum.GetChecksumSize(); this.peerCache = peerCache; }
///////////////////////////////////////////////////// // Common NameNode methods implementation for backup node. ///////////////////////////////////////////////////// protected internal override IPEndPoint GetRpcServerAddress(Configuration conf) { // NameNode string addr = conf.GetTrimmed(BnAddressNameKey, BnAddressDefault); return(NetUtils.CreateSocketAddr(addr)); }
public virtual void TestUnknownCall() { Configuration conf = new Configuration(); conf.Set(YarnConfiguration.IpcRpcImpl, typeof(HadoopYarnProtoRPC).FullName); YarnRPC rpc = YarnRPC.Create(conf); string bindAddr = "localhost:0"; IPEndPoint addr = NetUtils.CreateSocketAddr(bindAddr); Server server = rpc.GetServer(typeof(ContainerManagementProtocol), new TestRPC.DummyContainerManager (this), addr, conf, null, 1); server.Start(); // Any unrelated protocol would do ApplicationClientProtocol proxy = (ApplicationClientProtocol)rpc.GetProxy(typeof( ApplicationClientProtocol), NetUtils.GetConnectAddress(server), conf); try { proxy.GetNewApplication(Records.NewRecord <GetNewApplicationRequest>()); NUnit.Framework.Assert.Fail("Excepted RPC call to fail with unknown method."); } catch (YarnException e) { NUnit.Framework.Assert.IsTrue(e.Message.Matches("Unknown method getNewApplication called on.*" + "org.apache.hadoop.yarn.proto.ApplicationClientProtocol" + "\\$ApplicationClientProtocolService\\$BlockingInterface protocol." )); } catch (Exception e) { Sharpen.Runtime.PrintStackTrace(e); } }
/// <summary>Create a client proxy for the specified engine.</summary> /// <exception cref="System.IO.IOException"/> private RPCCallBenchmark.RpcServiceWrapper CreateRpcClient(RPCCallBenchmark.MyOptions opts) { IPEndPoint addr = NetUtils.CreateSocketAddr(opts.host, opts.GetPort()); if (opts.rpcEngine == typeof(ProtobufRpcEngine)) { TestProtoBufRpc.TestRpcService proxy = RPC.GetProxy <TestProtoBufRpc.TestRpcService >(0, addr, conf); return(new _RpcServiceWrapper_394(proxy)); } else { if (opts.rpcEngine == typeof(WritableRpcEngine)) { TestRPC.TestProtocol proxy = RPC.GetProxy <TestRPC.TestProtocol>(TestRPC.TestProtocol .versionID, addr, conf); return(new _RpcServiceWrapper_407(proxy)); } else { throw new RuntimeException("unsupported engine: " + opts.rpcEngine); } } }
/// <exception cref="System.IO.IOException"/> private static IList <IPEndPoint> GetLoggerAddresses(URI uri) { string authority = uri.GetAuthority(); Preconditions.CheckArgument(authority != null && !authority.IsEmpty(), "URI has no authority: " + uri); string[] parts = StringUtils.Split(authority, ';'); for (int i = 0; i < parts.Length; i++) { parts[i] = parts[i].Trim(); } if (parts.Length % 2 == 0) { Log.Warn("Quorum journal URI '" + uri + "' has an even number " + "of Journal Nodes specified. This is not recommended!" ); } IList <IPEndPoint> addrs = Lists.NewArrayList(); foreach (string addr in parts) { addrs.AddItem(NetUtils.CreateSocketAddr(addr, DFSConfigKeys.DfsJournalnodeRpcPortDefault )); } return(addrs); }
private static IPEndPoint GetAddress(Configuration conf) { string addr = conf.Get(DFSConfigKeys.DfsJournalnodeHttpAddressKey, DFSConfigKeys. DfsJournalnodeHttpAddressDefault); return(NetUtils.CreateSocketAddr(addr, DFSConfigKeys.DfsJournalnodeHttpPortDefault , DFSConfigKeys.DfsJournalnodeHttpAddressKey)); }
public virtual void TestShortCircuitRenewCancelDifferentHostDifferentPort() { IPEndPoint rmAddr = NetUtils.CreateSocketAddr(Sharpen.Runtime.GetLocalHost().GetHostName (), 123, null); CheckShortCircuitRenewCancel(rmAddr, new IPEndPoint("1.1.1.1", rmAddr.Port + 1), false); }
public virtual void TestShortCircuitRenewCancelWildcardAddress() { IPEndPoint rmAddr = new IPEndPoint(123); IPEndPoint serviceAddr = NetUtils.CreateSocketAddr(Sharpen.Runtime.GetLocalHost() .GetHostName(), rmAddr.Port, null); CheckShortCircuitRenewCancel(rmAddr, serviceAddr, true); }
internal static IPEndPoint GetAddress(Configuration conf) { string addr = conf.Get(DFSConfigKeys.DfsJournalnodeRpcAddressKey, DFSConfigKeys.DfsJournalnodeRpcAddressDefault ); return(NetUtils.CreateSocketAddr(addr, 0, DFSConfigKeys.DfsJournalnodeRpcAddressKey )); }
public virtual IPEndPoint GetHttpAddress(Configuration conf) { string addr = conf.Get(NfsConfigKeys.NfsHttpAddressKey, NfsConfigKeys.NfsHttpAddressDefault ); return(NetUtils.CreateSocketAddr(addr, NfsConfigKeys.NfsHttpPortDefault, NfsConfigKeys .NfsHttpAddressKey)); }
internal virtual void RunGoodCases(IPEndPoint addr, string host, int port) { Assert.Equal(addr, NetUtils.CreateSocketAddr(host, port)); Assert.Equal(addr, NetUtils.CreateSocketAddr("hdfs://" + host, port)); Assert.Equal(addr, NetUtils.CreateSocketAddr("hdfs://" + host + "/path", port)); }
public virtual void TestWebImageViewer() { WebImageViewer viewer = new WebImageViewer(NetUtils.CreateSocketAddr("localhost:0" )); try { viewer.InitServer(originalFsimage.GetAbsolutePath()); int port = viewer.GetPort(); // create a WebHdfsFileSystem instance URI uri = new URI("webhdfs://localhost:" + port.ToString()); Configuration conf = new Configuration(); WebHdfsFileSystem webhdfs = (WebHdfsFileSystem)FileSystem.Get(uri, conf); // verify the number of directories FileStatus[] statuses = webhdfs.ListStatus(new Path("/")); NUnit.Framework.Assert.AreEqual(NumDirs + 3, statuses.Length); // contains empty and xattr directory // verify the number of files in the directory statuses = webhdfs.ListStatus(new Path("/dir0")); NUnit.Framework.Assert.AreEqual(FilesPerDir, statuses.Length); // compare a file FileStatus status = webhdfs.ListStatus(new Path("/dir0/file0"))[0]; FileStatus expected = writtenFiles["/dir0/file0"]; CompareFile(expected, status); // LISTSTATUS operation to an empty directory statuses = webhdfs.ListStatus(new Path("/emptydir")); NUnit.Framework.Assert.AreEqual(0, statuses.Length); // LISTSTATUS operation to a invalid path Uri url = new Uri("http://localhost:" + port + "/webhdfs/v1/invalid/?op=LISTSTATUS" ); VerifyHttpResponseCode(HttpURLConnection.HttpNotFound, url); // LISTSTATUS operation to a invalid prefix url = new Uri("http://localhost:" + port + "/foo"); VerifyHttpResponseCode(HttpURLConnection.HttpNotFound, url); // GETFILESTATUS operation status = webhdfs.GetFileStatus(new Path("/dir0/file0")); CompareFile(expected, status); // GETFILESTATUS operation to a invalid path url = new Uri("http://localhost:" + port + "/webhdfs/v1/invalid/?op=GETFILESTATUS" ); VerifyHttpResponseCode(HttpURLConnection.HttpNotFound, url); // invalid operation url = new Uri("http://localhost:" + port + "/webhdfs/v1/?op=INVALID"); VerifyHttpResponseCode(HttpURLConnection.HttpBadRequest, url); // invalid method url = new Uri("http://localhost:" + port + "/webhdfs/v1/?op=LISTSTATUS"); HttpURLConnection connection = (HttpURLConnection)url.OpenConnection(); connection.SetRequestMethod("POST"); connection.Connect(); NUnit.Framework.Assert.AreEqual(HttpURLConnection.HttpBadMethod, connection.GetResponseCode ()); } finally { // shutdown the viewer viewer.Close(); } }
public virtual void CheckNameServiceId(Configuration conf, string addr, string expectedNameServiceId ) { IPEndPoint s = NetUtils.CreateSocketAddr(addr); string nameserviceId = DFSUtil.GetNameServiceIdFromAddress(conf, s, DFSConfigKeys .DfsNamenodeServiceRpcAddressKey, DFSConfigKeys.DfsNamenodeRpcAddressKey); NUnit.Framework.Assert.AreEqual(expectedNameServiceId, nameserviceId); }
protected internal override IPEndPoint GetHttpServerAddress(Configuration conf) { // NameNode System.Diagnostics.Debug.Assert(GetNameNodeAddress() != null, "rpcAddress should be calculated first" ); string addr = conf.GetTrimmed(BnHttpAddressNameKey, BnHttpAddressDefault); return(NetUtils.CreateSocketAddr(addr)); }
/// <exception cref="System.IO.IOException"/> public static Credentials GetDTfromRemote(URLConnectionFactory factory, URI nnUri , string renewer, string proxyUser) { StringBuilder buf = new StringBuilder(nnUri.ToString()).Append(GetDelegationTokenServlet .PathSpec); string separator = "?"; if (renewer != null) { buf.Append("?").Append(GetDelegationTokenServlet.Renewer).Append("=").Append(renewer ); separator = "&"; } if (proxyUser != null) { buf.Append(separator).Append("doas=").Append(proxyUser); } bool isHttps = nnUri.GetScheme().Equals("https"); HttpURLConnection conn = null; DataInputStream dis = null; IPEndPoint serviceAddr = NetUtils.CreateSocketAddr(nnUri.GetAuthority()); try { if (Log.IsDebugEnabled()) { Log.Debug("Retrieving token from: " + buf); } conn = Run(factory, new Uri(buf.ToString())); InputStream @in = conn.GetInputStream(); Credentials ts = new Credentials(); dis = new DataInputStream(@in); ts.ReadFields(dis); foreach (Org.Apache.Hadoop.Security.Token.Token <object> token in ts.GetAllTokens( )) { token.SetKind(isHttps ? HsftpFileSystem.TokenKind : HftpFileSystem.TokenKind); SecurityUtil.SetTokenService(token, serviceAddr); } return(ts); } catch (Exception e) { throw new IOException("Unable to obtain remote token", e); } finally { IOUtils.Cleanup(Log, dis); if (conn != null) { conn.Disconnect(); } } }
protected internal override IPEndPoint GetServiceRpcServerAddress(Configuration conf ) { string addr = conf.GetTrimmed(BnServiceRpcAddressKey); if (addr == null || addr.IsEmpty()) { return(null); } return(NetUtils.CreateSocketAddr(addr)); }
/// <exception cref="System.Exception"/> private void TestRPCTimeout(string rpcClass) { Configuration conf = new Configuration(); // set timeout low for the test conf.SetInt("yarn.rpc.nm-command-timeout", 3000); conf.Set(YarnConfiguration.IpcRpcImpl, rpcClass); YarnRPC rpc = YarnRPC.Create(conf); string bindAddr = "localhost:0"; IPEndPoint addr = NetUtils.CreateSocketAddr(bindAddr); Server server = rpc.GetServer(typeof(ContainerManagementProtocol), new TestContainerLaunchRPC.DummyContainerManager (this), addr, conf, null, 1); server.Start(); try { ContainerManagementProtocol proxy = (ContainerManagementProtocol)rpc.GetProxy(typeof( ContainerManagementProtocol), server.GetListenerAddress(), conf); ContainerLaunchContext containerLaunchContext = recordFactory.NewRecordInstance <ContainerLaunchContext >(); ApplicationId applicationId = ApplicationId.NewInstance(0, 0); ApplicationAttemptId applicationAttemptId = ApplicationAttemptId.NewInstance(applicationId , 0); ContainerId containerId = ContainerId.NewContainerId(applicationAttemptId, 100); NodeId nodeId = NodeId.NewInstance("localhost", 1234); Resource resource = Resource.NewInstance(1234, 2); ContainerTokenIdentifier containerTokenIdentifier = new ContainerTokenIdentifier( containerId, "localhost", "user", resource, Runtime.CurrentTimeMillis() + 10000, 42, 42, Priority.NewInstance(0), 0); Token containerToken = TestRPC.NewContainerToken(nodeId, Sharpen.Runtime.GetBytesForString ("password"), containerTokenIdentifier); StartContainerRequest scRequest = StartContainerRequest.NewInstance(containerLaunchContext , containerToken); IList <StartContainerRequest> list = new AList <StartContainerRequest>(); list.AddItem(scRequest); StartContainersRequest allRequests = StartContainersRequest.NewInstance(list); try { proxy.StartContainers(allRequests); } catch (Exception e) { Log.Info(StringUtils.StringifyException(e)); NUnit.Framework.Assert.AreEqual("Error, exception is not: " + typeof(SocketTimeoutException ).FullName, typeof(SocketTimeoutException).FullName, e.GetType().FullName); return; } } finally { server.Stop(); } NUnit.Framework.Assert.Fail("timeout exception should have occurred!"); }
public virtual void TestBuildTokenServiceSockAddr() { SecurityUtil.SetTokenServiceUseIp(true); Assert.Equal("127.0.0.1:123", SecurityUtil.BuildTokenService(new IPEndPoint("LocalHost", 123)).ToString()); Assert.Equal("127.0.0.1:123", SecurityUtil.BuildTokenService(new IPEndPoint("127.0.0.1", 123)).ToString()); // what goes in, comes out Assert.Equal("127.0.0.1:123", SecurityUtil.BuildTokenService(NetUtils .CreateSocketAddr("127.0.0.1", 123)).ToString()); }
/// <summary>create the service name for a Delegation token</summary> /// <param name="uri">of the service</param> /// <param name="defPort">is used if the uri lacks a port</param> /// <returns>the token service, or null if no authority</returns> /// <seealso cref="BuildTokenService(System.Net.IPEndPoint)"/> public static string BuildDTServiceName(URI uri, int defPort) { string authority = uri.GetAuthority(); if (authority == null) { return(null); } IPEndPoint addr = NetUtils.CreateSocketAddr(authority, defPort); return(BuildTokenService(addr).ToString()); }
/// <exception cref="System.Exception"/> protected override void ServiceInit(Configuration conf) { this.clientServiceBindAddress = TestClientRedirect.Rmaddress; /* * clientServiceBindAddress = conf.get( * YarnConfiguration.APPSMANAGER_ADDRESS, * YarnConfiguration.DEFAULT_APPSMANAGER_BIND_ADDRESS); */ this.clientBindAddress = NetUtils.CreateSocketAddr(this.clientServiceBindAddress); base.ServiceInit(conf); }
/// <seealso cref="Org.Apache.Hadoop.Hdfs.DFSUtil.GetHttpPolicy(Org.Apache.Hadoop.Conf.Configuration) /// "> /// for information related to the different configuration options and /// Http Policy is decided. /// </seealso> /// <exception cref="System.IO.IOException"/> internal virtual void Start() { HttpConfig.Policy policy = DFSUtil.GetHttpPolicy(conf); string infoHost = bindAddress.GetHostName(); IPEndPoint httpAddr = bindAddress; string httpsAddrString = conf.GetTrimmed(DFSConfigKeys.DfsNamenodeHttpsAddressKey , DFSConfigKeys.DfsNamenodeHttpsAddressDefault); IPEndPoint httpsAddr = NetUtils.CreateSocketAddr(httpsAddrString); if (httpsAddr != null) { // If DFS_NAMENODE_HTTPS_BIND_HOST_KEY exists then it overrides the // host name portion of DFS_NAMENODE_HTTPS_ADDRESS_KEY. string bindHost = conf.GetTrimmed(DFSConfigKeys.DfsNamenodeHttpsBindHostKey); if (bindHost != null && !bindHost.IsEmpty()) { httpsAddr = new IPEndPoint(bindHost, httpsAddr.Port); } } HttpServer2.Builder builder = DFSUtil.HttpServerTemplateForNNAndJN(conf, httpAddr , httpsAddr, "hdfs", DFSConfigKeys.DfsNamenodeKerberosInternalSpnegoPrincipalKey , DFSConfigKeys.DfsNamenodeKeytabFileKey); httpServer = builder.Build(); if (policy.IsHttpsEnabled()) { // assume same ssl port for all datanodes IPEndPoint datanodeSslPort = NetUtils.CreateSocketAddr(conf.GetTrimmed(DFSConfigKeys .DfsDatanodeHttpsAddressKey, infoHost + ":" + DFSConfigKeys.DfsDatanodeHttpsDefaultPort )); httpServer.SetAttribute(DFSConfigKeys.DfsDatanodeHttpsPortKey, datanodeSslPort.Port ); } InitWebHdfs(conf); httpServer.SetAttribute(NamenodeAttributeKey, nn); httpServer.SetAttribute(JspHelper.CurrentConf, conf); SetupServlets(httpServer, conf); httpServer.Start(); int connIdx = 0; if (policy.IsHttpEnabled()) { httpAddress = httpServer.GetConnectorAddress(connIdx++); conf.Set(DFSConfigKeys.DfsNamenodeHttpAddressKey, NetUtils.GetHostPortString(httpAddress )); } if (policy.IsHttpsEnabled()) { httpsAddress = httpServer.GetConnectorAddress(connIdx); conf.Set(DFSConfigKeys.DfsNamenodeHttpsAddressKey, NetUtils.GetHostPortString(httpsAddress )); } }
private static string GetHostnameForSpnegoPrincipal(Configuration conf) { string addr = conf.GetTrimmed(DFSConfigKeys.DfsDatanodeHttpAddressKey, null); if (addr == null) { addr = conf.GetTrimmed(DFSConfigKeys.DfsDatanodeHttpsAddressKey, DFSConfigKeys.DfsDatanodeHttpsAddressDefault ); } IPEndPoint inetSocker = NetUtils.CreateSocketAddr(addr); return(inetSocker.GetHostString()); }
///////////////////////////////////////////////////////////////////////////// /// <summary> /// <inheritDoc/> /// /// </summary> /// <exception cref="System.IO.IOException"/> internal override void Init(Job job) { lock (this) { Configuration conf = job.GetConfiguration(); if (cluster == null) { string jobTrackerStr = conf.Get("mapreduce.jobtracker.address", "localhost:8012"); cluster = new Cluster(NetUtils.CreateSocketAddr(jobTrackerStr), conf); } ChooseMachine(conf).Init(job); } }