public virtual void TestShortCircuitRenewCancel()
{
IPEndPoint addr = NetUtils.CreateSocketAddr(Sharpen.Runtime.GetLocalHost().GetHostName
(), 123, null);
CheckShortCircuitRenewCancel(addr, addr, true);
}
/*
* Replace block
*/
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Net.Sockets.SocketException"/>
private bool ReplaceBlock(ExtendedBlock block, DatanodeInfo source, DatanodeInfo
sourceProxy, DatanodeInfo destination, StorageType targetStorageType)
{
Socket sock = new Socket();
try
{
sock.Connect(NetUtils.CreateSocketAddr(destination.GetXferAddr()), HdfsServerConstants
.ReadTimeout);
sock.SetKeepAlive(true);
// sendRequest
DataOutputStream @out = new DataOutputStream(sock.GetOutputStream());
new Sender(@out).ReplaceBlock(block, targetStorageType, BlockTokenSecretManager.DummyToken
, source.GetDatanodeUuid(), sourceProxy);
@out.Flush();
// receiveResponse
DataInputStream reply = new DataInputStream(sock.GetInputStream());
DataTransferProtos.BlockOpResponseProto proto = DataTransferProtos.BlockOpResponseProto
.ParseDelimitedFrom(reply);
while (proto.GetStatus() == DataTransferProtos.Status.InProgress)
{
proto = DataTransferProtos.BlockOpResponseProto.ParseDelimitedFrom(reply);
}
return(proto.GetStatus() == DataTransferProtos.Status.Success);
}
finally
{
sock.Close();
}
}
public virtual void TestGetMasterAddress()
{
YarnConfiguration conf = new YarnConfiguration();
// Default is yarn framework
string masterHostname = Master.GetMasterAddress(conf).GetHostName();
// no address set so should default to default rm address
IPEndPoint rmAddr = NetUtils.CreateSocketAddr(YarnConfiguration.DefaultRmAddress);
NUnit.Framework.Assert.AreEqual(masterHostname, rmAddr.GetHostName());
// Trying invalid master address for classic
conf.Set(MRConfig.FrameworkName, MRConfig.ClassicFrameworkName);
conf.Set(MRConfig.MasterAddress, "local:invalid");
// should throw an exception for invalid value
try
{
Master.GetMasterAddress(conf);
NUnit.Framework.Assert.Fail("Should not reach here as there is a bad master address"
);
}
catch (Exception)
{
}
// Expected
// Change master address to a valid value
conf.Set(MRConfig.MasterAddress, "bar.com:8042");
masterHostname = Master.GetMasterAddress(conf).GetHostName();
NUnit.Framework.Assert.AreEqual(masterHostname, "bar.com");
// change framework to yarn
conf.Set(MRConfig.FrameworkName, MRConfig.YarnFrameworkName);
conf.Set(YarnConfiguration.RmAddress, "foo1.com:8192");
masterHostname = Master.GetMasterAddress(conf).GetHostName();
NUnit.Framework.Assert.AreEqual(masterHostname, "foo1.com");
}
/// <exception cref="Sharpen.UnknownHostException"/>
public static string GetApplicationWebURLOnJHSWithoutScheme(Configuration conf, ApplicationId
appId)
{
//construct the history url for job
string addr = GetJHSWebappURLWithoutScheme(conf);
IEnumerator <string> it = AddrSplitter.Split(addr).GetEnumerator();
it.Next();
// ignore the bind host
string port = it.Next();
// Use hs address to figure out the host for webapp
addr = conf.Get(JHAdminConfig.MrHistoryAddress, JHAdminConfig.DefaultMrHistoryAddress
);
string host = AddrSplitter.Split(addr).GetEnumerator().Next();
string hsAddress = Joiner.Join(host, ":", port);
IPEndPoint address = NetUtils.CreateSocketAddr(hsAddress, GetDefaultJHSWebappPort
(), GetDefaultJHSWebappURLWithoutScheme());
StringBuilder sb = new StringBuilder();
if (address.Address.IsAnyLocalAddress() || address.Address.IsLoopbackAddress())
{
sb.Append(Sharpen.Runtime.GetLocalHost().ToString());
}
else
{
sb.Append(address.GetHostName());
}
sb.Append(":").Append(address.Port);
sb.Append("/jobhistory/job/");
JobID jobId = TypeConverter.FromYarn(appId);
sb.Append(jobId.ToString());
return(sb.ToString());
}
/// <exception cref="System.IO.IOException"/>
internal EditLogBackupOutputStream(NamenodeRegistration bnReg, JournalInfo journalInfo
)
: base()
{
// RPC proxy to backup node
// backup node registration
// active node registration
// serialized output sent to backup node
// backup node
// active name-node
this.bnRegistration = bnReg;
this.journalInfo = journalInfo;
IPEndPoint bnAddress = NetUtils.CreateSocketAddr(bnRegistration.GetAddress());
try
{
this.backupNode = NameNodeProxies.CreateNonHAProxy <JournalProtocol>(new HdfsConfiguration
(), bnAddress, UserGroupInformation.GetCurrentUser(), true).GetProxy();
}
catch (IOException e)
{
Storage.Log.Error("Error connecting to: " + bnAddress, e);
throw;
}
this.doubleBuf = new EditsDoubleBuffer(DefaultBufferSize);
this.@out = new DataOutputBuffer(DefaultBufferSize);
}
/// <exception cref="System.IO.IOException"/>
private static ApplicationClientProtocol GetRmClient <_T0>(Org.Apache.Hadoop.Security.Token.Token
<_T0> token, Configuration conf)
where _T0 : TokenIdentifier
{
string[] services = token.GetService().ToString().Split(",");
foreach (string service in services)
{
IPEndPoint addr = NetUtils.CreateSocketAddr(service);
if (localSecretManager != null)
{
// return null if it's our token
if (localServiceAddress.Address.IsAnyLocalAddress())
{
if (NetUtils.IsLocalAddress(addr.Address) && addr.Port == localServiceAddress.Port)
{
return(null);
}
}
else
{
if (addr.Equals(localServiceAddress))
{
return(null);
}
}
}
}
return(ClientRMProxy.CreateRMProxy <ApplicationClientProtocol>(conf));
}
/// <exception cref="System.IO.IOException"/>
internal static ClientDatanodeProtocolPB CreateClientDatanodeProtocolProxy(DatanodeID
datanodeid, Configuration conf, int socketTimeout, bool connectToDnViaHostname,
LocatedBlock locatedBlock)
{
string dnAddr = datanodeid.GetIpcAddr(connectToDnViaHostname);
IPEndPoint addr = NetUtils.CreateSocketAddr(dnAddr);
if (Log.IsDebugEnabled())
{
Log.Debug("Connecting to datanode " + dnAddr + " addr=" + addr);
}
// Since we're creating a new UserGroupInformation here, we know that no
// future RPC proxies will be able to re-use the same connection. And
// usages of this proxy tend to be one-off calls.
//
// This is a temporary fix: callers should really achieve this by using
// RPC.stopProxy() on the resulting object, but this is currently not
// working in trunk. See the discussion on HDFS-1965.
Configuration confWithNoIpcIdle = new Configuration(conf);
confWithNoIpcIdle.SetInt(CommonConfigurationKeysPublic.IpcClientConnectionMaxidletimeKey
, 0);
UserGroupInformation ticket = UserGroupInformation.CreateRemoteUser(locatedBlock.
GetBlock().GetLocalBlock().ToString());
ticket.AddToken(locatedBlock.GetBlockToken());
return(CreateClientDatanodeProtocolProxy(addr, ticket, confWithNoIpcIdle, NetUtils
.GetDefaultSocketFactory(conf), socketTimeout));
}
private RemoteBlockReader(string file, string bpid, long blockId, DataInputStream
@in, DataChecksum checksum, bool verifyChecksum, long startOffset, long firstChunkOffset
, long bytesToRead, Peer peer, DatanodeID datanodeID, PeerCache peerCache)
: base(new Path("/" + Block.BlockFilePrefix + blockId + ":" + bpid + ":of:" + file
), 1, verifyChecksum, checksum.GetChecksumSize() > 0 ? checksum : null, checksum
.GetBytesPerChecksum(), checksum.GetChecksumSize())
{
// Path is used only for printing block and file information in debug
/*too non path-like?*/
this.isLocal = DFSClient.IsLocalAddress(NetUtils.CreateSocketAddr(datanodeID.GetXferAddr
()));
this.peer = peer;
this.datanodeID = datanodeID;
this.@in = @in;
this.checksum = checksum;
this.startOffset = Math.Max(startOffset, 0);
this.blockId = blockId;
// The total number of bytes that we need to transfer from the DN is
// the amount that the user wants (bytesToRead), plus the padding at
// the beginning in order to chunk-align. Note that the DN may elect
// to send more than this amount if the read starts/ends mid-chunk.
this.bytesNeededToFinish = bytesToRead + (startOffset - firstChunkOffset);
this.firstChunkOffset = firstChunkOffset;
lastChunkOffset = firstChunkOffset;
lastChunkLen = -1;
bytesPerChecksum = this.checksum.GetBytesPerChecksum();
checksumSize = this.checksum.GetChecksumSize();
this.peerCache = peerCache;
}
/////////////////////////////////////////////////////
// Common NameNode methods implementation for backup node.
/////////////////////////////////////////////////////
protected internal override IPEndPoint GetRpcServerAddress(Configuration conf)
{
// NameNode
string addr = conf.GetTrimmed(BnAddressNameKey, BnAddressDefault);
return(NetUtils.CreateSocketAddr(addr));
}
public virtual void TestUnknownCall()
{
Configuration conf = new Configuration();
conf.Set(YarnConfiguration.IpcRpcImpl, typeof(HadoopYarnProtoRPC).FullName);
YarnRPC rpc = YarnRPC.Create(conf);
string bindAddr = "localhost:0";
IPEndPoint addr = NetUtils.CreateSocketAddr(bindAddr);
Server server = rpc.GetServer(typeof(ContainerManagementProtocol), new TestRPC.DummyContainerManager
(this), addr, conf, null, 1);
server.Start();
// Any unrelated protocol would do
ApplicationClientProtocol proxy = (ApplicationClientProtocol)rpc.GetProxy(typeof(
ApplicationClientProtocol), NetUtils.GetConnectAddress(server), conf);
try
{
proxy.GetNewApplication(Records.NewRecord <GetNewApplicationRequest>());
NUnit.Framework.Assert.Fail("Excepted RPC call to fail with unknown method.");
}
catch (YarnException e)
{
NUnit.Framework.Assert.IsTrue(e.Message.Matches("Unknown method getNewApplication called on.*"
+ "org.apache.hadoop.yarn.proto.ApplicationClientProtocol" + "\\$ApplicationClientProtocolService\\$BlockingInterface protocol."
));
}
catch (Exception e)
{
Sharpen.Runtime.PrintStackTrace(e);
}
}
/// <summary>Create a client proxy for the specified engine.</summary>
/// <exception cref="System.IO.IOException"/>
private RPCCallBenchmark.RpcServiceWrapper CreateRpcClient(RPCCallBenchmark.MyOptions
opts)
{
IPEndPoint addr = NetUtils.CreateSocketAddr(opts.host, opts.GetPort());
if (opts.rpcEngine == typeof(ProtobufRpcEngine))
{
TestProtoBufRpc.TestRpcService proxy = RPC.GetProxy <TestProtoBufRpc.TestRpcService
>(0, addr, conf);
return(new _RpcServiceWrapper_394(proxy));
}
else
{
if (opts.rpcEngine == typeof(WritableRpcEngine))
{
TestRPC.TestProtocol proxy = RPC.GetProxy <TestRPC.TestProtocol>(TestRPC.TestProtocol
.versionID, addr, conf);
return(new _RpcServiceWrapper_407(proxy));
}
else
{
throw new RuntimeException("unsupported engine: " + opts.rpcEngine);
}
}
}
/// <exception cref="System.IO.IOException"/>
private static IList <IPEndPoint> GetLoggerAddresses(URI uri)
{
string authority = uri.GetAuthority();
Preconditions.CheckArgument(authority != null && !authority.IsEmpty(), "URI has no authority: "
+ uri);
string[] parts = StringUtils.Split(authority, ';');
for (int i = 0; i < parts.Length; i++)
{
parts[i] = parts[i].Trim();
}
if (parts.Length % 2 == 0)
{
Log.Warn("Quorum journal URI '" + uri + "' has an even number " + "of Journal Nodes specified. This is not recommended!"
);
}
IList <IPEndPoint> addrs = Lists.NewArrayList();
foreach (string addr in parts)
{
addrs.AddItem(NetUtils.CreateSocketAddr(addr, DFSConfigKeys.DfsJournalnodeRpcPortDefault
));
}
return(addrs);
}
private static IPEndPoint GetAddress(Configuration conf)
{
string addr = conf.Get(DFSConfigKeys.DfsJournalnodeHttpAddressKey, DFSConfigKeys.
DfsJournalnodeHttpAddressDefault);
return(NetUtils.CreateSocketAddr(addr, DFSConfigKeys.DfsJournalnodeHttpPortDefault
, DFSConfigKeys.DfsJournalnodeHttpAddressKey));
}
public virtual void TestShortCircuitRenewCancelDifferentHostDifferentPort()
{
IPEndPoint rmAddr = NetUtils.CreateSocketAddr(Sharpen.Runtime.GetLocalHost().GetHostName
(), 123, null);
CheckShortCircuitRenewCancel(rmAddr, new IPEndPoint("1.1.1.1", rmAddr.Port + 1),
false);
}
public virtual void TestShortCircuitRenewCancelWildcardAddress()
{
IPEndPoint rmAddr = new IPEndPoint(123);
IPEndPoint serviceAddr = NetUtils.CreateSocketAddr(Sharpen.Runtime.GetLocalHost()
.GetHostName(), rmAddr.Port, null);
CheckShortCircuitRenewCancel(rmAddr, serviceAddr, true);
}
internal static IPEndPoint GetAddress(Configuration conf)
{
string addr = conf.Get(DFSConfigKeys.DfsJournalnodeRpcAddressKey, DFSConfigKeys.DfsJournalnodeRpcAddressDefault
);
return(NetUtils.CreateSocketAddr(addr, 0, DFSConfigKeys.DfsJournalnodeRpcAddressKey
));
}
public virtual IPEndPoint GetHttpAddress(Configuration conf)
{
string addr = conf.Get(NfsConfigKeys.NfsHttpAddressKey, NfsConfigKeys.NfsHttpAddressDefault
);
return(NetUtils.CreateSocketAddr(addr, NfsConfigKeys.NfsHttpPortDefault, NfsConfigKeys
.NfsHttpAddressKey));
}
internal virtual void RunGoodCases(IPEndPoint addr, string host, int port)
{
Assert.Equal(addr, NetUtils.CreateSocketAddr(host, port));
Assert.Equal(addr, NetUtils.CreateSocketAddr("hdfs://" + host,
port));
Assert.Equal(addr, NetUtils.CreateSocketAddr("hdfs://" + host
+ "/path", port));
}
public virtual void TestWebImageViewer()
{
WebImageViewer viewer = new WebImageViewer(NetUtils.CreateSocketAddr("localhost:0"
));
try
{
viewer.InitServer(originalFsimage.GetAbsolutePath());
int port = viewer.GetPort();
// create a WebHdfsFileSystem instance
URI uri = new URI("webhdfs://localhost:" + port.ToString());
Configuration conf = new Configuration();
WebHdfsFileSystem webhdfs = (WebHdfsFileSystem)FileSystem.Get(uri, conf);
// verify the number of directories
FileStatus[] statuses = webhdfs.ListStatus(new Path("/"));
NUnit.Framework.Assert.AreEqual(NumDirs + 3, statuses.Length);
// contains empty and xattr directory
// verify the number of files in the directory
statuses = webhdfs.ListStatus(new Path("/dir0"));
NUnit.Framework.Assert.AreEqual(FilesPerDir, statuses.Length);
// compare a file
FileStatus status = webhdfs.ListStatus(new Path("/dir0/file0"))[0];
FileStatus expected = writtenFiles["/dir0/file0"];
CompareFile(expected, status);
// LISTSTATUS operation to an empty directory
statuses = webhdfs.ListStatus(new Path("/emptydir"));
NUnit.Framework.Assert.AreEqual(0, statuses.Length);
// LISTSTATUS operation to a invalid path
Uri url = new Uri("http://localhost:" + port + "/webhdfs/v1/invalid/?op=LISTSTATUS"
);
VerifyHttpResponseCode(HttpURLConnection.HttpNotFound, url);
// LISTSTATUS operation to a invalid prefix
url = new Uri("http://localhost:" + port + "/foo");
VerifyHttpResponseCode(HttpURLConnection.HttpNotFound, url);
// GETFILESTATUS operation
status = webhdfs.GetFileStatus(new Path("/dir0/file0"));
CompareFile(expected, status);
// GETFILESTATUS operation to a invalid path
url = new Uri("http://localhost:" + port + "/webhdfs/v1/invalid/?op=GETFILESTATUS"
);
VerifyHttpResponseCode(HttpURLConnection.HttpNotFound, url);
// invalid operation
url = new Uri("http://localhost:" + port + "/webhdfs/v1/?op=INVALID");
VerifyHttpResponseCode(HttpURLConnection.HttpBadRequest, url);
// invalid method
url = new Uri("http://localhost:" + port + "/webhdfs/v1/?op=LISTSTATUS");
HttpURLConnection connection = (HttpURLConnection)url.OpenConnection();
connection.SetRequestMethod("POST");
connection.Connect();
NUnit.Framework.Assert.AreEqual(HttpURLConnection.HttpBadMethod, connection.GetResponseCode
());
}
finally
{
// shutdown the viewer
viewer.Close();
}
}
public virtual void CheckNameServiceId(Configuration conf, string addr, string expectedNameServiceId
)
{
IPEndPoint s = NetUtils.CreateSocketAddr(addr);
string nameserviceId = DFSUtil.GetNameServiceIdFromAddress(conf, s, DFSConfigKeys
.DfsNamenodeServiceRpcAddressKey, DFSConfigKeys.DfsNamenodeRpcAddressKey);
NUnit.Framework.Assert.AreEqual(expectedNameServiceId, nameserviceId);
}
protected internal override IPEndPoint GetHttpServerAddress(Configuration conf)
{
// NameNode
System.Diagnostics.Debug.Assert(GetNameNodeAddress() != null, "rpcAddress should be calculated first"
);
string addr = conf.GetTrimmed(BnHttpAddressNameKey, BnHttpAddressDefault);
return(NetUtils.CreateSocketAddr(addr));
}
/// <exception cref="System.IO.IOException"/>
public static Credentials GetDTfromRemote(URLConnectionFactory factory, URI nnUri
, string renewer, string proxyUser)
{
StringBuilder buf = new StringBuilder(nnUri.ToString()).Append(GetDelegationTokenServlet
.PathSpec);
string separator = "?";
if (renewer != null)
{
buf.Append("?").Append(GetDelegationTokenServlet.Renewer).Append("=").Append(renewer
);
separator = "&";
}
if (proxyUser != null)
{
buf.Append(separator).Append("doas=").Append(proxyUser);
}
bool isHttps = nnUri.GetScheme().Equals("https");
HttpURLConnection conn = null;
DataInputStream dis = null;
IPEndPoint serviceAddr = NetUtils.CreateSocketAddr(nnUri.GetAuthority());
try
{
if (Log.IsDebugEnabled())
{
Log.Debug("Retrieving token from: " + buf);
}
conn = Run(factory, new Uri(buf.ToString()));
InputStream @in = conn.GetInputStream();
Credentials ts = new Credentials();
dis = new DataInputStream(@in);
ts.ReadFields(dis);
foreach (Org.Apache.Hadoop.Security.Token.Token <object> token in ts.GetAllTokens(
))
{
token.SetKind(isHttps ? HsftpFileSystem.TokenKind : HftpFileSystem.TokenKind);
SecurityUtil.SetTokenService(token, serviceAddr);
}
return(ts);
}
catch (Exception e)
{
throw new IOException("Unable to obtain remote token", e);
}
finally
{
IOUtils.Cleanup(Log, dis);
if (conn != null)
{
conn.Disconnect();
}
}
}
protected internal override IPEndPoint GetServiceRpcServerAddress(Configuration conf
)
{
string addr = conf.GetTrimmed(BnServiceRpcAddressKey);
if (addr == null || addr.IsEmpty())
{
return(null);
}
return(NetUtils.CreateSocketAddr(addr));
}
/// <exception cref="System.Exception"/>
private void TestRPCTimeout(string rpcClass)
{
Configuration conf = new Configuration();
// set timeout low for the test
conf.SetInt("yarn.rpc.nm-command-timeout", 3000);
conf.Set(YarnConfiguration.IpcRpcImpl, rpcClass);
YarnRPC rpc = YarnRPC.Create(conf);
string bindAddr = "localhost:0";
IPEndPoint addr = NetUtils.CreateSocketAddr(bindAddr);
Server server = rpc.GetServer(typeof(ContainerManagementProtocol), new TestContainerLaunchRPC.DummyContainerManager
(this), addr, conf, null, 1);
server.Start();
try
{
ContainerManagementProtocol proxy = (ContainerManagementProtocol)rpc.GetProxy(typeof(
ContainerManagementProtocol), server.GetListenerAddress(), conf);
ContainerLaunchContext containerLaunchContext = recordFactory.NewRecordInstance <ContainerLaunchContext
>();
ApplicationId applicationId = ApplicationId.NewInstance(0, 0);
ApplicationAttemptId applicationAttemptId = ApplicationAttemptId.NewInstance(applicationId
, 0);
ContainerId containerId = ContainerId.NewContainerId(applicationAttemptId, 100);
NodeId nodeId = NodeId.NewInstance("localhost", 1234);
Resource resource = Resource.NewInstance(1234, 2);
ContainerTokenIdentifier containerTokenIdentifier = new ContainerTokenIdentifier(
containerId, "localhost", "user", resource, Runtime.CurrentTimeMillis() + 10000,
42, 42, Priority.NewInstance(0), 0);
Token containerToken = TestRPC.NewContainerToken(nodeId, Sharpen.Runtime.GetBytesForString
("password"), containerTokenIdentifier);
StartContainerRequest scRequest = StartContainerRequest.NewInstance(containerLaunchContext
, containerToken);
IList <StartContainerRequest> list = new AList <StartContainerRequest>();
list.AddItem(scRequest);
StartContainersRequest allRequests = StartContainersRequest.NewInstance(list);
try
{
proxy.StartContainers(allRequests);
}
catch (Exception e)
{
Log.Info(StringUtils.StringifyException(e));
NUnit.Framework.Assert.AreEqual("Error, exception is not: " + typeof(SocketTimeoutException
).FullName, typeof(SocketTimeoutException).FullName, e.GetType().FullName);
return;
}
}
finally
{
server.Stop();
}
NUnit.Framework.Assert.Fail("timeout exception should have occurred!");
}
public virtual void TestBuildTokenServiceSockAddr()
{
SecurityUtil.SetTokenServiceUseIp(true);
Assert.Equal("127.0.0.1:123", SecurityUtil.BuildTokenService(new
IPEndPoint("LocalHost", 123)).ToString());
Assert.Equal("127.0.0.1:123", SecurityUtil.BuildTokenService(new
IPEndPoint("127.0.0.1", 123)).ToString());
// what goes in, comes out
Assert.Equal("127.0.0.1:123", SecurityUtil.BuildTokenService(NetUtils
.CreateSocketAddr("127.0.0.1", 123)).ToString());
}
/// <summary>create the service name for a Delegation token</summary>
/// <param name="uri">of the service</param>
/// <param name="defPort">is used if the uri lacks a port</param>
/// <returns>the token service, or null if no authority</returns>
/// <seealso cref="BuildTokenService(System.Net.IPEndPoint)"/>
public static string BuildDTServiceName(URI uri, int defPort)
{
string authority = uri.GetAuthority();
if (authority == null)
{
return(null);
}
IPEndPoint addr = NetUtils.CreateSocketAddr(authority, defPort);
return(BuildTokenService(addr).ToString());
}
/// <exception cref="System.Exception"/>
protected override void ServiceInit(Configuration conf)
{
this.clientServiceBindAddress = TestClientRedirect.Rmaddress;
/*
* clientServiceBindAddress = conf.get(
* YarnConfiguration.APPSMANAGER_ADDRESS,
* YarnConfiguration.DEFAULT_APPSMANAGER_BIND_ADDRESS);
*/
this.clientBindAddress = NetUtils.CreateSocketAddr(this.clientServiceBindAddress);
base.ServiceInit(conf);
}
/// <seealso cref="Org.Apache.Hadoop.Hdfs.DFSUtil.GetHttpPolicy(Org.Apache.Hadoop.Conf.Configuration)
/// ">
/// for information related to the different configuration options and
/// Http Policy is decided.
/// </seealso>
/// <exception cref="System.IO.IOException"/>
internal virtual void Start()
{
HttpConfig.Policy policy = DFSUtil.GetHttpPolicy(conf);
string infoHost = bindAddress.GetHostName();
IPEndPoint httpAddr = bindAddress;
string httpsAddrString = conf.GetTrimmed(DFSConfigKeys.DfsNamenodeHttpsAddressKey
, DFSConfigKeys.DfsNamenodeHttpsAddressDefault);
IPEndPoint httpsAddr = NetUtils.CreateSocketAddr(httpsAddrString);
if (httpsAddr != null)
{
// If DFS_NAMENODE_HTTPS_BIND_HOST_KEY exists then it overrides the
// host name portion of DFS_NAMENODE_HTTPS_ADDRESS_KEY.
string bindHost = conf.GetTrimmed(DFSConfigKeys.DfsNamenodeHttpsBindHostKey);
if (bindHost != null && !bindHost.IsEmpty())
{
httpsAddr = new IPEndPoint(bindHost, httpsAddr.Port);
}
}
HttpServer2.Builder builder = DFSUtil.HttpServerTemplateForNNAndJN(conf, httpAddr
, httpsAddr, "hdfs", DFSConfigKeys.DfsNamenodeKerberosInternalSpnegoPrincipalKey
, DFSConfigKeys.DfsNamenodeKeytabFileKey);
httpServer = builder.Build();
if (policy.IsHttpsEnabled())
{
// assume same ssl port for all datanodes
IPEndPoint datanodeSslPort = NetUtils.CreateSocketAddr(conf.GetTrimmed(DFSConfigKeys
.DfsDatanodeHttpsAddressKey, infoHost + ":" + DFSConfigKeys.DfsDatanodeHttpsDefaultPort
));
httpServer.SetAttribute(DFSConfigKeys.DfsDatanodeHttpsPortKey, datanodeSslPort.Port
);
}
InitWebHdfs(conf);
httpServer.SetAttribute(NamenodeAttributeKey, nn);
httpServer.SetAttribute(JspHelper.CurrentConf, conf);
SetupServlets(httpServer, conf);
httpServer.Start();
int connIdx = 0;
if (policy.IsHttpEnabled())
{
httpAddress = httpServer.GetConnectorAddress(connIdx++);
conf.Set(DFSConfigKeys.DfsNamenodeHttpAddressKey, NetUtils.GetHostPortString(httpAddress
));
}
if (policy.IsHttpsEnabled())
{
httpsAddress = httpServer.GetConnectorAddress(connIdx);
conf.Set(DFSConfigKeys.DfsNamenodeHttpsAddressKey, NetUtils.GetHostPortString(httpsAddress
));
}
}
private static string GetHostnameForSpnegoPrincipal(Configuration conf)
{
string addr = conf.GetTrimmed(DFSConfigKeys.DfsDatanodeHttpAddressKey, null);
if (addr == null)
{
addr = conf.GetTrimmed(DFSConfigKeys.DfsDatanodeHttpsAddressKey, DFSConfigKeys.DfsDatanodeHttpsAddressDefault
);
}
IPEndPoint inetSocker = NetUtils.CreateSocketAddr(addr);
return(inetSocker.GetHostString());
}
/////////////////////////////////////////////////////////////////////////////
/// <summary>
/// <inheritDoc/>
///
/// </summary>
/// <exception cref="System.IO.IOException"/>
internal override void Init(Job job)
{
lock (this)
{
Configuration conf = job.GetConfiguration();
if (cluster == null)
{
string jobTrackerStr = conf.Get("mapreduce.jobtracker.address", "localhost:8012");
cluster = new Cluster(NetUtils.CreateSocketAddr(jobTrackerStr), conf);
}
ChooseMachine(conf).Init(job);
}
}