/// <summary>Creates the namenode proxy with the passed protocol.</summary>
/// <remarks>
/// Creates the namenode proxy with the passed protocol. This will handle
/// creation of either HA- or non-HA-enabled proxy objects, depending upon
/// if the provided URI is a configured logical URI.
/// </remarks>
/// <param name="conf">
/// the configuration containing the required IPC
/// properties, client failover configurations, etc.
/// </param>
/// <param name="nameNodeUri">
/// the URI pointing either to a specific NameNode
/// or to a logical nameservice.
/// </param>
/// <param name="xface">the IPC interface which should be created</param>
/// <param name="fallbackToSimpleAuth">
/// set to true or false during calls to indicate if
/// a secure client falls back to simple auth
/// </param>
/// <returns>
/// an object containing both the proxy and the associated
/// delegation token service it corresponds to
/// </returns>
/// <exception cref="System.IO.IOException">if there is an error creating the proxy</exception>
public static NameNodeProxies.ProxyAndInfo <T> CreateProxy <T>(Configuration conf,
URI nameNodeUri, AtomicBoolean fallbackToSimpleAuth)
{
System.Type xface = typeof(T);
AbstractNNFailoverProxyProvider <T> failoverProxyProvider = CreateFailoverProxyProvider
(conf, nameNodeUri, xface, true, fallbackToSimpleAuth);
if (failoverProxyProvider == null)
{
// Non-HA case
return(CreateNonHAProxy(conf, NameNode.GetAddress(nameNodeUri), xface, UserGroupInformation
.GetCurrentUser(), true, fallbackToSimpleAuth));
}
else
{
// HA case
DFSClient.Conf config = new DFSClient.Conf(conf);
T proxy = (T)RetryProxy.Create(xface, failoverProxyProvider, RetryPolicies.FailoverOnNetworkException
(RetryPolicies.TryOnceThenFail, config.maxFailoverAttempts, config.maxRetryAttempts
, config.failoverSleepBaseMillis, config.failoverSleepMaxMillis));
Text dtService;
if (failoverProxyProvider.UseLogicalURI())
{
dtService = HAUtil.BuildTokenServiceForLogicalUri(nameNodeUri, HdfsConstants.HdfsUriScheme
);
}
else
{
dtService = SecurityUtil.BuildTokenService(NameNode.GetAddress(nameNodeUri));
}
return(new NameNodeProxies.ProxyAndInfo <T>(proxy, dtService, NameNode.GetAddress(
nameNodeUri)));
}
}
/// <summary>
/// Generate a dummy namenode proxy instance that utilizes our hacked
/// <see cref="Org.Apache.Hadoop.IO.Retry.LossyRetryInvocationHandler{T}"/>
/// . Proxy instance generated using this
/// method will proactively drop RPC responses. Currently this method only
/// support HA setup. null will be returned if the given configuration is not
/// for HA.
/// </summary>
/// <param name="config">
/// the configuration containing the required IPC
/// properties, client failover configurations, etc.
/// </param>
/// <param name="nameNodeUri">
/// the URI pointing either to a specific NameNode
/// or to a logical nameservice.
/// </param>
/// <param name="xface">the IPC interface which should be created</param>
/// <param name="numResponseToDrop">The number of responses to drop for each RPC call
/// </param>
/// <param name="fallbackToSimpleAuth">
/// set to true or false during calls to indicate if
/// a secure client falls back to simple auth
/// </param>
/// <returns>
/// an object containing both the proxy and the associated
/// delegation token service it corresponds to. Will return null of the
/// given configuration does not support HA.
/// </returns>
/// <exception cref="System.IO.IOException">if there is an error creating the proxy</exception>
public static NameNodeProxies.ProxyAndInfo <T> CreateProxyWithLossyRetryHandler <T>
(Configuration config, URI nameNodeUri, int numResponseToDrop, AtomicBoolean fallbackToSimpleAuth
)
{
System.Type xface = typeof(T);
Preconditions.CheckArgument(numResponseToDrop > 0);
AbstractNNFailoverProxyProvider <T> failoverProxyProvider = CreateFailoverProxyProvider
(config, nameNodeUri, xface, true, fallbackToSimpleAuth);
if (failoverProxyProvider != null)
{
// HA case
int delay = config.GetInt(DFSConfigKeys.DfsClientFailoverSleeptimeBaseKey, DFSConfigKeys
.DfsClientFailoverSleeptimeBaseDefault);
int maxCap = config.GetInt(DFSConfigKeys.DfsClientFailoverSleeptimeMaxKey, DFSConfigKeys
.DfsClientFailoverSleeptimeMaxDefault);
int maxFailoverAttempts = config.GetInt(DFSConfigKeys.DfsClientFailoverMaxAttemptsKey
, DFSConfigKeys.DfsClientFailoverMaxAttemptsDefault);
int maxRetryAttempts = config.GetInt(DFSConfigKeys.DfsClientRetryMaxAttemptsKey,
DFSConfigKeys.DfsClientRetryMaxAttemptsDefault);
InvocationHandler dummyHandler = new LossyRetryInvocationHandler <T>(numResponseToDrop
, failoverProxyProvider, RetryPolicies.FailoverOnNetworkException(RetryPolicies.
TryOnceThenFail, maxFailoverAttempts, Math.Max(numResponseToDrop + 1, maxRetryAttempts
), delay, maxCap));
T proxy = (T)Proxy.NewProxyInstance(failoverProxyProvider.GetInterface().GetClassLoader
(), new Type[] { xface }, dummyHandler);
Text dtService;
if (failoverProxyProvider.UseLogicalURI())
{
dtService = HAUtil.BuildTokenServiceForLogicalUri(nameNodeUri, HdfsConstants.HdfsUriScheme
);
}
else
{
dtService = SecurityUtil.BuildTokenService(NameNode.GetAddress(nameNodeUri));
}
return(new NameNodeProxies.ProxyAndInfo <T>(proxy, dtService, NameNode.GetAddress(
nameNodeUri)));
}
else
{
Log.Warn("Currently creating proxy using " + "LossyRetryInvocationHandler requires NN HA setup"
);
return(null);
}
}
public static RetryPolicy CreateRetryPolicy(Configuration conf)
{
long rmConnectWaitMS = conf.GetLong(YarnConfiguration.ResourcemanagerConnectMaxWaitMs
, YarnConfiguration.DefaultResourcemanagerConnectMaxWaitMs);
long rmConnectionRetryIntervalMS = conf.GetLong(YarnConfiguration.ResourcemanagerConnectRetryIntervalMs
, YarnConfiguration.DefaultResourcemanagerConnectRetryIntervalMs);
bool waitForEver = (rmConnectWaitMS == -1);
if (!waitForEver)
{
if (rmConnectWaitMS < 0)
{
throw new YarnRuntimeException("Invalid Configuration. " + YarnConfiguration.ResourcemanagerConnectMaxWaitMs
+ " can be -1, but can not be other negative numbers");
}
// try connect once
if (rmConnectWaitMS < rmConnectionRetryIntervalMS)
{
Log.Warn(YarnConfiguration.ResourcemanagerConnectMaxWaitMs + " is smaller than "
+ YarnConfiguration.ResourcemanagerConnectRetryIntervalMs + ". Only try connect once."
);
rmConnectWaitMS = 0;
}
}
// Handle HA case first
if (HAUtil.IsHAEnabled(conf))
{
long failoverSleepBaseMs = conf.GetLong(YarnConfiguration.ClientFailoverSleeptimeBaseMs
, rmConnectionRetryIntervalMS);
long failoverSleepMaxMs = conf.GetLong(YarnConfiguration.ClientFailoverSleeptimeMaxMs
, rmConnectionRetryIntervalMS);
int maxFailoverAttempts = conf.GetInt(YarnConfiguration.ClientFailoverMaxAttempts
, -1);
if (maxFailoverAttempts == -1)
{
if (waitForEver)
{
maxFailoverAttempts = int.MaxValue;
}
else
{
maxFailoverAttempts = (int)(rmConnectWaitMS / failoverSleepBaseMs);
}
}
return(RetryPolicies.FailoverOnNetworkException(RetryPolicies.TryOnceThenFail, maxFailoverAttempts
, failoverSleepBaseMs, failoverSleepMaxMs));
}
if (rmConnectionRetryIntervalMS < 0)
{
throw new YarnRuntimeException("Invalid Configuration. " + YarnConfiguration.ResourcemanagerConnectRetryIntervalMs
+ " should not be negative.");
}
RetryPolicy retryPolicy = null;
if (waitForEver)
{
retryPolicy = RetryPolicies.RetryForever;
}
else
{
retryPolicy = RetryPolicies.RetryUpToMaximumTimeWithFixedSleep(rmConnectWaitMS, rmConnectionRetryIntervalMS
, TimeUnit.Milliseconds);
}
IDictionary <Type, RetryPolicy> exceptionToPolicyMap = new Dictionary <Type, RetryPolicy
>();
exceptionToPolicyMap[typeof(EOFException)] = retryPolicy;
exceptionToPolicyMap[typeof(ConnectException)] = retryPolicy;
exceptionToPolicyMap[typeof(NoRouteToHostException)] = retryPolicy;
exceptionToPolicyMap[typeof(UnknownHostException)] = retryPolicy;
exceptionToPolicyMap[typeof(ConnectTimeoutException)] = retryPolicy;
exceptionToPolicyMap[typeof(RetriableException)] = retryPolicy;
exceptionToPolicyMap[typeof(SocketException)] = retryPolicy;
return(RetryPolicies.RetryByException(RetryPolicies.TryOnceThenFail, exceptionToPolicyMap
));
}
