Esempio n. 1
0
        /// <summary>Creates the namenode proxy with the passed protocol.</summary>
        /// <remarks>
        /// Creates the namenode proxy with the passed protocol. This will handle
        /// creation of either HA- or non-HA-enabled proxy objects, depending upon
        /// if the provided URI is a configured logical URI.
        /// </remarks>
        /// <param name="conf">
        /// the configuration containing the required IPC
        /// properties, client failover configurations, etc.
        /// </param>
        /// <param name="nameNodeUri">
        /// the URI pointing either to a specific NameNode
        /// or to a logical nameservice.
        /// </param>
        /// <param name="xface">the IPC interface which should be created</param>
        /// <param name="fallbackToSimpleAuth">
        /// set to true or false during calls to indicate if
        /// a secure client falls back to simple auth
        /// </param>
        /// <returns>
        /// an object containing both the proxy and the associated
        /// delegation token service it corresponds to
        /// </returns>
        /// <exception cref="System.IO.IOException">if there is an error creating the proxy</exception>
        public static NameNodeProxies.ProxyAndInfo <T> CreateProxy <T>(Configuration conf,
                                                                       URI nameNodeUri, AtomicBoolean fallbackToSimpleAuth)
        {
            System.Type xface = typeof(T);
            AbstractNNFailoverProxyProvider <T> failoverProxyProvider = CreateFailoverProxyProvider
                                                                            (conf, nameNodeUri, xface, true, fallbackToSimpleAuth);

            if (failoverProxyProvider == null)
            {
                // Non-HA case
                return(CreateNonHAProxy(conf, NameNode.GetAddress(nameNodeUri), xface, UserGroupInformation
                                        .GetCurrentUser(), true, fallbackToSimpleAuth));
            }
            else
            {
                // HA case
                DFSClient.Conf config = new DFSClient.Conf(conf);
                T proxy = (T)RetryProxy.Create(xface, failoverProxyProvider, RetryPolicies.FailoverOnNetworkException
                                                   (RetryPolicies.TryOnceThenFail, config.maxFailoverAttempts, config.maxRetryAttempts
                                                   , config.failoverSleepBaseMillis, config.failoverSleepMaxMillis));
                Text dtService;
                if (failoverProxyProvider.UseLogicalURI())
                {
                    dtService = HAUtil.BuildTokenServiceForLogicalUri(nameNodeUri, HdfsConstants.HdfsUriScheme
                                                                      );
                }
                else
                {
                    dtService = SecurityUtil.BuildTokenService(NameNode.GetAddress(nameNodeUri));
                }
                return(new NameNodeProxies.ProxyAndInfo <T>(proxy, dtService, NameNode.GetAddress(
                                                                nameNodeUri)));
            }
        }
Esempio n. 2
0
        /// <summary>
        /// Check whether logical URI is needed for the namenode and
        /// the corresponding failover proxy provider in the config.
        /// </summary>
        /// <param name="conf">Configuration</param>
        /// <param name="nameNodeUri">The URI of namenode</param>
        /// <returns>true if logical URI is needed. false, if not needed.</returns>
        /// <exception cref="System.IO.IOException">most likely due to misconfiguration.</exception>
        public static bool UseLogicalUri(Configuration conf, URI nameNodeUri)
        {
            // Create the proxy provider. Actual proxy is not created.
            AbstractNNFailoverProxyProvider <ClientProtocol> provider = NameNodeProxies.CreateFailoverProxyProvider
                                                                        <ClientProtocol>(conf, nameNodeUri, false, null);

            // No need to use logical URI since failover is not configured.
            if (provider == null)
            {
                return(false);
            }
            // Check whether the failover proxy provider uses logical URI.
            return(provider.UseLogicalURI());
        }
Esempio n. 3
0
        /// <summary>
        /// Generate a dummy namenode proxy instance that utilizes our hacked
        /// <see cref="Org.Apache.Hadoop.IO.Retry.LossyRetryInvocationHandler{T}"/>
        /// . Proxy instance generated using this
        /// method will proactively drop RPC responses. Currently this method only
        /// support HA setup. null will be returned if the given configuration is not
        /// for HA.
        /// </summary>
        /// <param name="config">
        /// the configuration containing the required IPC
        /// properties, client failover configurations, etc.
        /// </param>
        /// <param name="nameNodeUri">
        /// the URI pointing either to a specific NameNode
        /// or to a logical nameservice.
        /// </param>
        /// <param name="xface">the IPC interface which should be created</param>
        /// <param name="numResponseToDrop">The number of responses to drop for each RPC call
        ///     </param>
        /// <param name="fallbackToSimpleAuth">
        /// set to true or false during calls to indicate if
        /// a secure client falls back to simple auth
        /// </param>
        /// <returns>
        /// an object containing both the proxy and the associated
        /// delegation token service it corresponds to. Will return null of the
        /// given configuration does not support HA.
        /// </returns>
        /// <exception cref="System.IO.IOException">if there is an error creating the proxy</exception>
        public static NameNodeProxies.ProxyAndInfo <T> CreateProxyWithLossyRetryHandler <T>
            (Configuration config, URI nameNodeUri, int numResponseToDrop, AtomicBoolean fallbackToSimpleAuth
            )
        {
            System.Type xface = typeof(T);
            Preconditions.CheckArgument(numResponseToDrop > 0);
            AbstractNNFailoverProxyProvider <T> failoverProxyProvider = CreateFailoverProxyProvider
                                                                            (config, nameNodeUri, xface, true, fallbackToSimpleAuth);

            if (failoverProxyProvider != null)
            {
                // HA case
                int delay = config.GetInt(DFSConfigKeys.DfsClientFailoverSleeptimeBaseKey, DFSConfigKeys
                                          .DfsClientFailoverSleeptimeBaseDefault);
                int maxCap = config.GetInt(DFSConfigKeys.DfsClientFailoverSleeptimeMaxKey, DFSConfigKeys
                                           .DfsClientFailoverSleeptimeMaxDefault);
                int maxFailoverAttempts = config.GetInt(DFSConfigKeys.DfsClientFailoverMaxAttemptsKey
                                                        , DFSConfigKeys.DfsClientFailoverMaxAttemptsDefault);
                int maxRetryAttempts = config.GetInt(DFSConfigKeys.DfsClientRetryMaxAttemptsKey,
                                                     DFSConfigKeys.DfsClientRetryMaxAttemptsDefault);
                InvocationHandler dummyHandler = new LossyRetryInvocationHandler <T>(numResponseToDrop
                                                                                     , failoverProxyProvider, RetryPolicies.FailoverOnNetworkException(RetryPolicies.
                                                                                                                                                       TryOnceThenFail, maxFailoverAttempts, Math.Max(numResponseToDrop + 1, maxRetryAttempts
                                                                                                                                                                                                      ), delay, maxCap));
                T proxy = (T)Proxy.NewProxyInstance(failoverProxyProvider.GetInterface().GetClassLoader
                                                        (), new Type[] { xface }, dummyHandler);
                Text dtService;
                if (failoverProxyProvider.UseLogicalURI())
                {
                    dtService = HAUtil.BuildTokenServiceForLogicalUri(nameNodeUri, HdfsConstants.HdfsUriScheme
                                                                      );
                }
                else
                {
                    dtService = SecurityUtil.BuildTokenService(NameNode.GetAddress(nameNodeUri));
                }
                return(new NameNodeProxies.ProxyAndInfo <T>(proxy, dtService, NameNode.GetAddress(
                                                                nameNodeUri)));
            }
            else
            {
                Log.Warn("Currently creating proxy using " + "LossyRetryInvocationHandler requires NN HA setup"
                         );
                return(null);
            }
        }