Example #1
0
        /// <summary>Creates the namenode proxy with the passed protocol.</summary>
        /// <remarks>
        /// Creates the namenode proxy with the passed protocol. This will handle
        /// creation of either HA- or non-HA-enabled proxy objects, depending upon
        /// if the provided URI is a configured logical URI.
        /// </remarks>
        /// <param name="conf">
        /// the configuration containing the required IPC
        /// properties, client failover configurations, etc.
        /// </param>
        /// <param name="nameNodeUri">
        /// the URI pointing either to a specific NameNode
        /// or to a logical nameservice.
        /// </param>
        /// <param name="xface">the IPC interface which should be created</param>
        /// <param name="fallbackToSimpleAuth">
        /// set to true or false during calls to indicate if
        /// a secure client falls back to simple auth
        /// </param>
        /// <returns>
        /// an object containing both the proxy and the associated
        /// delegation token service it corresponds to
        /// </returns>
        /// <exception cref="System.IO.IOException">if there is an error creating the proxy</exception>
        public static NameNodeProxies.ProxyAndInfo <T> CreateProxy <T>(Configuration conf,
                                                                       URI nameNodeUri, AtomicBoolean fallbackToSimpleAuth)
        {
            System.Type xface = typeof(T);
            AbstractNNFailoverProxyProvider <T> failoverProxyProvider = CreateFailoverProxyProvider
                                                                            (conf, nameNodeUri, xface, true, fallbackToSimpleAuth);

            if (failoverProxyProvider == null)
            {
                // Non-HA case
                return(CreateNonHAProxy(conf, NameNode.GetAddress(nameNodeUri), xface, UserGroupInformation
                                        .GetCurrentUser(), true, fallbackToSimpleAuth));
            }
            else
            {
                // HA case
                DFSClient.Conf config = new DFSClient.Conf(conf);
                T proxy = (T)RetryProxy.Create(xface, failoverProxyProvider, RetryPolicies.FailoverOnNetworkException
                                                   (RetryPolicies.TryOnceThenFail, config.maxFailoverAttempts, config.maxRetryAttempts
                                                   , config.failoverSleepBaseMillis, config.failoverSleepMaxMillis));
                Text dtService;
                if (failoverProxyProvider.UseLogicalURI())
                {
                    dtService = HAUtil.BuildTokenServiceForLogicalUri(nameNodeUri, HdfsConstants.HdfsUriScheme
                                                                      );
                }
                else
                {
                    dtService = SecurityUtil.BuildTokenService(NameNode.GetAddress(nameNodeUri));
                }
                return(new NameNodeProxies.ProxyAndInfo <T>(proxy, dtService, NameNode.GetAddress(
                                                                nameNodeUri)));
            }
        }
        public virtual void SetupCluster()
        {
            conf = new Configuration();
            conf.SetInt(DFSConfigKeys.DfsHaTaileditsPeriodKey, 1);
            HAUtil.SetAllowStandbyReads(conf, true);
            fsHelper = new FileSystemTestHelper();
            string testRoot = fsHelper.GetTestRootDir();

            testRootDir = new FilePath(testRoot).GetAbsoluteFile();
            conf.Set(DFSConfigKeys.DfsEncryptionKeyProviderUri, JavaKeyStoreProvider.SchemeName
                     + "://file" + new Path(testRootDir.ToString(), "test.jks").ToUri());
            cluster = new MiniDFSCluster.Builder(conf).NnTopology(MiniDFSNNTopology.SimpleHATopology
                                                                      ()).NumDataNodes(1).Build();
            cluster.WaitActive();
            cluster.TransitionToActive(0);
            fs = (DistributedFileSystem)HATestUtil.ConfigureFailoverFs(cluster, conf);
            DFSTestUtil.CreateKey(TestKey, cluster, 0, conf);
            DFSTestUtil.CreateKey(TestKey, cluster, 1, conf);
            nn0       = cluster.GetNameNode(0);
            nn1       = cluster.GetNameNode(1);
            dfsAdmin0 = new HdfsAdmin(cluster.GetURI(0), conf);
            dfsAdmin1 = new HdfsAdmin(cluster.GetURI(1), conf);
            KeyProviderCryptoExtension nn0Provider = cluster.GetNameNode(0).GetNamesystem().GetProvider
                                                         ();

            fs.GetClient().SetKeyProvider(nn0Provider);
        }
        public virtual void TestIPFailoverProxyProviderLogicalUri()
        {
            // setup the config with the IP failover proxy provider class
            Configuration config = new HdfsConfiguration(conf);
            URI           nnUri  = cluster.GetURI(0);

            config.Set(DFSConfigKeys.DfsClientFailoverProxyProviderKeyPrefix + "." + nnUri.GetHost
                           (), typeof(IPFailoverProxyProvider).FullName);
            NUnit.Framework.Assert.IsFalse("IPFailoverProxyProvider should not use logical URI."
                                           , HAUtil.UseLogicalUri(config, nnUri));
        }
Example #4
0
        public virtual void TestHANameNodesWithFederation()
        {
            HdfsConfiguration conf       = new HdfsConfiguration();
            string            Ns1Nn1Host = "ns1-nn1.example.com:8020";
            string            Ns1Nn2Host = "ns1-nn2.example.com:8020";
            string            Ns2Nn1Host = "ns2-nn1.example.com:8020";
            string            Ns2Nn2Host = "ns2-nn2.example.com:8020";

            conf.Set(CommonConfigurationKeys.FsDefaultNameKey, "hdfs://ns1");
            // Two nameservices, each with two NNs.
            conf.Set(DFSConfigKeys.DfsNameservices, "ns1,ns2");
            conf.Set(DFSUtil.AddKeySuffixes(DFSConfigKeys.DfsHaNamenodesKeyPrefix, "ns1"), "ns1-nn1,ns1-nn2"
                     );
            conf.Set(DFSUtil.AddKeySuffixes(DFSConfigKeys.DfsHaNamenodesKeyPrefix, "ns2"), "ns2-nn1,ns2-nn2"
                     );
            conf.Set(DFSUtil.AddKeySuffixes(DFSConfigKeys.DfsNamenodeRpcAddressKey, "ns1", "ns1-nn1"
                                            ), Ns1Nn1Host);
            conf.Set(DFSUtil.AddKeySuffixes(DFSConfigKeys.DfsNamenodeRpcAddressKey, "ns1", "ns1-nn2"
                                            ), Ns1Nn2Host);
            conf.Set(DFSUtil.AddKeySuffixes(DFSConfigKeys.DfsNamenodeRpcAddressKey, "ns2", "ns2-nn1"
                                            ), Ns2Nn1Host);
            conf.Set(DFSUtil.AddKeySuffixes(DFSConfigKeys.DfsNamenodeRpcAddressKey, "ns2", "ns2-nn2"
                                            ), Ns2Nn2Host);
            IDictionary <string, IDictionary <string, IPEndPoint> > map = DFSUtil.GetHaNnRpcAddresses
                                                                              (conf);

            NUnit.Framework.Assert.IsTrue(HAUtil.IsHAEnabled(conf, "ns1"));
            NUnit.Framework.Assert.IsTrue(HAUtil.IsHAEnabled(conf, "ns2"));
            NUnit.Framework.Assert.IsFalse(HAUtil.IsHAEnabled(conf, "ns3"));
            NUnit.Framework.Assert.AreEqual(Ns1Nn1Host, map["ns1"]["ns1-nn1"].ToString());
            NUnit.Framework.Assert.AreEqual(Ns1Nn2Host, map["ns1"]["ns1-nn2"].ToString());
            NUnit.Framework.Assert.AreEqual(Ns2Nn1Host, map["ns2"]["ns2-nn1"].ToString());
            NUnit.Framework.Assert.AreEqual(Ns2Nn2Host, map["ns2"]["ns2-nn2"].ToString());
            NUnit.Framework.Assert.AreEqual(Ns1Nn1Host, DFSUtil.GetNamenodeServiceAddr(conf,
                                                                                       "ns1", "ns1-nn1"));
            NUnit.Framework.Assert.AreEqual(Ns1Nn2Host, DFSUtil.GetNamenodeServiceAddr(conf,
                                                                                       "ns1", "ns1-nn2"));
            NUnit.Framework.Assert.AreEqual(Ns2Nn1Host, DFSUtil.GetNamenodeServiceAddr(conf,
                                                                                       "ns2", "ns2-nn1"));
            // No nameservice was given and we can't determine which service addr
            // to use as two nameservices could share a namenode ID.
            NUnit.Framework.Assert.AreEqual(null, DFSUtil.GetNamenodeServiceAddr(conf, null,
                                                                                 "ns1-nn1"));
            // Ditto for nameservice IDs, if multiple are defined
            NUnit.Framework.Assert.AreEqual(null, DFSUtil.GetNamenodeNameServiceId(conf));
            NUnit.Framework.Assert.AreEqual(null, DFSUtil.GetSecondaryNameServiceId(conf));
            ICollection <URI> uris = DFSUtil.GetNameServiceUris(conf, DFSConfigKeys.DfsNamenodeRpcAddressKey
                                                                );

            NUnit.Framework.Assert.AreEqual(2, uris.Count);
            NUnit.Framework.Assert.IsTrue(uris.Contains(new URI("hdfs://ns1")));
            NUnit.Framework.Assert.IsTrue(uris.Contains(new URI("hdfs://ns2")));
        }
Example #5
0
        /// <summary>
        /// Generate a dummy namenode proxy instance that utilizes our hacked
        /// <see cref="Org.Apache.Hadoop.IO.Retry.LossyRetryInvocationHandler{T}"/>
        /// . Proxy instance generated using this
        /// method will proactively drop RPC responses. Currently this method only
        /// support HA setup. null will be returned if the given configuration is not
        /// for HA.
        /// </summary>
        /// <param name="config">
        /// the configuration containing the required IPC
        /// properties, client failover configurations, etc.
        /// </param>
        /// <param name="nameNodeUri">
        /// the URI pointing either to a specific NameNode
        /// or to a logical nameservice.
        /// </param>
        /// <param name="xface">the IPC interface which should be created</param>
        /// <param name="numResponseToDrop">The number of responses to drop for each RPC call
        ///     </param>
        /// <param name="fallbackToSimpleAuth">
        /// set to true or false during calls to indicate if
        /// a secure client falls back to simple auth
        /// </param>
        /// <returns>
        /// an object containing both the proxy and the associated
        /// delegation token service it corresponds to. Will return null of the
        /// given configuration does not support HA.
        /// </returns>
        /// <exception cref="System.IO.IOException">if there is an error creating the proxy</exception>
        public static NameNodeProxies.ProxyAndInfo <T> CreateProxyWithLossyRetryHandler <T>
            (Configuration config, URI nameNodeUri, int numResponseToDrop, AtomicBoolean fallbackToSimpleAuth
            )
        {
            System.Type xface = typeof(T);
            Preconditions.CheckArgument(numResponseToDrop > 0);
            AbstractNNFailoverProxyProvider <T> failoverProxyProvider = CreateFailoverProxyProvider
                                                                            (config, nameNodeUri, xface, true, fallbackToSimpleAuth);

            if (failoverProxyProvider != null)
            {
                // HA case
                int delay = config.GetInt(DFSConfigKeys.DfsClientFailoverSleeptimeBaseKey, DFSConfigKeys
                                          .DfsClientFailoverSleeptimeBaseDefault);
                int maxCap = config.GetInt(DFSConfigKeys.DfsClientFailoverSleeptimeMaxKey, DFSConfigKeys
                                           .DfsClientFailoverSleeptimeMaxDefault);
                int maxFailoverAttempts = config.GetInt(DFSConfigKeys.DfsClientFailoverMaxAttemptsKey
                                                        , DFSConfigKeys.DfsClientFailoverMaxAttemptsDefault);
                int maxRetryAttempts = config.GetInt(DFSConfigKeys.DfsClientRetryMaxAttemptsKey,
                                                     DFSConfigKeys.DfsClientRetryMaxAttemptsDefault);
                InvocationHandler dummyHandler = new LossyRetryInvocationHandler <T>(numResponseToDrop
                                                                                     , failoverProxyProvider, RetryPolicies.FailoverOnNetworkException(RetryPolicies.
                                                                                                                                                       TryOnceThenFail, maxFailoverAttempts, Math.Max(numResponseToDrop + 1, maxRetryAttempts
                                                                                                                                                                                                      ), delay, maxCap));
                T proxy = (T)Proxy.NewProxyInstance(failoverProxyProvider.GetInterface().GetClassLoader
                                                        (), new Type[] { xface }, dummyHandler);
                Text dtService;
                if (failoverProxyProvider.UseLogicalURI())
                {
                    dtService = HAUtil.BuildTokenServiceForLogicalUri(nameNodeUri, HdfsConstants.HdfsUriScheme
                                                                      );
                }
                else
                {
                    dtService = SecurityUtil.BuildTokenService(NameNode.GetAddress(nameNodeUri));
                }
                return(new NameNodeProxies.ProxyAndInfo <T>(proxy, dtService, NameNode.GetAddress(
                                                                nameNodeUri)));
            }
            else
            {
                Log.Warn("Currently creating proxy using " + "LossyRetryInvocationHandler requires NN HA setup"
                         );
                return(null);
            }
        }
Example #6
0
        public virtual void TestMultipleNamenodes()
        {
            HdfsConfiguration conf = new HdfsConfiguration();

            conf.Set(DFSConfigKeys.DfsNameservices, "nn1,nn2");
            // Test - configured list of namenodes are returned
            string Nn1Address = "localhost:9000";
            string Nn2Address = "localhost:9001";
            string Nn3Address = "localhost:9002";

            conf.Set(DFSUtil.AddKeySuffixes(DFSConfigKeys.DfsNamenodeRpcAddressKey, "nn1"), Nn1Address
                     );
            conf.Set(DFSUtil.AddKeySuffixes(DFSConfigKeys.DfsNamenodeRpcAddressKey, "nn2"), Nn2Address
                     );
            IDictionary <string, IDictionary <string, IPEndPoint> > nnMap = DFSUtil.GetNNServiceRpcAddresses
                                                                                (conf);

            NUnit.Framework.Assert.AreEqual(2, nnMap.Count);
            IDictionary <string, IPEndPoint> nn1Map = nnMap["nn1"];

            NUnit.Framework.Assert.AreEqual(1, nn1Map.Count);
            IPEndPoint addr = nn1Map[null];

            NUnit.Framework.Assert.AreEqual("localhost", addr.GetHostName());
            NUnit.Framework.Assert.AreEqual(9000, addr.Port);
            IDictionary <string, IPEndPoint> nn2Map = nnMap["nn2"];

            NUnit.Framework.Assert.AreEqual(1, nn2Map.Count);
            addr = nn2Map[null];
            NUnit.Framework.Assert.AreEqual("localhost", addr.GetHostName());
            NUnit.Framework.Assert.AreEqual(9001, addr.Port);
            // Test - can look up nameservice ID from service address
            CheckNameServiceId(conf, Nn1Address, "nn1");
            CheckNameServiceId(conf, Nn2Address, "nn2");
            CheckNameServiceId(conf, Nn3Address, null);
            // HA is not enabled in a purely federated config
            NUnit.Framework.Assert.IsFalse(HAUtil.IsHAEnabled(conf, "nn1"));
            NUnit.Framework.Assert.IsFalse(HAUtil.IsHAEnabled(conf, "nn2"));
        }
        public virtual void TestWrappedFailoverProxyProvider()
        {
            // setup the config with the dummy provider class
            Configuration config      = new HdfsConfiguration(conf);
            string        logicalName = HATestUtil.GetLogicalHostname(cluster);

            HATestUtil.SetFailoverConfigurations(cluster, config, logicalName);
            config.Set(DFSConfigKeys.DfsClientFailoverProxyProviderKeyPrefix + "." + logicalName
                       , typeof(TestDFSClientFailover.DummyLegacyFailoverProxyProvider).FullName);
            Path p = new Path("hdfs://" + logicalName + "/");

            // not to use IP address for token service
            SecurityUtil.SetTokenServiceUseIp(false);
            // Logical URI should be used.
            NUnit.Framework.Assert.IsTrue("Legacy proxy providers should use logical URI.", HAUtil
                                          .UseLogicalUri(config, p.ToUri()));
        }