コード例 #1
0
ファイル: TestDFSUtil.cs プロジェクト: orf53975/hadoop.net
        public virtual void TestGetNNServiceRpcAddressesForNsIds()
        {
            Configuration conf = new HdfsConfiguration();

            conf.Set(DFSConfigKeys.DfsNameservices, "nn1,nn2");
            conf.Set(DFSConfigKeys.DfsInternalNameservicesKey, "nn1");
            // Test - configured list of namenodes are returned
            string Nn1Address = "localhost:9000";
            string Nn2Address = "localhost:9001";

            conf.Set(DFSUtil.AddKeySuffixes(DFSConfigKeys.DfsNamenodeRpcAddressKey, "nn1"), Nn1Address
                     );
            conf.Set(DFSUtil.AddKeySuffixes(DFSConfigKeys.DfsNamenodeRpcAddressKey, "nn2"), Nn2Address
                     );
            IDictionary <string, IDictionary <string, IPEndPoint> > nnMap = DFSUtil.GetNNServiceRpcAddressesForCluster
                                                                                (conf);

            NUnit.Framework.Assert.AreEqual(1, nnMap.Count);
            NUnit.Framework.Assert.IsTrue(nnMap.Contains("nn1"));
            conf.Set(DFSConfigKeys.DfsInternalNameservicesKey, "nn3");
            try
            {
                DFSUtil.GetNNServiceRpcAddressesForCluster(conf);
                NUnit.Framework.Assert.Fail("Should fail for misconfiguration");
            }
            catch (IOException)
            {
            }
        }
コード例 #2
0
        /// <summary>
        /// Create a test configuration that will exercise the initializeGenericKeys
        /// code path.
        /// </summary>
        /// <remarks>
        /// Create a test configuration that will exercise the initializeGenericKeys
        /// code path.  This is a regression test for HDFS-4279.
        /// </remarks>
        /// <exception cref="System.IO.IOException"/>
        internal static void SetupRecoveryTestConf(Configuration conf)
        {
            conf.Set(DFSConfigKeys.DfsNameservices, "ns1");
            conf.Set(DFSConfigKeys.DfsHaNamenodeIdKey, "nn1");
            conf.Set(DFSUtil.AddKeySuffixes(DFSConfigKeys.DfsHaNamenodesKeyPrefix, "ns1"), "nn1,nn2"
                     );
            string baseDir = Runtime.GetProperty(MiniDFSCluster.PropTestBuildData, "build/test/data"
                                                 ) + "/dfs/";
            FilePath nameDir      = new FilePath(baseDir, "nameR");
            FilePath secondaryDir = new FilePath(baseDir, "namesecondaryR");

            conf.Set(DFSUtil.AddKeySuffixes(DFSConfigKeys.DfsNamenodeNameDirKey, "ns1", "nn1"
                                            ), nameDir.GetCanonicalPath());
            conf.Set(DFSUtil.AddKeySuffixes(DFSConfigKeys.DfsNamenodeCheckpointDirKey, "ns1",
                                            "nn1"), secondaryDir.GetCanonicalPath());
            conf.Unset(DFSConfigKeys.DfsNamenodeNameDirKey);
            conf.Unset(DFSConfigKeys.DfsNamenodeCheckpointDirKey);
            FileUtils.DeleteQuietly(nameDir);
            if (!nameDir.Mkdirs())
            {
                throw new RuntimeException("failed to make directory " + nameDir.GetAbsolutePath(
                                               ));
            }
            FileUtils.DeleteQuietly(secondaryDir);
            if (!secondaryDir.Mkdirs())
            {
                throw new RuntimeException("failed to make directory " + secondaryDir.GetAbsolutePath
                                               ());
            }
        }
コード例 #3
0
ファイル: TestDFSUtil.cs プロジェクト: orf53975/hadoop.net
        public virtual void TestGetNNUris()
        {
            HdfsConfiguration conf        = new HdfsConfiguration();
            string            Ns1Nn1Addr  = "ns1-nn1.example.com:8020";
            string            Ns1Nn2Addr  = "ns1-nn2.example.com:8020";
            string            Ns2NnAddr   = "ns2-nn.example.com:8020";
            string            Nn1Addr     = "nn.example.com:8020";
            string            Nn1SrvcAddr = "nn.example.com:8021";
            string            Nn2Addr     = "nn2.example.com:8020";

            conf.Set(DFSConfigKeys.DfsNameservices, "ns1,ns2");
            conf.Set(DFSUtil.AddKeySuffixes(DFSConfigKeys.DfsHaNamenodesKeyPrefix, "ns1"), "nn1,nn2"
                     );
            conf.Set(DFSUtil.AddKeySuffixes(DFSConfigKeys.DfsNamenodeRpcAddressKey, "ns1", "nn1"
                                            ), Ns1Nn1Addr);
            conf.Set(DFSUtil.AddKeySuffixes(DFSConfigKeys.DfsNamenodeRpcAddressKey, "ns1", "nn2"
                                            ), Ns1Nn2Addr);
            conf.Set(DFSUtil.AddKeySuffixes(DFSConfigKeys.DfsNamenodeServiceRpcAddressKey, "ns2"
                                            ), Ns2NnAddr);
            conf.Set(DFSConfigKeys.DfsNamenodeRpcAddressKey, "hdfs://" + Nn1Addr);
            conf.Set(CommonConfigurationKeys.FsDefaultNameKey, "hdfs://" + Nn2Addr);
            ICollection <URI> uris = DFSUtil.GetNameServiceUris(conf, DFSConfigKeys.DfsNamenodeServiceRpcAddressKey
                                                                , DFSConfigKeys.DfsNamenodeRpcAddressKey);

            NUnit.Framework.Assert.AreEqual(4, uris.Count);
            NUnit.Framework.Assert.IsTrue(uris.Contains(new URI("hdfs://ns1")));
            NUnit.Framework.Assert.IsTrue(uris.Contains(new URI("hdfs://" + Ns2NnAddr)));
            NUnit.Framework.Assert.IsTrue(uris.Contains(new URI("hdfs://" + Nn1Addr)));
            NUnit.Framework.Assert.IsTrue(uris.Contains(new URI("hdfs://" + Nn2Addr)));
            // Make sure that non-HDFS URIs in fs.defaultFS don't get included.
            conf.Set(CommonConfigurationKeys.FsDefaultNameKey, "viewfs://vfs-name.example.com"
                     );
            uris = DFSUtil.GetNameServiceUris(conf, DFSConfigKeys.DfsNamenodeServiceRpcAddressKey
                                              , DFSConfigKeys.DfsNamenodeRpcAddressKey);
            NUnit.Framework.Assert.AreEqual(3, uris.Count);
            NUnit.Framework.Assert.IsTrue(uris.Contains(new URI("hdfs://ns1")));
            NUnit.Framework.Assert.IsTrue(uris.Contains(new URI("hdfs://" + Ns2NnAddr)));
            NUnit.Framework.Assert.IsTrue(uris.Contains(new URI("hdfs://" + Nn1Addr)));
            // Make sure that an HA URI being the default URI doesn't result in multiple
            // entries being returned.
            conf.Set(CommonConfigurationKeys.FsDefaultNameKey, "hdfs://ns1");
            uris = DFSUtil.GetNameServiceUris(conf, DFSConfigKeys.DfsNamenodeServiceRpcAddressKey
                                              , DFSConfigKeys.DfsNamenodeRpcAddressKey);
            NUnit.Framework.Assert.AreEqual(3, uris.Count);
            NUnit.Framework.Assert.IsTrue(uris.Contains(new URI("hdfs://ns1")));
            NUnit.Framework.Assert.IsTrue(uris.Contains(new URI("hdfs://" + Ns2NnAddr)));
            NUnit.Framework.Assert.IsTrue(uris.Contains(new URI("hdfs://" + Nn1Addr)));
            // Make sure that when a service RPC address is used that is distinct from
            // the client RPC address, and that client RPC address is also used as the
            // default URI, that the client URI does not end up in the set of URIs
            // returned.
            conf = new HdfsConfiguration();
            conf.Set(CommonConfigurationKeys.FsDefaultNameKey, "hdfs://" + Nn1Addr);
            conf.Set(DFSConfigKeys.DfsNamenodeRpcAddressKey, Nn1Addr);
            conf.Set(DFSConfigKeys.DfsNamenodeServiceRpcAddressKey, Nn1SrvcAddr);
            uris = DFSUtil.GetNameServiceUris(conf, DFSConfigKeys.DfsNamenodeServiceRpcAddressKey
                                              , DFSConfigKeys.DfsNamenodeRpcAddressKey);
            NUnit.Framework.Assert.AreEqual(1, uris.Count);
            NUnit.Framework.Assert.IsTrue(uris.Contains(new URI("hdfs://" + Nn1SrvcAddr)));
        }
コード例 #4
0
ファイル: TestDFSUtil.cs プロジェクト: orf53975/hadoop.net
        private Configuration SetupAddress(string key)
        {
            HdfsConfiguration conf = new HdfsConfiguration();

            conf.Set(DFSConfigKeys.DfsNameservices, "nn1");
            conf.Set(DFSUtil.AddKeySuffixes(key, "nn1"), "localhost:9000");
            return(conf);
        }
コード例 #5
0
ファイル: TestDFSUtil.cs プロジェクト: orf53975/hadoop.net
        public virtual void TestHANameNodesWithFederation()
        {
            HdfsConfiguration conf       = new HdfsConfiguration();
            string            Ns1Nn1Host = "ns1-nn1.example.com:8020";
            string            Ns1Nn2Host = "ns1-nn2.example.com:8020";
            string            Ns2Nn1Host = "ns2-nn1.example.com:8020";
            string            Ns2Nn2Host = "ns2-nn2.example.com:8020";

            conf.Set(CommonConfigurationKeys.FsDefaultNameKey, "hdfs://ns1");
            // Two nameservices, each with two NNs.
            conf.Set(DFSConfigKeys.DfsNameservices, "ns1,ns2");
            conf.Set(DFSUtil.AddKeySuffixes(DFSConfigKeys.DfsHaNamenodesKeyPrefix, "ns1"), "ns1-nn1,ns1-nn2"
                     );
            conf.Set(DFSUtil.AddKeySuffixes(DFSConfigKeys.DfsHaNamenodesKeyPrefix, "ns2"), "ns2-nn1,ns2-nn2"
                     );
            conf.Set(DFSUtil.AddKeySuffixes(DFSConfigKeys.DfsNamenodeRpcAddressKey, "ns1", "ns1-nn1"
                                            ), Ns1Nn1Host);
            conf.Set(DFSUtil.AddKeySuffixes(DFSConfigKeys.DfsNamenodeRpcAddressKey, "ns1", "ns1-nn2"
                                            ), Ns1Nn2Host);
            conf.Set(DFSUtil.AddKeySuffixes(DFSConfigKeys.DfsNamenodeRpcAddressKey, "ns2", "ns2-nn1"
                                            ), Ns2Nn1Host);
            conf.Set(DFSUtil.AddKeySuffixes(DFSConfigKeys.DfsNamenodeRpcAddressKey, "ns2", "ns2-nn2"
                                            ), Ns2Nn2Host);
            IDictionary <string, IDictionary <string, IPEndPoint> > map = DFSUtil.GetHaNnRpcAddresses
                                                                              (conf);

            NUnit.Framework.Assert.IsTrue(HAUtil.IsHAEnabled(conf, "ns1"));
            NUnit.Framework.Assert.IsTrue(HAUtil.IsHAEnabled(conf, "ns2"));
            NUnit.Framework.Assert.IsFalse(HAUtil.IsHAEnabled(conf, "ns3"));
            NUnit.Framework.Assert.AreEqual(Ns1Nn1Host, map["ns1"]["ns1-nn1"].ToString());
            NUnit.Framework.Assert.AreEqual(Ns1Nn2Host, map["ns1"]["ns1-nn2"].ToString());
            NUnit.Framework.Assert.AreEqual(Ns2Nn1Host, map["ns2"]["ns2-nn1"].ToString());
            NUnit.Framework.Assert.AreEqual(Ns2Nn2Host, map["ns2"]["ns2-nn2"].ToString());
            NUnit.Framework.Assert.AreEqual(Ns1Nn1Host, DFSUtil.GetNamenodeServiceAddr(conf,
                                                                                       "ns1", "ns1-nn1"));
            NUnit.Framework.Assert.AreEqual(Ns1Nn2Host, DFSUtil.GetNamenodeServiceAddr(conf,
                                                                                       "ns1", "ns1-nn2"));
            NUnit.Framework.Assert.AreEqual(Ns2Nn1Host, DFSUtil.GetNamenodeServiceAddr(conf,
                                                                                       "ns2", "ns2-nn1"));
            // No nameservice was given and we can't determine which service addr
            // to use as two nameservices could share a namenode ID.
            NUnit.Framework.Assert.AreEqual(null, DFSUtil.GetNamenodeServiceAddr(conf, null,
                                                                                 "ns1-nn1"));
            // Ditto for nameservice IDs, if multiple are defined
            NUnit.Framework.Assert.AreEqual(null, DFSUtil.GetNamenodeNameServiceId(conf));
            NUnit.Framework.Assert.AreEqual(null, DFSUtil.GetSecondaryNameServiceId(conf));
            ICollection <URI> uris = DFSUtil.GetNameServiceUris(conf, DFSConfigKeys.DfsNamenodeRpcAddressKey
                                                                );

            NUnit.Framework.Assert.AreEqual(2, uris.Count);
            NUnit.Framework.Assert.IsTrue(uris.Contains(new URI("hdfs://ns1")));
            NUnit.Framework.Assert.IsTrue(uris.Contains(new URI("hdfs://ns2")));
        }
コード例 #6
0
ファイル: TestDFSUtil.cs プロジェクト: orf53975/hadoop.net
        /// <summary>
        /// Test
        /// <see cref="DFSUtil.GetNamenodeNameServiceId(Org.Apache.Hadoop.Conf.Configuration)
        ///     "/>
        /// to ensure
        /// exception is thrown when multiple rpc addresses match the local node's
        /// address
        /// </summary>
        public virtual void TestGetNameServiceIdException()
        {
            HdfsConfiguration conf = new HdfsConfiguration();

            conf.Set(DFSConfigKeys.DfsNameservices, "nn1,nn2");
            conf.Set(DFSUtil.AddKeySuffixes(DFSConfigKeys.DfsNamenodeRpcAddressKey, "nn1"), "localhost:9000"
                     );
            conf.Set(DFSUtil.AddKeySuffixes(DFSConfigKeys.DfsNamenodeRpcAddressKey, "nn2"), "localhost:9001"
                     );
            DFSUtil.GetNamenodeNameServiceId(conf);
            NUnit.Framework.Assert.Fail("Expected exception is not thrown");
        }
コード例 #7
0
 private void SetHAConf(Configuration conf, string nn1Addr, string nn2Addr)
 {
     conf.Set(CommonConfigurationKeysPublic.FsDefaultNameKey, "hdfs://" + Nsid);
     conf.Set(DFSConfigKeys.DfsNameservices, Nsid);
     conf.Set(DFSConfigKeys.DfsNameserviceId, Nsid);
     conf.Set(DFSUtil.AddKeySuffixes(DFSConfigKeys.DfsHaNamenodesKeyPrefix, Nsid), "nn1,nn2"
              );
     conf.Set(DFSConfigKeys.DfsHaNamenodeIdKey, "nn1");
     conf.Set(DFSUtil.AddKeySuffixes(DFSConfigKeys.DfsNamenodeRpcAddressKey, Nsid, "nn1"
                                     ), nn1Addr);
     conf.Set(DFSUtil.AddKeySuffixes(DFSConfigKeys.DfsNamenodeRpcAddressKey, Nsid, "nn2"
                                     ), nn2Addr);
 }
コード例 #8
0
 /// <summary>Set a given key with value as address, for all the nameServiceIds.</summary>
 /// <param name="conf">configuration to set the addresses in</param>
 /// <param name="key">configuration key</param>
 /// <param name="nameServiceIdCount">Number of nameServices for which the key is set</param>
 /// <param name="portOffset">starting port offset</param>
 /// <returns>list of addresses that are set in the configuration</returns>
 private string[] SetupAddress(HdfsConfiguration conf, string key, int nameServiceIdCount
                               , int portOffset)
 {
     string[] values = new string[nameServiceIdCount];
     for (int i = 0; i < nameServiceIdCount; i++, portOffset++)
     {
         string nsID        = GetNameServiceId(i);
         string specificKey = DFSUtil.AddKeySuffixes(key, nsID);
         values[i] = "nn" + i + ":" + portOffset;
         conf.Set(specificKey, values[i]);
     }
     return(values);
 }
コード例 #9
0
        public virtual void TestInitializeSharedEditsConfiguresGenericConfKeys()
        {
            Configuration conf = new Configuration();

            conf.Set(DFSConfigKeys.DfsNameservices, "ns1");
            conf.Set(DFSUtil.AddKeySuffixes(DFSConfigKeys.DfsHaNamenodesKeyPrefix, "ns1"), "nn1,nn2"
                     );
            conf.Set(DFSUtil.AddKeySuffixes(DFSConfigKeys.DfsNamenodeRpcAddressKey, "ns1", "nn1"
                                            ), "localhost:1234");
            NUnit.Framework.Assert.IsNull(conf.Get(DFSConfigKeys.DfsNamenodeRpcAddressKey));
            NameNode.InitializeSharedEdits(conf);
            NUnit.Framework.Assert.IsNotNull(conf.Get(DFSConfigKeys.DfsNamenodeRpcAddressKey)
                                             );
        }
コード例 #10
0
        private Configuration GetHAConf(string nsId, string host1, string host2)
        {
            Configuration conf = new Configuration();

            conf.Set(DFSConfigKeys.DfsNameservices, nsId);
            conf.Set(DFSUtil.AddKeySuffixes(DFSConfigKeys.DfsHaNamenodesKeyPrefix, nsId), "nn1,nn2"
                     );
            conf.Set(DFSConfigKeys.DfsHaNamenodeIdKey, "nn1");
            conf.Set(DFSUtil.AddKeySuffixes(DFSConfigKeys.DfsNamenodeRpcAddressKey, nsId, "nn1"
                                            ), host1 + ":12345");
            conf.Set(DFSUtil.AddKeySuffixes(DFSConfigKeys.DfsNamenodeRpcAddressKey, nsId, "nn2"
                                            ), host2 + ":12345");
            return(conf);
        }
コード例 #11
0
ファイル: TestDFSHAAdmin.cs プロジェクト: orf53975/hadoop.net
        // Fencer shell commands that always return true and false respectively
        // on Unix.
        // Fencer shell commands that always return true and false respectively
        // on Windows. Lacking POSIX 'true' and 'false' commands we use the DOS
        // commands 'rem' and 'help.exe'.
        private HdfsConfiguration GetHAConf()
        {
            HdfsConfiguration conf = new HdfsConfiguration();

            conf.Set(DFSConfigKeys.DfsNameservices, Nsid);
            conf.Set(DFSConfigKeys.DfsNameserviceId, Nsid);
            conf.Set(DFSUtil.AddKeySuffixes(DFSConfigKeys.DfsHaNamenodesKeyPrefix, Nsid), "nn1,nn2"
                     );
            conf.Set(DFSConfigKeys.DfsHaNamenodeIdKey, "nn1");
            conf.Set(DFSUtil.AddKeySuffixes(DFSConfigKeys.DfsNamenodeRpcAddressKey, Nsid, "nn1"
                                            ), HostA + ":12345");
            conf.Set(DFSUtil.AddKeySuffixes(DFSConfigKeys.DfsNamenodeRpcAddressKey, Nsid, "nn2"
                                            ), HostB + ":12345");
            return(conf);
        }
コード例 #12
0
ファイル: TestDFSUtil.cs プロジェクト: orf53975/hadoop.net
        private static Configuration CreateWebHDFSHAConfiguration(string logicalHostName,
                                                                  string nnaddr1, string nnaddr2)
        {
            HdfsConfiguration conf = new HdfsConfiguration();

            conf.Set(DFSConfigKeys.DfsNameservices, "ns1");
            conf.Set(DFSUtil.AddKeySuffixes(DFSConfigKeys.DfsHaNamenodesKeyPrefix, "ns1"), "nn1,nn2"
                     );
            conf.Set(DFSUtil.AddKeySuffixes(DFSConfigKeys.DfsNamenodeHttpAddressKey, "ns1", "nn1"
                                            ), nnaddr1);
            conf.Set(DFSUtil.AddKeySuffixes(DFSConfigKeys.DfsNamenodeHttpAddressKey, "ns1", "nn2"
                                            ), nnaddr2);
            conf.Set(DFSConfigKeys.DfsClientFailoverProxyProviderKeyPrefix + "." + logicalHostName
                     , typeof(ConfiguredFailoverProxyProvider).FullName);
            return(conf);
        }
コード例 #13
0
        public virtual void TestIsValidRequestor()
        {
            Configuration conf = new HdfsConfiguration();

            KerberosName.SetRules("RULE:[1:$1]\nRULE:[2:$1]");
            // Set up generic HA configs.
            conf.Set(DFSConfigKeys.DfsNameservices, "ns1");
            conf.Set(DFSUtil.AddKeySuffixes(DFSConfigKeys.DfsHaNamenodesKeyPrefix, "ns1"), "nn1,nn2"
                     );
            // Set up NN1 HA configs.
            conf.Set(DFSUtil.AddKeySuffixes(DFSConfigKeys.DfsNamenodeRpcAddressKey, "ns1", "nn1"
                                            ), "host1:1234");
            conf.Set(DFSUtil.AddKeySuffixes(DFSConfigKeys.DfsNamenodeKerberosPrincipalKey, "ns1"
                                            , "nn1"), "hdfs/[email protected]");
            // Set up NN2 HA configs.
            conf.Set(DFSUtil.AddKeySuffixes(DFSConfigKeys.DfsNamenodeRpcAddressKey, "ns1", "nn2"
                                            ), "host2:1234");
            conf.Set(DFSUtil.AddKeySuffixes(DFSConfigKeys.DfsNamenodeKerberosPrincipalKey, "ns1"
                                            , "nn2"), "hdfs/[email protected]");
            // Initialize this conf object as though we're running on NN1.
            NameNode.InitializeGenericKeys(conf, "ns1", "nn1");
            AccessControlList acls = Org.Mockito.Mockito.Mock <AccessControlList>();

            Org.Mockito.Mockito.When(acls.IsUserAllowed(Org.Mockito.Mockito.Any <UserGroupInformation
                                                                                 >())).ThenReturn(false);
            ServletContext context = Org.Mockito.Mockito.Mock <ServletContext>();

            Org.Mockito.Mockito.When(context.GetAttribute(HttpServer2.AdminsAcl)).ThenReturn(
                acls);
            // Make sure that NN2 is considered a valid fsimage/edits requestor.
            NUnit.Framework.Assert.IsTrue(ImageServlet.IsValidRequestor(context, "hdfs/[email protected]"
                                                                        , conf));
            // Mark atm as an admin.
            Org.Mockito.Mockito.When(acls.IsUserAllowed(Org.Mockito.Mockito.ArgThat(new _ArgumentMatcher_76
                                                                                        ()))).ThenReturn(true);
            // Make sure that NN2 is still considered a valid requestor.
            NUnit.Framework.Assert.IsTrue(ImageServlet.IsValidRequestor(context, "hdfs/[email protected]"
                                                                        , conf));
            // Make sure an admin is considered a valid requestor.
            NUnit.Framework.Assert.IsTrue(ImageServlet.IsValidRequestor(context, "*****@*****.**"
                                                                        , conf));
            // Make sure other users are *not* considered valid requestors.
            NUnit.Framework.Assert.IsFalse(ImageServlet.IsValidRequestor(context, "*****@*****.**"
                                                                         , conf));
        }
コード例 #14
0
        private Configuration InitHAConf(URI journalURI, Configuration conf)
        {
            conf.Set(DFSConfigKeys.DfsNamenodeSharedEditsDirKey, journalURI.ToString());
            string address1 = "127.0.0.1:" + basePort;
            string address2 = "127.0.0.1:" + (basePort + 2);

            conf.Set(DFSUtil.AddKeySuffixes(DFSConfigKeys.DfsNamenodeRpcAddressKey, Nameservice
                                            , Nn1), address1);
            conf.Set(DFSUtil.AddKeySuffixes(DFSConfigKeys.DfsNamenodeRpcAddressKey, Nameservice
                                            , Nn2), address2);
            conf.Set(DFSConfigKeys.DfsNameservices, Nameservice);
            conf.Set(DFSUtil.AddKeySuffixes(DFSConfigKeys.DfsHaNamenodesKeyPrefix, Nameservice
                                            ), Nn1 + "," + Nn2);
            conf.Set(DFSConfigKeys.DfsClientFailoverProxyProviderKeyPrefix + "." + Nameservice
                     , typeof(ConfiguredFailoverProxyProvider).FullName);
            conf.Set("fs.defaultFS", "hdfs://" + Nameservice);
            return(conf);
        }
コード例 #15
0
ファイル: HATestUtil.cs プロジェクト: orf53975/hadoop.net
        /// <summary>Sets the required configurations for performing failover</summary>
        public static void SetFailoverConfigurations(Configuration conf, string logicalName
                                                     , IPEndPoint nnAddr1, IPEndPoint nnAddr2)
        {
            string nameNodeId1 = "nn1";
            string nameNodeId2 = "nn2";
            string address1    = "hdfs://" + nnAddr1.GetHostName() + ":" + nnAddr1.Port;
            string address2    = "hdfs://" + nnAddr2.GetHostName() + ":" + nnAddr2.Port;

            conf.Set(DFSUtil.AddKeySuffixes(DFSConfigKeys.DfsNamenodeRpcAddressKey, logicalName
                                            , nameNodeId1), address1);
            conf.Set(DFSUtil.AddKeySuffixes(DFSConfigKeys.DfsNamenodeRpcAddressKey, logicalName
                                            , nameNodeId2), address2);
            conf.Set(DFSConfigKeys.DfsNameservices, logicalName);
            conf.Set(DFSUtil.AddKeySuffixes(DFSConfigKeys.DfsHaNamenodesKeyPrefix, logicalName
                                            ), nameNodeId1 + "," + nameNodeId2);
            conf.Set(DFSConfigKeys.DfsClientFailoverProxyProviderKeyPrefix + "." + logicalName
                     , typeof(ConfiguredFailoverProxyProvider).FullName);
            conf.Set("fs.defaultFS", "hdfs://" + logicalName);
        }
コード例 #16
0
ファイル: TestDFSUtil.cs プロジェクト: orf53975/hadoop.net
        public virtual void TestMultipleNamenodes()
        {
            HdfsConfiguration conf = new HdfsConfiguration();

            conf.Set(DFSConfigKeys.DfsNameservices, "nn1,nn2");
            // Test - configured list of namenodes are returned
            string Nn1Address = "localhost:9000";
            string Nn2Address = "localhost:9001";
            string Nn3Address = "localhost:9002";

            conf.Set(DFSUtil.AddKeySuffixes(DFSConfigKeys.DfsNamenodeRpcAddressKey, "nn1"), Nn1Address
                     );
            conf.Set(DFSUtil.AddKeySuffixes(DFSConfigKeys.DfsNamenodeRpcAddressKey, "nn2"), Nn2Address
                     );
            IDictionary <string, IDictionary <string, IPEndPoint> > nnMap = DFSUtil.GetNNServiceRpcAddresses
                                                                                (conf);

            NUnit.Framework.Assert.AreEqual(2, nnMap.Count);
            IDictionary <string, IPEndPoint> nn1Map = nnMap["nn1"];

            NUnit.Framework.Assert.AreEqual(1, nn1Map.Count);
            IPEndPoint addr = nn1Map[null];

            NUnit.Framework.Assert.AreEqual("localhost", addr.GetHostName());
            NUnit.Framework.Assert.AreEqual(9000, addr.Port);
            IDictionary <string, IPEndPoint> nn2Map = nnMap["nn2"];

            NUnit.Framework.Assert.AreEqual(1, nn2Map.Count);
            addr = nn2Map[null];
            NUnit.Framework.Assert.AreEqual("localhost", addr.GetHostName());
            NUnit.Framework.Assert.AreEqual(9001, addr.Port);
            // Test - can look up nameservice ID from service address
            CheckNameServiceId(conf, Nn1Address, "nn1");
            CheckNameServiceId(conf, Nn2Address, "nn2");
            CheckNameServiceId(conf, Nn3Address, null);
            // HA is not enabled in a purely federated config
            NUnit.Framework.Assert.IsFalse(HAUtil.IsHAEnabled(conf, "nn1"));
            NUnit.Framework.Assert.IsFalse(HAUtil.IsHAEnabled(conf, "nn2"));
        }
コード例 #17
0
ファイル: TestDFSUtil.cs プロジェクト: orf53975/hadoop.net
        public virtual void TestConfModificationFederationOnly()
        {
            HdfsConfiguration conf = new HdfsConfiguration();
            string            nsId = "ns1";

            conf.Set(DFSConfigKeys.DfsNameservices, nsId);
            conf.Set(DFSConfigKeys.DfsNameserviceId, nsId);
            // Set the nameservice specific keys with nameserviceId in the config key
            foreach (string key in NameNode.NamenodeSpecificKeys)
            {
                // Note: value is same as the key
                conf.Set(DFSUtil.AddKeySuffixes(key, nsId), key);
            }
            // Initialize generic keys from specific keys
            NameNode.InitializeGenericKeys(conf, nsId, null);
            // Retrieve the keys without nameserviceId and Ensure generic keys are set
            // to the correct value
            foreach (string key_1 in NameNode.NamenodeSpecificKeys)
            {
                NUnit.Framework.Assert.AreEqual(key_1, conf.Get(key_1));
            }
        }
コード例 #18
0
ファイル: TestDFSUtil.cs プロジェクト: orf53975/hadoop.net
        public virtual void GetNameNodeServiceAddr()
        {
            HdfsConfiguration conf = new HdfsConfiguration();
            // One nameservice with two NNs
            string Ns1Nn1Host    = "ns1-nn1.example.com:8020";
            string Ns1Nn1HostSvc = "ns1-nn2.example.com:8021";
            string Ns1Nn2Host    = "ns1-nn1.example.com:8020";
            string Ns1Nn2HostSvc = "ns1-nn2.example.com:8021";

            conf.Set(DFSConfigKeys.DfsNameservices, "ns1");
            conf.Set(DFSUtil.AddKeySuffixes(DFSConfigKeys.DfsHaNamenodesKeyPrefix, "ns1"), "nn1,nn2"
                     );
            conf.Set(DFSUtil.AddKeySuffixes(DFSConfigKeys.DfsNamenodeRpcAddressKey, "ns1", "nn1"
                                            ), Ns1Nn1Host);
            conf.Set(DFSUtil.AddKeySuffixes(DFSConfigKeys.DfsNamenodeRpcAddressKey, "ns1", "nn2"
                                            ), Ns1Nn2Host);
            // The rpc address is used if no service address is defined
            NUnit.Framework.Assert.AreEqual(Ns1Nn1Host, DFSUtil.GetNamenodeServiceAddr(conf,
                                                                                       null, "nn1"));
            NUnit.Framework.Assert.AreEqual(Ns1Nn2Host, DFSUtil.GetNamenodeServiceAddr(conf,
                                                                                       null, "nn2"));
            // A nameservice is specified explicitly
            NUnit.Framework.Assert.AreEqual(Ns1Nn1Host, DFSUtil.GetNamenodeServiceAddr(conf,
                                                                                       "ns1", "nn1"));
            NUnit.Framework.Assert.AreEqual(null, DFSUtil.GetNamenodeServiceAddr(conf, "invalid"
                                                                                 , "nn1"));
            // The service addrs are used when they are defined
            conf.Set(DFSUtil.AddKeySuffixes(DFSConfigKeys.DfsNamenodeServiceRpcAddressKey, "ns1"
                                            , "nn1"), Ns1Nn1HostSvc);
            conf.Set(DFSUtil.AddKeySuffixes(DFSConfigKeys.DfsNamenodeServiceRpcAddressKey, "ns1"
                                            , "nn2"), Ns1Nn2HostSvc);
            NUnit.Framework.Assert.AreEqual(Ns1Nn1HostSvc, DFSUtil.GetNamenodeServiceAddr(conf
                                                                                          , null, "nn1"));
            NUnit.Framework.Assert.AreEqual(Ns1Nn2HostSvc, DFSUtil.GetNamenodeServiceAddr(conf
                                                                                          , null, "nn2"));
            // We can determine the nameservice ID, there's only one listed
            NUnit.Framework.Assert.AreEqual("ns1", DFSUtil.GetNamenodeNameServiceId(conf));
            NUnit.Framework.Assert.AreEqual("ns1", DFSUtil.GetSecondaryNameServiceId(conf));
        }
コード例 #19
0
ファイル: HAUtil.cs プロジェクト: orf53975/hadoop.net
        /// <summary>Get the NN ID of the other node in an HA setup.</summary>
        /// <param name="conf">the configuration of this node</param>
        /// <returns>the NN ID of the other node in this nameservice</returns>
        public static string GetNameNodeIdOfOtherNode(Configuration conf, string nsId)
        {
            Preconditions.CheckArgument(nsId != null, "Could not determine namespace id. Please ensure that this "
                                        + "machine is one of the machines listed as a NN RPC address, " + "or configure "
                                        + DFSConfigKeys.DfsNameserviceId);
            ICollection <string> nnIds = DFSUtil.GetNameNodeIds(conf, nsId);
            string myNNId = conf.Get(DFSConfigKeys.DfsHaNamenodeIdKey);

            Preconditions.CheckArgument(nnIds != null, "Could not determine namenode ids in namespace '%s'. "
                                        + "Please configure " + DFSUtil.AddKeySuffixes(DFSConfigKeys.DfsHaNamenodesKeyPrefix
                                                                                       , nsId), nsId);
            Preconditions.CheckArgument(nnIds.Count == 2, "Expected exactly 2 NameNodes in namespace '%s'. "
                                        + "Instead, got only %s (NN ids were '%s'", nsId, nnIds.Count, Joiner.On("','")
                                        .Join(nnIds));
            Preconditions.CheckState(myNNId != null && !myNNId.IsEmpty(), "Could not determine own NN ID in namespace '%s'. Please "
                                     + "ensure that this node is one of the machines listed as an " + "NN RPC address, or configure "
                                     + DFSConfigKeys.DfsHaNamenodeIdKey, nsId);
            AList <string> nnSet = Lists.NewArrayList(nnIds);

            nnSet.Remove(myNNId);
            System.Diagnostics.Debug.Assert(nnSet.Count == 1);
            return(nnSet[0]);
        }
コード例 #20
0
        public virtual void TestFormatShouldBeIgnoredForNonFileBasedDirs()
        {
            Configuration conf        = new HdfsConfiguration();
            string        logicalName = "mycluster";
            // DFS_NAMENODE_RPC_ADDRESS_KEY are required to identify the NameNode
            // is configured in HA, then only DFS_NAMENODE_SHARED_EDITS_DIR_KEY
            // is considered.
            string     localhost = "127.0.0.1";
            IPEndPoint nnAddr1   = new IPEndPoint(localhost, 8020);
            IPEndPoint nnAddr2   = new IPEndPoint(localhost, 9020);

            HATestUtil.SetFailoverConfigurations(conf, logicalName, nnAddr1, nnAddr2);
            conf.Set(DFSConfigKeys.DfsNamenodeNameDirKey, new FilePath(DfsBaseDir, "name").GetAbsolutePath
                         ());
            conf.SetBoolean(DFSConfigKeys.DfsNamenodeSupportAllowFormatKey, true);
            conf.Set(DFSUtil.AddKeySuffixes(DFSConfigKeys.DfsNamenodeEditsPluginPrefix, "dummy"
                                            ), typeof(TestGenericJournalConf.DummyJournalManager).FullName);
            conf.Set(DFSConfigKeys.DfsNamenodeSharedEditsDirKey, "dummy://" + localhost + ":2181/ledgers"
                     );
            conf.Set(DFSConfigKeys.DfsHaNamenodeIdKey, "nn1");
            // An internal assert is added to verify the working of test
            NameNode.Format(conf);
        }
コード例 #21
0
        private static void AddNN(Configuration conf, string ns, string addr)
        {
            string key = DFSUtil.AddKeySuffixes(DFSConfigKeys.DfsNamenodeRpcAddressKey, ns);

            conf.Set(key, addr);
        }