Ejemplo n.º 1
0
        public virtual void TestRefreshCallQueueProtocol()
        {
            RefreshCallQueueProtocolClientSideTranslatorPB translator = (RefreshCallQueueProtocolClientSideTranslatorPB
                                                                         )NameNodeProxies.CreateNonHAProxy <RefreshCallQueueProtocol>(conf, nnAddress, UserGroupInformation
                                                                                                                                      .GetCurrentUser(), true).GetProxy();

            NUnit.Framework.Assert.IsTrue(translator.IsMethodSupported("refreshCallQueue"));
        }
Ejemplo n.º 2
0
        public virtual void TestGetUserMappingsProtocol()
        {
            GetUserMappingsProtocolClientSideTranslatorPB translator = (GetUserMappingsProtocolClientSideTranslatorPB
                                                                        )NameNodeProxies.CreateNonHAProxy <GetUserMappingsProtocol>(conf, nnAddress, UserGroupInformation
                                                                                                                                    .GetCurrentUser(), true).GetProxy();

            NUnit.Framework.Assert.IsTrue(translator.IsMethodSupported("getGroupsForUser"));
        }
Ejemplo n.º 3
0
        public virtual void TestClientNamenodeProtocol()
        {
            ClientProtocol cp = NameNodeProxies.CreateNonHAProxy <ClientProtocol>(conf, nnAddress
                                                                                  , UserGroupInformation.GetCurrentUser(), true).GetProxy();

            RpcClientUtil.IsMethodSupported(cp, typeof(ClientNamenodeProtocolPB), RPC.RpcKind
                                            .RpcProtocolBuffer, RPC.GetProtocolVersion(typeof(ClientNamenodeProtocolPB)), "mkdirs"
                                            );
        }
Ejemplo n.º 4
0
        public virtual void TesJournalProtocol()
        {
            JournalProtocolTranslatorPB translator = (JournalProtocolTranslatorPB)NameNodeProxies
                                                     .CreateNonHAProxy <JournalProtocol>(conf, nnAddress, UserGroupInformation.GetCurrentUser
                                                                                             (), true).GetProxy();

            //Nameode doesn't implement JournalProtocol
            NUnit.Framework.Assert.IsFalse(translator.IsMethodSupported("startLogSegment"));
        }
Ejemplo n.º 5
0
 public DummyLegacyFailoverProxyProvider(Configuration conf, URI uri, Type xface)
 {
     try
     {
         this.proxy = NameNodeProxies.CreateNonHAProxy(conf, NameNode.GetAddress(uri), xface
                                                       , UserGroupInformation.GetCurrentUser(), false).GetProxy();
         this.xface = xface;
     }
     catch (IOException)
     {
     }
 }
Ejemplo n.º 6
0
        /// <summary>
        /// Check whether logical URI is needed for the namenode and
        /// the corresponding failover proxy provider in the config.
        /// </summary>
        /// <param name="conf">Configuration</param>
        /// <param name="nameNodeUri">The URI of namenode</param>
        /// <returns>true if logical URI is needed. false, if not needed.</returns>
        /// <exception cref="System.IO.IOException">most likely due to misconfiguration.</exception>
        public static bool UseLogicalUri(Configuration conf, URI nameNodeUri)
        {
            // Create the proxy provider. Actual proxy is not created.
            AbstractNNFailoverProxyProvider <ClientProtocol> provider = NameNodeProxies.CreateFailoverProxyProvider
                                                                        <ClientProtocol>(conf, nameNodeUri, false, null);

            // No need to use logical URI since failover is not configured.
            if (provider == null)
            {
                return(false);
            }
            // Check whether the failover proxy provider uses logical URI.
            return(provider.UseLogicalURI());
        }
Ejemplo n.º 7
0
        public virtual void TestNamenodeProtocol()
        {
            NamenodeProtocol np = NameNodeProxies.CreateNonHAProxy <NamenodeProtocol>(conf, nnAddress
                                                                                      , UserGroupInformation.GetCurrentUser(), true).GetProxy();
            bool exists = RpcClientUtil.IsMethodSupported(np, typeof(NamenodeProtocolPB), RPC.RpcKind
                                                          .RpcProtocolBuffer, RPC.GetProtocolVersion(typeof(NamenodeProtocolPB)), "rollEditLog"
                                                          );

            NUnit.Framework.Assert.IsTrue(exists);
            exists = RpcClientUtil.IsMethodSupported(np, typeof(NamenodeProtocolPB), RPC.RpcKind
                                                     .RpcProtocolBuffer, RPC.GetProtocolVersion(typeof(NamenodeProtocolPB)), "bogusMethod"
                                                     );
            NUnit.Framework.Assert.IsFalse(exists);
        }
Ejemplo n.º 8
0
        /// <summary>Get an RPC proxy for each NN in an HA nameservice.</summary>
        /// <remarks>
        /// Get an RPC proxy for each NN in an HA nameservice. Used when a given RPC
        /// call should be made on every NN in an HA nameservice, not just the active.
        /// </remarks>
        /// <param name="conf">configuration</param>
        /// <param name="nsId">the nameservice to get all of the proxies for.</param>
        /// <param name="xface">the protocol class.</param>
        /// <returns>a list of RPC proxies for each NN in the nameservice.</returns>
        /// <exception cref="System.IO.IOException">in the event of error.</exception>
        public static IList <NameNodeProxies.ProxyAndInfo <T> > GetProxiesForAllNameNodesInNameservice
        <T>(Configuration conf, string nsId)
        {
            System.Type xface = typeof(T);
            IDictionary <string, IPEndPoint> nnAddresses = DFSUtil.GetRpcAddressesForNameserviceId
                                                               (conf, nsId, null);
            IList <NameNodeProxies.ProxyAndInfo <T> > proxies = new AList <NameNodeProxies.ProxyAndInfo
                                                                           <T> >(nnAddresses.Count);

            foreach (IPEndPoint nnAddress in nnAddresses.Values)
            {
                NameNodeProxies.ProxyAndInfo <T> proxyInfo = null;
                proxyInfo = NameNodeProxies.CreateNonHAProxy(conf, nnAddress, xface, UserGroupInformation
                                                             .GetCurrentUser(), false);
                proxies.AddItem(proxyInfo);
            }
            return(proxies);
        }
Ejemplo n.º 9
0
        /* check if there are at least two nodes are on the same rack */
        /// <exception cref="System.IO.IOException"/>
        private void CheckFile(FileSystem fileSys, Path name, int repl)
        {
            Configuration  conf     = fileSys.GetConf();
            ClientProtocol namenode = NameNodeProxies.CreateProxy <ClientProtocol>(conf, fileSys
                                                                                   .GetUri()).GetProxy();

            WaitForBlockReplication(name.ToString(), namenode, Math.Min(numDatanodes, repl),
                                    -1);
            LocatedBlocks locations = namenode.GetBlockLocations(name.ToString(), 0, long.MaxValue
                                                                 );
            FileStatus stat = fileSys.GetFileStatus(name);

            BlockLocation[] blockLocations = fileSys.GetFileBlockLocations(stat, 0L, long.MaxValue
                                                                           );
            // verify that rack locations match
            NUnit.Framework.Assert.IsTrue(blockLocations.Length == locations.LocatedBlockCount
                                              ());
            for (int i = 0; i < blockLocations.Length; i++)
            {
                LocatedBlock   blk           = locations.Get(i);
                DatanodeInfo[] datanodes     = blk.GetLocations();
                string[]       topologyPaths = blockLocations[i].GetTopologyPaths();
                NUnit.Framework.Assert.IsTrue(topologyPaths.Length == datanodes.Length);
                for (int j = 0; j < topologyPaths.Length; j++)
                {
                    bool found = false;
                    for (int k = 0; k < racks.Length; k++)
                    {
                        if (topologyPaths[j].StartsWith(racks[k]))
                        {
                            found = true;
                            break;
                        }
                    }
                    NUnit.Framework.Assert.IsTrue(found);
                }
            }
            bool isOnSameRack    = true;
            bool isNotOnSameRack = true;

            foreach (LocatedBlock blk_1 in locations.GetLocatedBlocks())
            {
                DatanodeInfo[] datanodes = blk_1.GetLocations();
                if (datanodes.Length <= 1)
                {
                    break;
                }
                if (datanodes.Length == 2)
                {
                    isNotOnSameRack = !(datanodes[0].GetNetworkLocation().Equals(datanodes[1].GetNetworkLocation
                                                                                     ()));
                    break;
                }
                isOnSameRack    = false;
                isNotOnSameRack = false;
                for (int i_1 = 0; i_1 < datanodes.Length - 1; i_1++)
                {
                    Log.Info("datanode " + i_1 + ": " + datanodes[i_1]);
                    bool onRack = false;
                    for (int j = i_1 + 1; j < datanodes.Length; j++)
                    {
                        if (datanodes[i_1].GetNetworkLocation().Equals(datanodes[j].GetNetworkLocation()))
                        {
                            onRack = true;
                        }
                    }
                    if (onRack)
                    {
                        isOnSameRack = true;
                    }
                    if (!onRack)
                    {
                        isNotOnSameRack = true;
                    }
                    if (isOnSameRack && isNotOnSameRack)
                    {
                        break;
                    }
                }
                if (!isOnSameRack || !isNotOnSameRack)
                {
                    break;
                }
            }
            NUnit.Framework.Assert.IsTrue(isOnSameRack);
            NUnit.Framework.Assert.IsTrue(isNotOnSameRack);
        }
Ejemplo n.º 10
0
        public virtual void TestGetBlocks()
        {
            Configuration Conf = new HdfsConfiguration();
            short         ReplicationFactor = (short)2;
            int           DefaultBlockSize  = 1024;
            Random        r = new Random();

            Conf.SetLong(DFSConfigKeys.DfsBlockSizeKey, DefaultBlockSize);
            MiniDFSCluster cluster = new MiniDFSCluster.Builder(Conf).NumDataNodes(ReplicationFactor
                                                                                   ).Build();

            try
            {
                cluster.WaitActive();
                // create a file with two blocks
                FileSystem         fs   = cluster.GetFileSystem();
                FSDataOutputStream @out = fs.Create(new Path("/tmp.txt"), ReplicationFactor);
                byte[]             data = new byte[1024];
                long fileLen            = 2 * DefaultBlockSize;
                long bytesToWrite       = fileLen;
                while (bytesToWrite > 0)
                {
                    r.NextBytes(data);
                    int bytesToWriteNext = (1024 < bytesToWrite) ? 1024 : (int)bytesToWrite;
                    @out.Write(data, 0, bytesToWriteNext);
                    bytesToWrite -= bytesToWriteNext;
                }
                @out.Close();
                // get blocks & data nodes
                IList <LocatedBlock> locatedBlocks;
                DatanodeInfo[]       dataNodes = null;
                bool notWritten;
                do
                {
                    DFSClient dfsclient = new DFSClient(NameNode.GetAddress(Conf), Conf);
                    locatedBlocks = dfsclient.GetNamenode().GetBlockLocations("/tmp.txt", 0, fileLen)
                                    .GetLocatedBlocks();
                    NUnit.Framework.Assert.AreEqual(2, locatedBlocks.Count);
                    notWritten = false;
                    for (int i = 0; i < 2; i++)
                    {
                        dataNodes = locatedBlocks[i].GetLocations();
                        if (dataNodes.Length != ReplicationFactor)
                        {
                            notWritten = true;
                            try
                            {
                                Sharpen.Thread.Sleep(10);
                            }
                            catch (Exception)
                            {
                            }
                            break;
                        }
                    }
                }while (notWritten);
                // get RPC client to namenode
                IPEndPoint       addr     = new IPEndPoint("localhost", cluster.GetNameNodePort());
                NamenodeProtocol namenode = NameNodeProxies.CreateProxy <NamenodeProtocol>(Conf, NameNode
                                                                                           .GetUri(addr)).GetProxy();
                // get blocks of size fileLen from dataNodes[0]
                BlocksWithLocations.BlockWithLocations[] locs;
                locs = namenode.GetBlocks(dataNodes[0], fileLen).GetBlocks();
                NUnit.Framework.Assert.AreEqual(locs.Length, 2);
                NUnit.Framework.Assert.AreEqual(locs[0].GetStorageIDs().Length, 2);
                NUnit.Framework.Assert.AreEqual(locs[1].GetStorageIDs().Length, 2);
                // get blocks of size BlockSize from dataNodes[0]
                locs = namenode.GetBlocks(dataNodes[0], DefaultBlockSize).GetBlocks();
                NUnit.Framework.Assert.AreEqual(locs.Length, 1);
                NUnit.Framework.Assert.AreEqual(locs[0].GetStorageIDs().Length, 2);
                // get blocks of size 1 from dataNodes[0]
                locs = namenode.GetBlocks(dataNodes[0], 1).GetBlocks();
                NUnit.Framework.Assert.AreEqual(locs.Length, 1);
                NUnit.Framework.Assert.AreEqual(locs[0].GetStorageIDs().Length, 2);
                // get blocks of size 0 from dataNodes[0]
                GetBlocksWithException(namenode, dataNodes[0], 0);
                // get blocks of size -1 from dataNodes[0]
                GetBlocksWithException(namenode, dataNodes[0], -1);
                // get blocks of size BlockSize from a non-existent datanode
                DatanodeInfo info = DFSTestUtil.GetDatanodeInfo("1.2.3.4");
                GetBlocksWithException(namenode, info, 2);
            }
            finally
            {
                cluster.Shutdown();
            }
        }