Example #1
0
        /// <exception cref="System.IO.IOException"/>
        private void OnGetFileChecksum(ChannelHandlerContext ctx)
        {
            MD5MD5CRC32FileChecksum checksum = null;
            string    nnId      = @params.NamenodeId();
            DFSClient dfsclient = NewDfsClient(nnId, conf);

            try
            {
                checksum = dfsclient.GetFileChecksum(path, long.MaxValue);
                dfsclient.Close();
                dfsclient = null;
            }
            finally
            {
                IOUtils.Cleanup(Log, dfsclient);
            }
            byte[] js = Sharpen.Runtime.GetBytesForString(JsonUtil.ToJsonString(checksum), Charsets
                                                          .Utf8);
            DefaultFullHttpResponse resp = new DefaultFullHttpResponse(HttpVersion.Http11, HttpResponseStatus
                                                                       .Ok, Unpooled.WrappedBuffer(js));

            resp.Headers().Set(HttpHeaders.Names.ContentType, ApplicationJsonUtf8);
            resp.Headers().Set(HttpHeaders.Names.ContentLength, js.Length);
            resp.Headers().Set(HttpHeaders.Names.Connection, HttpHeaders.Values.Close);
            ctx.WriteAndFlush(resp).AddListener(ChannelFutureListener.Close);
        }
 public virtual void ShutDownCluster()
 {
     client.Close();
     fs.Close();
     cluster.ShutdownDataNodes();
     cluster.Shutdown();
 }
Example #3
0
 /// <summary>Close all DFSClient instances in the Cache.</summary>
 /// <param name="onlyAutomatic">only close those that are marked for automatic closing
 ///     </param>
 /// <exception cref="System.IO.IOException"/>
 internal virtual void CloseAll(bool onlyAutomatic)
 {
     lock (this)
     {
         IList <IOException> exceptions        = new AList <IOException>();
         ConcurrentMap <string, DFSClient> map = clientCache.AsMap();
         foreach (KeyValuePair <string, DFSClient> item in map)
         {
             DFSClient client = item.Value;
             if (client != null)
             {
                 try
                 {
                     client.Close();
                 }
                 catch (IOException ioe)
                 {
                     exceptions.AddItem(ioe);
                 }
             }
         }
         if (!exceptions.IsEmpty())
         {
             throw MultipleIOException.CreateIOException(exceptions);
         }
     }
 }
Example #4
0
 /// <exception cref="System.Exception"/>
 public virtual void TestDatanodePeersXceiver()
 {
     // Test DatanodeXceiver has correct peer-dataxceiver pairs for sending OOB message
     try
     {
         StartCluster();
         // Create files in DFS.
         string          testFile1 = "/" + GenericTestUtils.GetMethodName() + ".01.dat";
         string          testFile2 = "/" + GenericTestUtils.GetMethodName() + ".02.dat";
         string          testFile3 = "/" + GenericTestUtils.GetMethodName() + ".03.dat";
         DFSClient       client1   = new DFSClient(NameNode.GetAddress(conf), conf);
         DFSClient       client2   = new DFSClient(NameNode.GetAddress(conf), conf);
         DFSClient       client3   = new DFSClient(NameNode.GetAddress(conf), conf);
         DFSOutputStream s1        = (DFSOutputStream)client1.Create(testFile1, true);
         DFSOutputStream s2        = (DFSOutputStream)client2.Create(testFile2, true);
         DFSOutputStream s3        = (DFSOutputStream)client3.Create(testFile3, true);
         byte[]          toWrite   = new byte[1024 * 1024 * 8];
         Random          rb        = new Random(1111);
         rb.NextBytes(toWrite);
         s1.Write(toWrite, 0, 1024 * 1024 * 8);
         s1.Flush();
         s2.Write(toWrite, 0, 1024 * 1024 * 8);
         s2.Flush();
         s3.Write(toWrite, 0, 1024 * 1024 * 8);
         s3.Flush();
         NUnit.Framework.Assert.IsTrue(dn0.GetXferServer().GetNumPeersXceiver() == dn0.GetXferServer
                                           ().GetNumPeersXceiver());
         s1.Close();
         s2.Close();
         s3.Close();
         NUnit.Framework.Assert.IsTrue(dn0.GetXferServer().GetNumPeersXceiver() == dn0.GetXferServer
                                           ().GetNumPeersXceiver());
         client1.Close();
         client2.Close();
         client3.Close();
     }
     finally
     {
         ShutdownCluster();
     }
 }
Example #5
0
        public virtual void TestReadFromOneDN()
        {
            HdfsConfiguration configuration = new HdfsConfiguration();
            // One of the goals of this test is to verify that we don't open more
            // than one socket.  So use a different client context, so that we
            // get our own socket cache, rather than sharing with the other test
            // instances.  Also use a really long socket timeout so that nothing
            // gets closed before we get around to checking the cache size at the end.
            string contextName = "testReadFromOneDNContext";

            configuration.Set(DFSConfigKeys.DfsClientContext, contextName);
            configuration.SetLong(DFSConfigKeys.DfsClientSocketTimeoutKey, 100000000L);
            BlockReaderTestUtil util = new BlockReaderTestUtil(1, configuration);
            Path testFile            = new Path("/testConnCache.dat");

            byte[]    authenticData = util.WriteFile(testFile, FileSize / 1024);
            DFSClient client        = new DFSClient(new IPEndPoint("localhost", util.GetCluster().GetNameNodePort
                                                                       ()), util.GetConf());
            ClientContext  cacheContext = ClientContext.Get(contextName, client.GetConf());
            DFSInputStream @in          = client.Open(testFile.ToString());

            Log.Info("opened " + testFile.ToString());
            byte[] dataBuf = new byte[BlockSize];
            // Initial read
            Pread(@in, 0, dataBuf, 0, dataBuf.Length, authenticData);
            // Read again and verify that the socket is the same
            Pread(@in, FileSize - dataBuf.Length, dataBuf, 0, dataBuf.Length, authenticData);
            Pread(@in, 1024, dataBuf, 0, dataBuf.Length, authenticData);
            // No seek; just read
            Pread(@in, -1, dataBuf, 0, dataBuf.Length, authenticData);
            Pread(@in, 64, dataBuf, 0, dataBuf.Length / 2, authenticData);
            @in.Close();
            client.Close();
            NUnit.Framework.Assert.AreEqual(1, ClientContext.GetFromConf(configuration).GetPeerCache
                                                ().Size());
        }
Example #6
0
        public virtual void TestDeletedBlockWhenAddBlockIsInEdit()
        {
            Configuration conf = new HdfsConfiguration();

            cluster = new MiniDFSCluster.Builder(conf).NnTopology(MiniDFSNNTopology.SimpleHATopology
                                                                      ()).NumDataNodes(1).Build();
            DFSClient client = null;

            try
            {
                cluster.WaitActive();
                NUnit.Framework.Assert.AreEqual("Number of namenodes is not 2", 2, cluster.GetNumNameNodes
                                                    ());
                // Transitioning the namenode 0 to active.
                cluster.TransitionToActive(0);
                NUnit.Framework.Assert.IsTrue("Namenode 0 should be in active state", cluster.GetNameNode
                                                  (0).IsActiveState());
                NUnit.Framework.Assert.IsTrue("Namenode 1 should be in standby state", cluster.GetNameNode
                                                  (1).IsStandbyState());
                // Trigger heartbeat to mark DatanodeStorageInfo#heartbeatedSinceFailover
                // to true.
                DataNodeTestUtils.TriggerHeartbeat(cluster.GetDataNodes()[0]);
                FileSystem fs = cluster.GetFileSystem(0);
                // Trigger blockReport to mark DatanodeStorageInfo#blockContentsStale
                // to false.
                cluster.GetDataNodes()[0].TriggerBlockReport(new BlockReportOptions.Factory().SetIncremental
                                                                 (false).Build());
                Path fileName = new Path("/tmp.txt");
                // create a file with one block
                DFSTestUtil.CreateFile(fs, fileName, 10L, (short)1, 1234L);
                DFSTestUtil.WaitReplication(fs, fileName, (short)1);
                client = new DFSClient(cluster.GetFileSystem(0).GetUri(), conf);
                IList <LocatedBlock> locatedBlocks = client.GetNamenode().GetBlockLocations("/tmp.txt"
                                                                                            , 0, 10L).GetLocatedBlocks();
                NUnit.Framework.Assert.IsTrue(locatedBlocks.Count == 1);
                NUnit.Framework.Assert.IsTrue(locatedBlocks[0].GetLocations().Length == 1);
                // add a second datanode to the cluster
                cluster.StartDataNodes(conf, 1, true, null, null, null, null);
                NUnit.Framework.Assert.AreEqual("Number of datanodes should be 2", 2, cluster.GetDataNodes
                                                    ().Count);
                DataNode           dn0          = cluster.GetDataNodes()[0];
                DataNode           dn1          = cluster.GetDataNodes()[1];
                string             activeNNBPId = cluster.GetNamesystem(0).GetBlockPoolId();
                DatanodeDescriptor sourceDnDesc = NameNodeAdapter.GetDatanode(cluster.GetNamesystem
                                                                                  (0), dn0.GetDNRegistrationForBP(activeNNBPId));
                DatanodeDescriptor destDnDesc = NameNodeAdapter.GetDatanode(cluster.GetNamesystem
                                                                                (0), dn1.GetDNRegistrationForBP(activeNNBPId));
                ExtendedBlock block = DFSTestUtil.GetFirstBlock(fs, fileName);
                Log.Info("replaceBlock:  " + ReplaceBlock(block, (DatanodeInfo)sourceDnDesc, (DatanodeInfo
                                                                                              )sourceDnDesc, (DatanodeInfo)destDnDesc));
                // Waiting for the FsDatasetAsyncDsikService to delete the block
                Sharpen.Thread.Sleep(3000);
                // Triggering the incremental block report to report the deleted block to
                // namnemode
                cluster.GetDataNodes()[0].TriggerBlockReport(new BlockReportOptions.Factory().SetIncremental
                                                                 (true).Build());
                cluster.TransitionToStandby(0);
                cluster.TransitionToActive(1);
                NUnit.Framework.Assert.IsTrue("Namenode 1 should be in active state", cluster.GetNameNode
                                                  (1).IsActiveState());
                NUnit.Framework.Assert.IsTrue("Namenode 0 should be in standby state", cluster.GetNameNode
                                                  (0).IsStandbyState());
                client.Close();
                // Opening a new client for new active  namenode
                client = new DFSClient(cluster.GetFileSystem(1).GetUri(), conf);
                IList <LocatedBlock> locatedBlocks1 = client.GetNamenode().GetBlockLocations("/tmp.txt"
                                                                                             , 0, 10L).GetLocatedBlocks();
                NUnit.Framework.Assert.AreEqual(1, locatedBlocks1.Count);
                NUnit.Framework.Assert.AreEqual("The block should be only on 1 datanode ", 1, locatedBlocks1
                                                [0].GetLocations().Length);
            }
            finally
            {
                IOUtils.Cleanup(null, client);
                cluster.Shutdown();
            }
        }
Example #7
0
        public virtual void TestReadSelectNonStaleDatanode()
        {
            HdfsConfiguration conf = new HdfsConfiguration();

            conf.SetBoolean(DFSConfigKeys.DfsNamenodeAvoidStaleDatanodeForReadKey, true);
            long staleInterval = 30 * 1000 * 60;

            conf.SetLong(DFSConfigKeys.DfsNamenodeStaleDatanodeIntervalKey, staleInterval);
            MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(numDatanodes
                                                                                   ).Racks(racks).Build();

            cluster.WaitActive();
            IPEndPoint addr   = new IPEndPoint("localhost", cluster.GetNameNodePort());
            DFSClient  client = new DFSClient(addr, conf);
            IList <DatanodeDescriptor> nodeInfoList = cluster.GetNameNode().GetNamesystem().GetBlockManager
                                                          ().GetDatanodeManager().GetDatanodeListForReport(HdfsConstants.DatanodeReportType
                                                                                                           .Live);

            NUnit.Framework.Assert.AreEqual("Unexpected number of datanodes", numDatanodes, nodeInfoList
                                            .Count);
            FileSystem         fileSys = cluster.GetFileSystem();
            FSDataOutputStream stm     = null;

            try
            {
                // do the writing but do not close the FSDataOutputStream
                // in order to mimic the ongoing writing
                Path fileName = new Path("/file1");
                stm = fileSys.Create(fileName, true, fileSys.GetConf().GetInt(CommonConfigurationKeys
                                                                              .IoFileBufferSizeKey, 4096), (short)3, blockSize);
                stm.Write(new byte[(blockSize * 3) / 2]);
                // We do not close the stream so that
                // the writing seems to be still ongoing
                stm.Hflush();
                LocatedBlocks blocks = client.GetNamenode().GetBlockLocations(fileName.ToString()
                                                                              , 0, blockSize);
                DatanodeInfo[] nodes = blocks.Get(0).GetLocations();
                NUnit.Framework.Assert.AreEqual(nodes.Length, 3);
                DataNode           staleNode     = null;
                DatanodeDescriptor staleNodeInfo = null;
                // stop the heartbeat of the first node
                staleNode = this.StopDataNodeHeartbeat(cluster, nodes[0].GetHostName());
                NUnit.Framework.Assert.IsNotNull(staleNode);
                // set the first node as stale
                staleNodeInfo = cluster.GetNameNode().GetNamesystem().GetBlockManager().GetDatanodeManager
                                    ().GetDatanode(staleNode.GetDatanodeId());
                DFSTestUtil.ResetLastUpdatesWithOffset(staleNodeInfo, -(staleInterval + 1));
                LocatedBlocks blocksAfterStale = client.GetNamenode().GetBlockLocations(fileName.
                                                                                        ToString(), 0, blockSize);
                DatanodeInfo[] nodesAfterStale = blocksAfterStale.Get(0).GetLocations();
                NUnit.Framework.Assert.AreEqual(nodesAfterStale.Length, 3);
                NUnit.Framework.Assert.AreEqual(nodesAfterStale[2].GetHostName(), nodes[0].GetHostName
                                                    ());
                // restart the staleNode's heartbeat
                DataNodeTestUtils.SetHeartbeatsDisabledForTests(staleNode, false);
                // reset the first node as non-stale, so as to avoid two stale nodes
                DFSTestUtil.ResetLastUpdatesWithOffset(staleNodeInfo, 0);
                LocatedBlock lastBlock = client.GetLocatedBlocks(fileName.ToString(), 0, long.MaxValue
                                                                 ).GetLastLocatedBlock();
                nodes = lastBlock.GetLocations();
                NUnit.Framework.Assert.AreEqual(nodes.Length, 3);
                // stop the heartbeat of the first node for the last block
                staleNode = this.StopDataNodeHeartbeat(cluster, nodes[0].GetHostName());
                NUnit.Framework.Assert.IsNotNull(staleNode);
                // set the node as stale
                DatanodeDescriptor dnDesc = cluster.GetNameNode().GetNamesystem().GetBlockManager
                                                ().GetDatanodeManager().GetDatanode(staleNode.GetDatanodeId());
                DFSTestUtil.ResetLastUpdatesWithOffset(dnDesc, -(staleInterval + 1));
                LocatedBlock lastBlockAfterStale = client.GetLocatedBlocks(fileName.ToString(), 0
                                                                           , long.MaxValue).GetLastLocatedBlock();
                nodesAfterStale = lastBlockAfterStale.GetLocations();
                NUnit.Framework.Assert.AreEqual(nodesAfterStale.Length, 3);
                NUnit.Framework.Assert.AreEqual(nodesAfterStale[2].GetHostName(), nodes[0].GetHostName
                                                    ());
            }
            finally
            {
                if (stm != null)
                {
                    stm.Close();
                }
                client.Close();
                cluster.Shutdown();
            }
        }
Example #8
0
        public virtual void TestCopyOnWrite()
        {
            Configuration  conf    = new HdfsConfiguration();
            MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).Build();
            FileSystem     fs      = cluster.GetFileSystem();
            IPEndPoint     addr    = new IPEndPoint("localhost", cluster.GetNameNodePort());
            DFSClient      client  = new DFSClient(addr, conf);

            try
            {
                // create a new file, write to it and close it.
                //
                Path file1             = new Path("/filestatus.dat");
                FSDataOutputStream stm = AppendTestUtil.CreateFile(fs, file1, 1);
                WriteFile(stm);
                stm.Close();
                // Get a handle to the datanode
                DataNode[] dn = cluster.ListDataNodes();
                NUnit.Framework.Assert.IsTrue("There should be only one datanode but found " + dn
                                              .Length, dn.Length == 1);
                LocatedBlocks locations = client.GetNamenode().GetBlockLocations(file1.ToString()
                                                                                 , 0, long.MaxValue);
                IList <LocatedBlock> blocks = locations.GetLocatedBlocks();
                //
                // Create hard links for a few of the blocks
                //
                for (int i = 0; i < blocks.Count; i = i + 2)
                {
                    ExtendedBlock b = blocks[i].GetBlock();
                    FilePath      f = DataNodeTestUtils.GetFile(dn[0], b.GetBlockPoolId(), b.GetLocalBlock
                                                                    ().GetBlockId());
                    FilePath link = new FilePath(f.ToString() + ".link");
                    System.Console.Out.WriteLine("Creating hardlink for File " + f + " to " + link);
                    HardLink.CreateHardLink(f, link);
                }
                //
                // Detach all blocks. This should remove hardlinks (if any)
                //
                for (int i_1 = 0; i_1 < blocks.Count; i_1++)
                {
                    ExtendedBlock b = blocks[i_1].GetBlock();
                    System.Console.Out.WriteLine("testCopyOnWrite detaching block " + b);
                    NUnit.Framework.Assert.IsTrue("Detaching block " + b + " should have returned true"
                                                  , DataNodeTestUtils.UnlinkBlock(dn[0], b, 1));
                }
                // Since the blocks were already detached earlier, these calls should
                // return false
                //
                for (int i_2 = 0; i_2 < blocks.Count; i_2++)
                {
                    ExtendedBlock b = blocks[i_2].GetBlock();
                    System.Console.Out.WriteLine("testCopyOnWrite detaching block " + b);
                    NUnit.Framework.Assert.IsTrue("Detaching block " + b + " should have returned false"
                                                  , !DataNodeTestUtils.UnlinkBlock(dn[0], b, 1));
                }
            }
            finally
            {
                client.Close();
                fs.Close();
                cluster.Shutdown();
            }
        }
        public virtual void TestRead()
        {
            MiniDFSCluster cluster      = null;
            int            numDataNodes = 2;
            Configuration  conf         = GetConf(numDataNodes);

            try
            {
                cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(numDataNodes).Build();
                cluster.WaitActive();
                NUnit.Framework.Assert.AreEqual(numDataNodes, cluster.GetDataNodes().Count);
                NameNode                nn      = cluster.GetNameNode();
                NamenodeProtocols       nnProto = nn.GetRpcServer();
                BlockManager            bm      = nn.GetNamesystem().GetBlockManager();
                BlockTokenSecretManager sm      = bm.GetBlockTokenSecretManager();
                // set a short token lifetime (1 second) initially
                SecurityTestUtil.SetBlockTokenLifetime(sm, 1000L);
                Path       fileToRead = new Path(FileToRead);
                FileSystem fs         = cluster.GetFileSystem();
                CreateFile(fs, fileToRead);

                /*
                 * setup for testing expiration handling of cached tokens
                 */
                // read using blockSeekTo(). Acquired tokens are cached in in1
                FSDataInputStream in1 = fs.Open(fileToRead);
                NUnit.Framework.Assert.IsTrue(CheckFile1(in1));
                // read using blockSeekTo(). Acquired tokens are cached in in2
                FSDataInputStream in2 = fs.Open(fileToRead);
                NUnit.Framework.Assert.IsTrue(CheckFile1(in2));
                // read using fetchBlockByteRange(). Acquired tokens are cached in in3
                FSDataInputStream in3 = fs.Open(fileToRead);
                NUnit.Framework.Assert.IsTrue(CheckFile2(in3));

                /*
                 * testing READ interface on DN using a BlockReader
                 */
                DFSClient client = null;
                try
                {
                    client = new DFSClient(new IPEndPoint("localhost", cluster.GetNameNodePort()), conf
                                           );
                }
                finally
                {
                    if (client != null)
                    {
                        client.Close();
                    }
                }
                IList <LocatedBlock> locatedBlocks = nnProto.GetBlockLocations(FileToRead, 0, FileSize
                                                                               ).GetLocatedBlocks();
                LocatedBlock lblock = locatedBlocks[0];
                // first block
                Org.Apache.Hadoop.Security.Token.Token <BlockTokenIdentifier> myToken = lblock.GetBlockToken
                                                                                            ();
                // verify token is not expired
                NUnit.Framework.Assert.IsFalse(SecurityTestUtil.IsBlockTokenExpired(myToken));
                // read with valid token, should succeed
                TryRead(conf, lblock, true);

                /*
                 * wait till myToken and all cached tokens in in1, in2 and in3 expire
                 */
                while (!SecurityTestUtil.IsBlockTokenExpired(myToken))
                {
                    try
                    {
                        Sharpen.Thread.Sleep(10);
                    }
                    catch (Exception)
                    {
                    }
                }

                /*
                 * continue testing READ interface on DN using a BlockReader
                 */
                // verify token is expired
                NUnit.Framework.Assert.IsTrue(SecurityTestUtil.IsBlockTokenExpired(myToken));
                // read should fail
                TryRead(conf, lblock, false);
                // use a valid new token
                lblock.SetBlockToken(sm.GenerateToken(lblock.GetBlock(), EnumSet.Of(BlockTokenSecretManager.AccessMode
                                                                                    .Read)));
                // read should succeed
                TryRead(conf, lblock, true);
                // use a token with wrong blockID
                ExtendedBlock wrongBlock = new ExtendedBlock(lblock.GetBlock().GetBlockPoolId(),
                                                             lblock.GetBlock().GetBlockId() + 1);
                lblock.SetBlockToken(sm.GenerateToken(wrongBlock, EnumSet.Of(BlockTokenSecretManager.AccessMode
                                                                             .Read)));
                // read should fail
                TryRead(conf, lblock, false);
                // use a token with wrong access modes
                lblock.SetBlockToken(sm.GenerateToken(lblock.GetBlock(), EnumSet.Of(BlockTokenSecretManager.AccessMode
                                                                                    .Write, BlockTokenSecretManager.AccessMode.Copy, BlockTokenSecretManager.AccessMode
                                                                                    .Replace)));
                // read should fail
                TryRead(conf, lblock, false);
                // set a long token lifetime for future tokens
                SecurityTestUtil.SetBlockTokenLifetime(sm, 600 * 1000L);

                /*
                 * testing that when cached tokens are expired, DFSClient will re-fetch
                 * tokens transparently for READ.
                 */
                // confirm all tokens cached in in1 are expired by now
                IList <LocatedBlock> lblocks = DFSTestUtil.GetAllBlocks(in1);
                foreach (LocatedBlock blk in lblocks)
                {
                    NUnit.Framework.Assert.IsTrue(SecurityTestUtil.IsBlockTokenExpired(blk.GetBlockToken
                                                                                           ()));
                }
                // verify blockSeekTo() is able to re-fetch token transparently
                in1.Seek(0);
                NUnit.Framework.Assert.IsTrue(CheckFile1(in1));
                // confirm all tokens cached in in2 are expired by now
                IList <LocatedBlock> lblocks2 = DFSTestUtil.GetAllBlocks(in2);
                foreach (LocatedBlock blk_1 in lblocks2)
                {
                    NUnit.Framework.Assert.IsTrue(SecurityTestUtil.IsBlockTokenExpired(blk_1.GetBlockToken
                                                                                           ()));
                }
                // verify blockSeekTo() is able to re-fetch token transparently (testing
                // via another interface method)
                NUnit.Framework.Assert.IsTrue(in2.SeekToNewSource(0));
                NUnit.Framework.Assert.IsTrue(CheckFile1(in2));
                // confirm all tokens cached in in3 are expired by now
                IList <LocatedBlock> lblocks3 = DFSTestUtil.GetAllBlocks(in3);
                foreach (LocatedBlock blk_2 in lblocks3)
                {
                    NUnit.Framework.Assert.IsTrue(SecurityTestUtil.IsBlockTokenExpired(blk_2.GetBlockToken
                                                                                           ()));
                }
                // verify fetchBlockByteRange() is able to re-fetch token transparently
                NUnit.Framework.Assert.IsTrue(CheckFile2(in3));

                /*
                 * testing that after datanodes are restarted on the same ports, cached
                 * tokens should still work and there is no need to fetch new tokens from
                 * namenode. This test should run while namenode is down (to make sure no
                 * new tokens can be fetched from namenode).
                 */
                // restart datanodes on the same ports that they currently use
                NUnit.Framework.Assert.IsTrue(cluster.RestartDataNodes(true));
                cluster.WaitActive();
                NUnit.Framework.Assert.AreEqual(numDataNodes, cluster.GetDataNodes().Count);
                cluster.ShutdownNameNode(0);
                // confirm tokens cached in in1 are still valid
                lblocks = DFSTestUtil.GetAllBlocks(in1);
                foreach (LocatedBlock blk_3 in lblocks)
                {
                    NUnit.Framework.Assert.IsFalse(SecurityTestUtil.IsBlockTokenExpired(blk_3.GetBlockToken
                                                                                            ()));
                }
                // verify blockSeekTo() still works (forced to use cached tokens)
                in1.Seek(0);
                NUnit.Framework.Assert.IsTrue(CheckFile1(in1));
                // confirm tokens cached in in2 are still valid
                lblocks2 = DFSTestUtil.GetAllBlocks(in2);
                foreach (LocatedBlock blk_4 in lblocks2)
                {
                    NUnit.Framework.Assert.IsFalse(SecurityTestUtil.IsBlockTokenExpired(blk_4.GetBlockToken
                                                                                            ()));
                }
                // verify blockSeekTo() still works (forced to use cached tokens)
                in2.SeekToNewSource(0);
                NUnit.Framework.Assert.IsTrue(CheckFile1(in2));
                // confirm tokens cached in in3 are still valid
                lblocks3 = DFSTestUtil.GetAllBlocks(in3);
                foreach (LocatedBlock blk_5 in lblocks3)
                {
                    NUnit.Framework.Assert.IsFalse(SecurityTestUtil.IsBlockTokenExpired(blk_5.GetBlockToken
                                                                                            ()));
                }
                // verify fetchBlockByteRange() still works (forced to use cached tokens)
                NUnit.Framework.Assert.IsTrue(CheckFile2(in3));

                /*
                 * testing that when namenode is restarted, cached tokens should still
                 * work and there is no need to fetch new tokens from namenode. Like the
                 * previous test, this test should also run while namenode is down. The
                 * setup for this test depends on the previous test.
                 */
                // restart the namenode and then shut it down for test
                cluster.RestartNameNode(0);
                cluster.ShutdownNameNode(0);
                // verify blockSeekTo() still works (forced to use cached tokens)
                in1.Seek(0);
                NUnit.Framework.Assert.IsTrue(CheckFile1(in1));
                // verify again blockSeekTo() still works (forced to use cached tokens)
                in2.SeekToNewSource(0);
                NUnit.Framework.Assert.IsTrue(CheckFile1(in2));
                // verify fetchBlockByteRange() still works (forced to use cached tokens)
                NUnit.Framework.Assert.IsTrue(CheckFile2(in3));

                /*
                 * testing that after both namenode and datanodes got restarted (namenode
                 * first, followed by datanodes), DFSClient can't access DN without
                 * re-fetching tokens and is able to re-fetch tokens transparently. The
                 * setup of this test depends on the previous test.
                 */
                // restore the cluster and restart the datanodes for test
                cluster.RestartNameNode(0);
                NUnit.Framework.Assert.IsTrue(cluster.RestartDataNodes(true));
                cluster.WaitActive();
                NUnit.Framework.Assert.AreEqual(numDataNodes, cluster.GetDataNodes().Count);
                // shutdown namenode so that DFSClient can't get new tokens from namenode
                cluster.ShutdownNameNode(0);
                // verify blockSeekTo() fails (cached tokens become invalid)
                in1.Seek(0);
                NUnit.Framework.Assert.IsFalse(CheckFile1(in1));
                // verify fetchBlockByteRange() fails (cached tokens become invalid)
                NUnit.Framework.Assert.IsFalse(CheckFile2(in3));
                // restart the namenode to allow DFSClient to re-fetch tokens
                cluster.RestartNameNode(0);
                // verify blockSeekTo() works again (by transparently re-fetching
                // tokens from namenode)
                in1.Seek(0);
                NUnit.Framework.Assert.IsTrue(CheckFile1(in1));
                in2.SeekToNewSource(0);
                NUnit.Framework.Assert.IsTrue(CheckFile1(in2));
                // verify fetchBlockByteRange() works again (by transparently
                // re-fetching tokens from namenode)
                NUnit.Framework.Assert.IsTrue(CheckFile2(in3));

                /*
                 * testing that when datanodes are restarted on different ports, DFSClient
                 * is able to re-fetch tokens transparently to connect to them
                 */
                // restart datanodes on newly assigned ports
                NUnit.Framework.Assert.IsTrue(cluster.RestartDataNodes(false));
                cluster.WaitActive();
                NUnit.Framework.Assert.AreEqual(numDataNodes, cluster.GetDataNodes().Count);
                // verify blockSeekTo() is able to re-fetch token transparently
                in1.Seek(0);
                NUnit.Framework.Assert.IsTrue(CheckFile1(in1));
                // verify blockSeekTo() is able to re-fetch token transparently
                in2.SeekToNewSource(0);
                NUnit.Framework.Assert.IsTrue(CheckFile1(in2));
                // verify fetchBlockByteRange() is able to re-fetch token transparently
                NUnit.Framework.Assert.IsTrue(CheckFile2(in3));
            }
            finally
            {
                if (cluster != null)
                {
                    cluster.Shutdown();
                }
            }
        }