示例#1
0
        public virtual void TestAbandonBlock()
        {
            string src = FileNamePrefix + "foo";
            // Start writing a file but do not close it
            FSDataOutputStream fout = fs.Create(new Path(src), true, 4096, (short)1, 512L);

            for (int i = 0; i < 1024; i++)
            {
                fout.Write(123);
            }
            fout.Hflush();
            long fileId = ((DFSOutputStream)fout.GetWrappedStream()).GetFileId();
            // Now abandon the last block
            DFSClient     dfsclient = DFSClientAdapter.GetDFSClient(fs);
            LocatedBlocks blocks    = dfsclient.GetNamenode().GetBlockLocations(src, 0, int.MaxValue
                                                                                );
            int          orginalNumBlocks = blocks.LocatedBlockCount();
            LocatedBlock b = blocks.GetLastLocatedBlock();

            dfsclient.GetNamenode().AbandonBlock(b.GetBlock(), fileId, src, dfsclient.clientName
                                                 );
            // call abandonBlock again to make sure the operation is idempotent
            dfsclient.GetNamenode().AbandonBlock(b.GetBlock(), fileId, src, dfsclient.clientName
                                                 );
            // And close the file
            fout.Close();
            // Close cluster and check the block has been abandoned after restart
            cluster.RestartNameNode();
            blocks = dfsclient.GetNamenode().GetBlockLocations(src, 0, int.MaxValue);
            NUnit.Framework.Assert.AreEqual("Blocks " + b + " has not been abandoned.", orginalNumBlocks
                                            , blocks.LocatedBlockCount() + 1);
        }
示例#2
0
        /// <exception cref="System.IO.IOException"/>
        /// <exception cref="System.Exception"/>
        /// <exception cref="Sharpen.TimeoutException"/>
        private void ChangeBlockLen(MiniDFSCluster cluster, int lenDelta)
        {
            Path       fileName          = new Path("/file1");
            short      ReplicationFactor = (short)1;
            FileSystem fs      = cluster.GetFileSystem();
            int        fileLen = fs.GetConf().GetInt(DFSConfigKeys.DfsBytesPerChecksumKey, 512);

            DFSTestUtil.CreateFile(fs, fileName, fileLen, ReplicationFactor, 0);
            DFSTestUtil.WaitReplication(fs, fileName, ReplicationFactor);
            ExtendedBlock block = DFSTestUtil.GetFirstBlock(fs, fileName);

            // Change the length of a replica
            for (int i = 0; i < cluster.GetDataNodes().Count; i++)
            {
                if (DFSTestUtil.ChangeReplicaLength(cluster, block, i, lenDelta))
                {
                    break;
                }
            }
            // increase the file's replication factor
            fs.SetReplication(fileName, (short)(ReplicationFactor + 1));
            // block replication triggers corrupt block detection
            DFSClient dfsClient = new DFSClient(new IPEndPoint("localhost", cluster.GetNameNodePort
                                                                   ()), fs.GetConf());
            LocatedBlocks blocks = dfsClient.GetNamenode().GetBlockLocations(fileName.ToString
                                                                                 (), 0, fileLen);

            if (lenDelta < 0)
            {
                // replica truncated
                while (!blocks.Get(0).IsCorrupt() || ReplicationFactor != blocks.Get(0).GetLocations
                           ().Length)
                {
                    Sharpen.Thread.Sleep(100);
                    blocks = dfsClient.GetNamenode().GetBlockLocations(fileName.ToString(), 0, fileLen
                                                                       );
                }
            }
            else
            {
                // no corruption detected; block replicated
                while (ReplicationFactor + 1 != blocks.Get(0).GetLocations().Length)
                {
                    Sharpen.Thread.Sleep(100);
                    blocks = dfsClient.GetNamenode().GetBlockLocations(fileName.ToString(), 0, fileLen
                                                                       );
                }
            }
            fs.Delete(fileName, true);
        }
示例#3
0
        /// <exception cref="System.Exception"/>
        private void TestBadBlockReportOnTransfer(bool corruptBlockByDeletingBlockFile)
        {
            Configuration  conf         = new HdfsConfiguration();
            FileSystem     fs           = null;
            DFSClient      dfsClient    = null;
            LocatedBlocks  blocks       = null;
            int            replicaCount = 0;
            short          replFactor   = 1;
            MiniDFSCluster cluster      = new MiniDFSCluster.Builder(conf).NumDataNodes(2).Build();

            cluster.WaitActive();
            fs        = cluster.GetFileSystem();
            dfsClient = new DFSClient(new IPEndPoint("localhost", cluster.GetNameNodePort()),
                                      conf);
            // Create file with replication factor of 1
            Path file1 = new Path("/tmp/testBadBlockReportOnTransfer/file1");

            DFSTestUtil.CreateFile(fs, file1, 1024, replFactor, 0);
            DFSTestUtil.WaitReplication(fs, file1, replFactor);
            // Corrupt the block belonging to the created file
            ExtendedBlock block = DFSTestUtil.GetFirstBlock(fs, file1);
            int           blockFilesCorrupted = corruptBlockByDeletingBlockFile ? cluster.CorruptBlockOnDataNodesByDeletingBlockFile
                                                    (block) : cluster.CorruptBlockOnDataNodes(block);

            NUnit.Framework.Assert.AreEqual("Corrupted too few blocks", replFactor, blockFilesCorrupted
                                            );
            // Increase replication factor, this should invoke transfer request
            // Receiving datanode fails on checksum and reports it to namenode
            replFactor = 2;
            fs.SetReplication(file1, replFactor);
            // Now get block details and check if the block is corrupt
            blocks = dfsClient.GetNamenode().GetBlockLocations(file1.ToString(), 0, long.MaxValue
                                                               );
            while (blocks.Get(0).IsCorrupt() != true)
            {
                try
                {
                    Log.Info("Waiting until block is marked as corrupt...");
                    Sharpen.Thread.Sleep(1000);
                }
                catch (Exception)
                {
                }
                blocks = dfsClient.GetNamenode().GetBlockLocations(file1.ToString(), 0, long.MaxValue
                                                                   );
            }
            replicaCount = blocks.Get(0).GetLocations().Length;
            NUnit.Framework.Assert.IsTrue(replicaCount == 1);
            cluster.Shutdown();
        }
示例#4
0
        /// <summary>Get the list of Blocks for a file.</summary>
        /// <exception cref="System.IO.IOException"/>
        public virtual IList <LocatedBlock> GetFileBlocks(Path filepath, int sizeKB)
        {
            // Return the blocks we just wrote
            DFSClient dfsclient = GetDFSClient();

            return(dfsclient.GetNamenode().GetBlockLocations(filepath.ToString(), 0, sizeKB *
                                                             1024).GetLocatedBlocks());
        }
示例#5
0
        /// <summary>Get the internet address of the currently-active NN.</summary>
        /// <remarks>
        /// Get the internet address of the currently-active NN. This should rarely be
        /// used, since callers of this method who connect directly to the NN using the
        /// resulting InetSocketAddress will not be able to connect to the active NN if
        /// a failover were to occur after this method has been called.
        /// </remarks>
        /// <param name="fs">the file system to get the active address of.</param>
        /// <returns>the internet address of the currently-active NN.</returns>
        /// <exception cref="System.IO.IOException">if an error occurs while resolving the active NN.
        ///     </exception>
        public static IPEndPoint GetAddressOfActive(FileSystem fs)
        {
            if (!(fs is DistributedFileSystem))
            {
                throw new ArgumentException("FileSystem " + fs + " is not a DFS.");
            }
            // force client address resolution.
            fs.Exists(new Path("/"));
            DistributedFileSystem dfs       = (DistributedFileSystem)fs;
            DFSClient             dfsClient = dfs.GetClient();

            return(RPC.GetServerAddress(dfsClient.GetNamenode()));
        }
示例#6
0
        public virtual void TestPendingReplicationRetry()
        {
            MiniDFSCluster cluster      = null;
            int            numDataNodes = 4;
            string         testFile     = "/replication-test-file";
            Path           testPath     = new Path(testFile);

            byte[] buffer = new byte[1024];
            for (int i = 0; i < buffer.Length; i++)
            {
                buffer[i] = (byte)('1');
            }
            try
            {
                Configuration conf = new HdfsConfiguration();
                conf.Set(DFSConfigKeys.DfsReplicationKey, Sharpen.Extensions.ToString(numDataNodes
                                                                                      ));
                //first time format
                cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(numDataNodes).Build();
                cluster.WaitActive();
                DFSClient dfsClient = new DFSClient(new IPEndPoint("localhost", cluster.GetNameNodePort
                                                                       ()), conf);
                OutputStream @out = cluster.GetFileSystem().Create(testPath);
                @out.Write(buffer);
                @out.Close();
                WaitForBlockReplication(testFile, dfsClient.GetNamenode(), numDataNodes, -1);
                // get first block of the file.
                ExtendedBlock block = dfsClient.GetNamenode().GetBlockLocations(testFile, 0, long.MaxValue
                                                                                ).Get(0).GetBlock();
                cluster.Shutdown();
                for (int i_1 = 0; i_1 < 25; i_1++)
                {
                    buffer[i_1] = (byte)('0');
                }
                int fileCount = 0;
                // Choose 3 copies of block file - delete 1 and corrupt the remaining 2
                for (int dnIndex = 0; dnIndex < 3; dnIndex++)
                {
                    FilePath blockFile = cluster.GetBlockFile(dnIndex, block);
                    Log.Info("Checking for file " + blockFile);
                    if (blockFile != null && blockFile.Exists())
                    {
                        if (fileCount == 0)
                        {
                            Log.Info("Deleting file " + blockFile);
                            NUnit.Framework.Assert.IsTrue(blockFile.Delete());
                        }
                        else
                        {
                            // corrupt it.
                            Log.Info("Corrupting file " + blockFile);
                            long len = blockFile.Length();
                            NUnit.Framework.Assert.IsTrue(len > 50);
                            RandomAccessFile blockOut = new RandomAccessFile(blockFile, "rw");
                            try
                            {
                                blockOut.Seek(len / 3);
                                blockOut.Write(buffer, 0, 25);
                            }
                            finally
                            {
                                blockOut.Close();
                            }
                        }
                        fileCount++;
                    }
                }
                NUnit.Framework.Assert.AreEqual(3, fileCount);

                /* Start the MiniDFSCluster with more datanodes since once a writeBlock
                 * to a datanode node fails, same block can not be written to it
                 * immediately. In our case some replication attempts will fail.
                 */
                Log.Info("Restarting minicluster after deleting a replica and corrupting 2 crcs");
                conf = new HdfsConfiguration();
                conf.Set(DFSConfigKeys.DfsReplicationKey, Sharpen.Extensions.ToString(numDataNodes
                                                                                      ));
                conf.Set(DFSConfigKeys.DfsNamenodeReplicationPendingTimeoutSecKey, Sharpen.Extensions.ToString
                             (2));
                conf.Set("dfs.datanode.block.write.timeout.sec", Sharpen.Extensions.ToString(5));
                conf.Set(DFSConfigKeys.DfsNamenodeSafemodeThresholdPctKey, "0.75f");
                // only 3 copies exist
                cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(numDataNodes * 2).Format(
                    false).Build();
                cluster.WaitActive();
                dfsClient = new DFSClient(new IPEndPoint("localhost", cluster.GetNameNodePort()),
                                          conf);
                WaitForBlockReplication(testFile, dfsClient.GetNamenode(), numDataNodes, -1);
            }
            finally
            {
                if (cluster != null)
                {
                    cluster.Shutdown();
                }
            }
        }
        public virtual void TestInjection()
        {
            MiniDFSCluster cluster  = null;
            string         testFile = "/replication-test-file";
            Path           testPath = new Path(testFile);

            byte[] buffer = new byte[1024];
            for (int i = 0; i < buffer.Length; i++)
            {
                buffer[i] = (byte)('1');
            }
            try
            {
                Configuration conf = new HdfsConfiguration();
                conf.Set(DFSConfigKeys.DfsReplicationKey, Sharpen.Extensions.ToString(numDataNodes
                                                                                      ));
                conf.SetInt(DFSConfigKeys.DfsBytesPerChecksumKey, checksumSize);
                SimulatedFSDataset.SetFactory(conf);
                //first time format
                cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(numDataNodes).Build();
                cluster.WaitActive();
                string    bpid      = cluster.GetNamesystem().GetBlockPoolId();
                DFSClient dfsClient = new DFSClient(new IPEndPoint("localhost", cluster.GetNameNodePort
                                                                       ()), conf);
                WriteFile(cluster.GetFileSystem(), testPath, numDataNodes);
                WaitForBlockReplication(testFile, dfsClient.GetNamenode(), numDataNodes, 20);
                IList <IDictionary <DatanodeStorage, BlockListAsLongs> > blocksList = cluster.GetAllBlockReports
                                                                                          (bpid);
                cluster.Shutdown();
                cluster = null;

                /* Start the MiniDFSCluster with more datanodes since once a writeBlock
                 * to a datanode node fails, same block can not be written to it
                 * immediately. In our case some replication attempts will fail.
                 */
                Log.Info("Restarting minicluster");
                conf = new HdfsConfiguration();
                SimulatedFSDataset.SetFactory(conf);
                conf.Set(DFSConfigKeys.DfsNamenodeSafemodeThresholdPctKey, "0.0f");
                cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(numDataNodes * 2).Format(
                    false).Build();
                cluster.WaitActive();
                ICollection <Block> uniqueBlocks = new HashSet <Block>();
                foreach (IDictionary <DatanodeStorage, BlockListAsLongs> map in blocksList)
                {
                    foreach (BlockListAsLongs blockList in map.Values)
                    {
                        foreach (Block b in blockList)
                        {
                            uniqueBlocks.AddItem(new Block(b));
                        }
                    }
                }
                // Insert all the blocks in the first data node
                Log.Info("Inserting " + uniqueBlocks.Count + " blocks");
                cluster.InjectBlocks(0, uniqueBlocks, null);
                dfsClient = new DFSClient(new IPEndPoint("localhost", cluster.GetNameNodePort()),
                                          conf);
                WaitForBlockReplication(testFile, dfsClient.GetNamenode(), numDataNodes, -1);
            }
            finally
            {
                if (cluster != null)
                {
                    cluster.Shutdown();
                }
            }
        }
示例#8
0
        public virtual void TestBlockReplacement()
        {
            Configuration Conf = new HdfsConfiguration();

            string[] InitialRacks      = new string[] { "/RACK0", "/RACK1", "/RACK2" };
            string[] NewRacks          = new string[] { "/RACK2" };
            short    ReplicationFactor = (short)3;
            int      DefaultBlockSize  = 1024;
            Random   r = new Random();

            Conf.SetLong(DFSConfigKeys.DfsBlockSizeKey, DefaultBlockSize);
            Conf.SetInt(DFSConfigKeys.DfsBytesPerChecksumKey, DefaultBlockSize / 2);
            Conf.SetLong(DFSConfigKeys.DfsBlockreportIntervalMsecKey, 500);
            cluster = new MiniDFSCluster.Builder(Conf).NumDataNodes(ReplicationFactor).Racks(
                InitialRacks).Build();
            try
            {
                cluster.WaitActive();
                FileSystem fs       = cluster.GetFileSystem();
                Path       fileName = new Path("/tmp.txt");
                // create a file with one block
                DFSTestUtil.CreateFile(fs, fileName, DefaultBlockSize, ReplicationFactor, r.NextLong
                                           ());
                DFSTestUtil.WaitReplication(fs, fileName, ReplicationFactor);
                // get all datanodes
                IPEndPoint           addr          = new IPEndPoint("localhost", cluster.GetNameNodePort());
                DFSClient            client        = new DFSClient(addr, Conf);
                IList <LocatedBlock> locatedBlocks = client.GetNamenode().GetBlockLocations("/tmp.txt"
                                                                                            , 0, DefaultBlockSize).GetLocatedBlocks();
                NUnit.Framework.Assert.AreEqual(1, locatedBlocks.Count);
                LocatedBlock   block    = locatedBlocks[0];
                DatanodeInfo[] oldNodes = block.GetLocations();
                NUnit.Framework.Assert.AreEqual(oldNodes.Length, 3);
                ExtendedBlock b = block.GetBlock();
                // add a fourth datanode to the cluster
                cluster.StartDataNodes(Conf, 1, true, null, NewRacks);
                cluster.WaitActive();
                DatanodeInfo[] datanodes = client.DatanodeReport(HdfsConstants.DatanodeReportType
                                                                 .All);
                // find out the new node
                DatanodeInfo newNode = null;
                foreach (DatanodeInfo node in datanodes)
                {
                    bool isNewNode = true;
                    foreach (DatanodeInfo oldNode in oldNodes)
                    {
                        if (node.Equals(oldNode))
                        {
                            isNewNode = false;
                            break;
                        }
                    }
                    if (isNewNode)
                    {
                        newNode = node;
                        break;
                    }
                }
                NUnit.Framework.Assert.IsTrue(newNode != null);
                DatanodeInfo         source  = null;
                AList <DatanodeInfo> proxies = new AList <DatanodeInfo>(2);
                foreach (DatanodeInfo node_1 in datanodes)
                {
                    if (node_1 != newNode)
                    {
                        if (node_1.GetNetworkLocation().Equals(newNode.GetNetworkLocation()))
                        {
                            source = node_1;
                        }
                        else
                        {
                            proxies.AddItem(node_1);
                        }
                    }
                }
                //current state: the newNode is on RACK2, and "source" is the other dn on RACK2.
                //the two datanodes on RACK0 and RACK1 are in "proxies".
                //"source" and both "proxies" all contain the block, while newNode doesn't yet.
                NUnit.Framework.Assert.IsTrue(source != null && proxies.Count == 2);
                // start to replace the block
                // case 1: proxySource does not contain the block
                Log.Info("Testcase 1: Proxy " + newNode + " does not contain the block " + b);
                NUnit.Framework.Assert.IsFalse(ReplaceBlock(b, source, newNode, proxies[0]));
                // case 2: destination already contains the block
                Log.Info("Testcase 2: Destination " + proxies[1] + " contains the block " + b);
                NUnit.Framework.Assert.IsFalse(ReplaceBlock(b, source, proxies[0], proxies[1]));
                // case 3: correct case
                Log.Info("Testcase 3: Source=" + source + " Proxy=" + proxies[0] + " Destination="
                         + newNode);
                NUnit.Framework.Assert.IsTrue(ReplaceBlock(b, source, proxies[0], newNode));
                // after cluster has time to resolve the over-replication,
                // block locations should contain two proxies and newNode
                // but not source
                CheckBlocks(new DatanodeInfo[] { newNode, proxies[0], proxies[1] }, fileName.ToString
                                (), DefaultBlockSize, ReplicationFactor, client);
                // case 4: proxies.get(0) is not a valid del hint
                // expect either source or newNode replica to be deleted instead
                Log.Info("Testcase 4: invalid del hint " + proxies[0]);
                NUnit.Framework.Assert.IsTrue(ReplaceBlock(b, proxies[0], proxies[1], source));
                // after cluster has time to resolve the over-replication,
                // block locations should contain two proxies,
                // and either source or newNode, but not both.
                CheckBlocks(Sharpen.Collections.ToArray(proxies, new DatanodeInfo[proxies.Count])
                            , fileName.ToString(), DefaultBlockSize, ReplicationFactor, client);
            }
            finally
            {
                cluster.Shutdown();
            }
        }
示例#9
0
        public virtual void TestDeletedBlockWhenAddBlockIsInEdit()
        {
            Configuration conf = new HdfsConfiguration();

            cluster = new MiniDFSCluster.Builder(conf).NnTopology(MiniDFSNNTopology.SimpleHATopology
                                                                      ()).NumDataNodes(1).Build();
            DFSClient client = null;

            try
            {
                cluster.WaitActive();
                NUnit.Framework.Assert.AreEqual("Number of namenodes is not 2", 2, cluster.GetNumNameNodes
                                                    ());
                // Transitioning the namenode 0 to active.
                cluster.TransitionToActive(0);
                NUnit.Framework.Assert.IsTrue("Namenode 0 should be in active state", cluster.GetNameNode
                                                  (0).IsActiveState());
                NUnit.Framework.Assert.IsTrue("Namenode 1 should be in standby state", cluster.GetNameNode
                                                  (1).IsStandbyState());
                // Trigger heartbeat to mark DatanodeStorageInfo#heartbeatedSinceFailover
                // to true.
                DataNodeTestUtils.TriggerHeartbeat(cluster.GetDataNodes()[0]);
                FileSystem fs = cluster.GetFileSystem(0);
                // Trigger blockReport to mark DatanodeStorageInfo#blockContentsStale
                // to false.
                cluster.GetDataNodes()[0].TriggerBlockReport(new BlockReportOptions.Factory().SetIncremental
                                                                 (false).Build());
                Path fileName = new Path("/tmp.txt");
                // create a file with one block
                DFSTestUtil.CreateFile(fs, fileName, 10L, (short)1, 1234L);
                DFSTestUtil.WaitReplication(fs, fileName, (short)1);
                client = new DFSClient(cluster.GetFileSystem(0).GetUri(), conf);
                IList <LocatedBlock> locatedBlocks = client.GetNamenode().GetBlockLocations("/tmp.txt"
                                                                                            , 0, 10L).GetLocatedBlocks();
                NUnit.Framework.Assert.IsTrue(locatedBlocks.Count == 1);
                NUnit.Framework.Assert.IsTrue(locatedBlocks[0].GetLocations().Length == 1);
                // add a second datanode to the cluster
                cluster.StartDataNodes(conf, 1, true, null, null, null, null);
                NUnit.Framework.Assert.AreEqual("Number of datanodes should be 2", 2, cluster.GetDataNodes
                                                    ().Count);
                DataNode           dn0          = cluster.GetDataNodes()[0];
                DataNode           dn1          = cluster.GetDataNodes()[1];
                string             activeNNBPId = cluster.GetNamesystem(0).GetBlockPoolId();
                DatanodeDescriptor sourceDnDesc = NameNodeAdapter.GetDatanode(cluster.GetNamesystem
                                                                                  (0), dn0.GetDNRegistrationForBP(activeNNBPId));
                DatanodeDescriptor destDnDesc = NameNodeAdapter.GetDatanode(cluster.GetNamesystem
                                                                                (0), dn1.GetDNRegistrationForBP(activeNNBPId));
                ExtendedBlock block = DFSTestUtil.GetFirstBlock(fs, fileName);
                Log.Info("replaceBlock:  " + ReplaceBlock(block, (DatanodeInfo)sourceDnDesc, (DatanodeInfo
                                                                                              )sourceDnDesc, (DatanodeInfo)destDnDesc));
                // Waiting for the FsDatasetAsyncDsikService to delete the block
                Sharpen.Thread.Sleep(3000);
                // Triggering the incremental block report to report the deleted block to
                // namnemode
                cluster.GetDataNodes()[0].TriggerBlockReport(new BlockReportOptions.Factory().SetIncremental
                                                                 (true).Build());
                cluster.TransitionToStandby(0);
                cluster.TransitionToActive(1);
                NUnit.Framework.Assert.IsTrue("Namenode 1 should be in active state", cluster.GetNameNode
                                                  (1).IsActiveState());
                NUnit.Framework.Assert.IsTrue("Namenode 0 should be in standby state", cluster.GetNameNode
                                                  (0).IsStandbyState());
                client.Close();
                // Opening a new client for new active  namenode
                client = new DFSClient(cluster.GetFileSystem(1).GetUri(), conf);
                IList <LocatedBlock> locatedBlocks1 = client.GetNamenode().GetBlockLocations("/tmp.txt"
                                                                                             , 0, 10L).GetLocatedBlocks();
                NUnit.Framework.Assert.AreEqual(1, locatedBlocks1.Count);
                NUnit.Framework.Assert.AreEqual("The block should be only on 1 datanode ", 1, locatedBlocks1
                                                [0].GetLocations().Length);
            }
            finally
            {
                IOUtils.Cleanup(null, client);
                cluster.Shutdown();
            }
        }
示例#10
0
        /* check if file's blocks have expected number of replicas,
         * and exist at all of includeNodes
         */
        /// <exception cref="System.IO.IOException"/>
        /// <exception cref="Sharpen.TimeoutException"/>
        private void CheckBlocks(DatanodeInfo[] includeNodes, string fileName, long fileLen
                                 , short replFactor, DFSClient client)
        {
            bool notDone;
            long Timeout   = 20000L;
            long starttime = Time.MonotonicNow();
            long failtime  = starttime + Timeout;

            do
            {
                try
                {
                    Sharpen.Thread.Sleep(100);
                }
                catch (Exception)
                {
                }
                IList <LocatedBlock> blocks = client.GetNamenode().GetBlockLocations(fileName, 0,
                                                                                     fileLen).GetLocatedBlocks();
                NUnit.Framework.Assert.AreEqual(1, blocks.Count);
                DatanodeInfo[] nodes = blocks[0].GetLocations();
                notDone = (nodes.Length != replFactor);
                if (notDone)
                {
                    Log.Info("Expected replication factor is " + replFactor + " but the real replication factor is "
                             + nodes.Length);
                }
                else
                {
                    IList <DatanodeInfo> nodeLocations = Arrays.AsList(nodes);
                    foreach (DatanodeInfo node in includeNodes)
                    {
                        if (!nodeLocations.Contains(node))
                        {
                            notDone = true;
                            Log.Info("Block is not located at " + node);
                            break;
                        }
                    }
                }
                if (Time.MonotonicNow() > failtime)
                {
                    string expectedNodesList = string.Empty;
                    string currentNodesList  = string.Empty;
                    foreach (DatanodeInfo dn in includeNodes)
                    {
                        expectedNodesList += dn + ", ";
                    }
                    foreach (DatanodeInfo dn_1 in nodes)
                    {
                        currentNodesList += dn_1 + ", ";
                    }
                    Log.Info("Expected replica nodes are: " + expectedNodesList);
                    Log.Info("Current actual replica nodes are: " + currentNodesList);
                    throw new TimeoutException("Did not achieve expected replication to expected nodes "
                                               + "after more than " + Timeout + " msec.  See logs for details.");
                }
            }while (notDone);
            Log.Info("Achieved expected replication values in " + (Time.Now() - starttime) +
                     " msec.");
        }
示例#11
0
        public virtual void TestReadSelectNonStaleDatanode()
        {
            HdfsConfiguration conf = new HdfsConfiguration();

            conf.SetBoolean(DFSConfigKeys.DfsNamenodeAvoidStaleDatanodeForReadKey, true);
            long staleInterval = 30 * 1000 * 60;

            conf.SetLong(DFSConfigKeys.DfsNamenodeStaleDatanodeIntervalKey, staleInterval);
            MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(numDatanodes
                                                                                   ).Racks(racks).Build();

            cluster.WaitActive();
            IPEndPoint addr   = new IPEndPoint("localhost", cluster.GetNameNodePort());
            DFSClient  client = new DFSClient(addr, conf);
            IList <DatanodeDescriptor> nodeInfoList = cluster.GetNameNode().GetNamesystem().GetBlockManager
                                                          ().GetDatanodeManager().GetDatanodeListForReport(HdfsConstants.DatanodeReportType
                                                                                                           .Live);

            NUnit.Framework.Assert.AreEqual("Unexpected number of datanodes", numDatanodes, nodeInfoList
                                            .Count);
            FileSystem         fileSys = cluster.GetFileSystem();
            FSDataOutputStream stm     = null;

            try
            {
                // do the writing but do not close the FSDataOutputStream
                // in order to mimic the ongoing writing
                Path fileName = new Path("/file1");
                stm = fileSys.Create(fileName, true, fileSys.GetConf().GetInt(CommonConfigurationKeys
                                                                              .IoFileBufferSizeKey, 4096), (short)3, blockSize);
                stm.Write(new byte[(blockSize * 3) / 2]);
                // We do not close the stream so that
                // the writing seems to be still ongoing
                stm.Hflush();
                LocatedBlocks blocks = client.GetNamenode().GetBlockLocations(fileName.ToString()
                                                                              , 0, blockSize);
                DatanodeInfo[] nodes = blocks.Get(0).GetLocations();
                NUnit.Framework.Assert.AreEqual(nodes.Length, 3);
                DataNode           staleNode     = null;
                DatanodeDescriptor staleNodeInfo = null;
                // stop the heartbeat of the first node
                staleNode = this.StopDataNodeHeartbeat(cluster, nodes[0].GetHostName());
                NUnit.Framework.Assert.IsNotNull(staleNode);
                // set the first node as stale
                staleNodeInfo = cluster.GetNameNode().GetNamesystem().GetBlockManager().GetDatanodeManager
                                    ().GetDatanode(staleNode.GetDatanodeId());
                DFSTestUtil.ResetLastUpdatesWithOffset(staleNodeInfo, -(staleInterval + 1));
                LocatedBlocks blocksAfterStale = client.GetNamenode().GetBlockLocations(fileName.
                                                                                        ToString(), 0, blockSize);
                DatanodeInfo[] nodesAfterStale = blocksAfterStale.Get(0).GetLocations();
                NUnit.Framework.Assert.AreEqual(nodesAfterStale.Length, 3);
                NUnit.Framework.Assert.AreEqual(nodesAfterStale[2].GetHostName(), nodes[0].GetHostName
                                                    ());
                // restart the staleNode's heartbeat
                DataNodeTestUtils.SetHeartbeatsDisabledForTests(staleNode, false);
                // reset the first node as non-stale, so as to avoid two stale nodes
                DFSTestUtil.ResetLastUpdatesWithOffset(staleNodeInfo, 0);
                LocatedBlock lastBlock = client.GetLocatedBlocks(fileName.ToString(), 0, long.MaxValue
                                                                 ).GetLastLocatedBlock();
                nodes = lastBlock.GetLocations();
                NUnit.Framework.Assert.AreEqual(nodes.Length, 3);
                // stop the heartbeat of the first node for the last block
                staleNode = this.StopDataNodeHeartbeat(cluster, nodes[0].GetHostName());
                NUnit.Framework.Assert.IsNotNull(staleNode);
                // set the node as stale
                DatanodeDescriptor dnDesc = cluster.GetNameNode().GetNamesystem().GetBlockManager
                                                ().GetDatanodeManager().GetDatanode(staleNode.GetDatanodeId());
                DFSTestUtil.ResetLastUpdatesWithOffset(dnDesc, -(staleInterval + 1));
                LocatedBlock lastBlockAfterStale = client.GetLocatedBlocks(fileName.ToString(), 0
                                                                           , long.MaxValue).GetLastLocatedBlock();
                nodesAfterStale = lastBlockAfterStale.GetLocations();
                NUnit.Framework.Assert.AreEqual(nodesAfterStale.Length, 3);
                NUnit.Framework.Assert.AreEqual(nodesAfterStale[2].GetHostName(), nodes[0].GetHostName
                                                    ());
            }
            finally
            {
                if (stm != null)
                {
                    stm.Close();
                }
                client.Close();
                cluster.Shutdown();
            }
        }
示例#12
0
        public virtual void TestGetBlocks()
        {
            Configuration Conf = new HdfsConfiguration();
            short         ReplicationFactor = (short)2;
            int           DefaultBlockSize  = 1024;
            Random        r = new Random();

            Conf.SetLong(DFSConfigKeys.DfsBlockSizeKey, DefaultBlockSize);
            MiniDFSCluster cluster = new MiniDFSCluster.Builder(Conf).NumDataNodes(ReplicationFactor
                                                                                   ).Build();

            try
            {
                cluster.WaitActive();
                // create a file with two blocks
                FileSystem         fs   = cluster.GetFileSystem();
                FSDataOutputStream @out = fs.Create(new Path("/tmp.txt"), ReplicationFactor);
                byte[]             data = new byte[1024];
                long fileLen            = 2 * DefaultBlockSize;
                long bytesToWrite       = fileLen;
                while (bytesToWrite > 0)
                {
                    r.NextBytes(data);
                    int bytesToWriteNext = (1024 < bytesToWrite) ? 1024 : (int)bytesToWrite;
                    @out.Write(data, 0, bytesToWriteNext);
                    bytesToWrite -= bytesToWriteNext;
                }
                @out.Close();
                // get blocks & data nodes
                IList <LocatedBlock> locatedBlocks;
                DatanodeInfo[]       dataNodes = null;
                bool notWritten;
                do
                {
                    DFSClient dfsclient = new DFSClient(NameNode.GetAddress(Conf), Conf);
                    locatedBlocks = dfsclient.GetNamenode().GetBlockLocations("/tmp.txt", 0, fileLen)
                                    .GetLocatedBlocks();
                    NUnit.Framework.Assert.AreEqual(2, locatedBlocks.Count);
                    notWritten = false;
                    for (int i = 0; i < 2; i++)
                    {
                        dataNodes = locatedBlocks[i].GetLocations();
                        if (dataNodes.Length != ReplicationFactor)
                        {
                            notWritten = true;
                            try
                            {
                                Sharpen.Thread.Sleep(10);
                            }
                            catch (Exception)
                            {
                            }
                            break;
                        }
                    }
                }while (notWritten);
                // get RPC client to namenode
                IPEndPoint       addr     = new IPEndPoint("localhost", cluster.GetNameNodePort());
                NamenodeProtocol namenode = NameNodeProxies.CreateProxy <NamenodeProtocol>(Conf, NameNode
                                                                                           .GetUri(addr)).GetProxy();
                // get blocks of size fileLen from dataNodes[0]
                BlocksWithLocations.BlockWithLocations[] locs;
                locs = namenode.GetBlocks(dataNodes[0], fileLen).GetBlocks();
                NUnit.Framework.Assert.AreEqual(locs.Length, 2);
                NUnit.Framework.Assert.AreEqual(locs[0].GetStorageIDs().Length, 2);
                NUnit.Framework.Assert.AreEqual(locs[1].GetStorageIDs().Length, 2);
                // get blocks of size BlockSize from dataNodes[0]
                locs = namenode.GetBlocks(dataNodes[0], DefaultBlockSize).GetBlocks();
                NUnit.Framework.Assert.AreEqual(locs.Length, 1);
                NUnit.Framework.Assert.AreEqual(locs[0].GetStorageIDs().Length, 2);
                // get blocks of size 1 from dataNodes[0]
                locs = namenode.GetBlocks(dataNodes[0], 1).GetBlocks();
                NUnit.Framework.Assert.AreEqual(locs.Length, 1);
                NUnit.Framework.Assert.AreEqual(locs[0].GetStorageIDs().Length, 2);
                // get blocks of size 0 from dataNodes[0]
                GetBlocksWithException(namenode, dataNodes[0], 0);
                // get blocks of size -1 from dataNodes[0]
                GetBlocksWithException(namenode, dataNodes[0], -1);
                // get blocks of size BlockSize from a non-existent datanode
                DatanodeInfo info = DFSTestUtil.GetDatanodeInfo("1.2.3.4");
                GetBlocksWithException(namenode, info, 2);
            }
            finally
            {
                cluster.Shutdown();
            }
        }
示例#13
0
        public virtual void TestRestartDfsWithAbandonedBlock()
        {
            Configuration conf = new HdfsConfiguration();

            // Turn off persistent IPC, so that the DFSClient can survive NN restart
            conf.SetInt(CommonConfigurationKeysPublic.IpcClientConnectionMaxidletimeKey, 0);
            MiniDFSCluster     cluster = null;
            long               len     = 0;
            FSDataOutputStream stream;

            try
            {
                cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(3).Build();
                FileSystem fs = cluster.GetFileSystem();
                // Creating a file with 4096 blockSize to write multiple blocks
                stream = fs.Create(FilePath, true, BlockSize, (short)1, BlockSize);
                stream.Write(DataBeforeRestart);
                stream.Hflush();
                // Wait for all of the blocks to get through
                while (len < BlockSize * (NumBlocks - 1))
                {
                    FileStatus status = fs.GetFileStatus(FilePath);
                    len = status.GetLen();
                    Sharpen.Thread.Sleep(100);
                }
                // Abandon the last block
                DFSClient      dfsclient  = DFSClientAdapter.GetDFSClient((DistributedFileSystem)fs);
                HdfsFileStatus fileStatus = dfsclient.GetNamenode().GetFileInfo(FileName);
                LocatedBlocks  blocks     = dfsclient.GetNamenode().GetBlockLocations(FileName, 0, BlockSize
                                                                                      * NumBlocks);
                NUnit.Framework.Assert.AreEqual(NumBlocks, blocks.GetLocatedBlocks().Count);
                LocatedBlock b = blocks.GetLastLocatedBlock();
                dfsclient.GetNamenode().AbandonBlock(b.GetBlock(), fileStatus.GetFileId(), FileName
                                                     , dfsclient.clientName);
                // explicitly do NOT close the file.
                cluster.RestartNameNode();
                // Check that the file has no less bytes than before the restart
                // This would mean that blocks were successfully persisted to the log
                FileStatus status_1 = fs.GetFileStatus(FilePath);
                NUnit.Framework.Assert.IsTrue("Length incorrect: " + status_1.GetLen(), status_1.
                                              GetLen() == len - BlockSize);
                // Verify the data showed up from before restart, sans abandoned block.
                FSDataInputStream readStream = fs.Open(FilePath);
                try
                {
                    byte[] verifyBuf = new byte[DataBeforeRestart.Length - BlockSize];
                    IOUtils.ReadFully(readStream, verifyBuf, 0, verifyBuf.Length);
                    byte[] expectedBuf = new byte[DataBeforeRestart.Length - BlockSize];
                    System.Array.Copy(DataBeforeRestart, 0, expectedBuf, 0, expectedBuf.Length);
                    Assert.AssertArrayEquals(expectedBuf, verifyBuf);
                }
                finally
                {
                    IOUtils.CloseStream(readStream);
                }
            }
            finally
            {
                if (cluster != null)
                {
                    cluster.Shutdown();
                }
            }
        }
示例#14
0
        public virtual void TestCopyOnWrite()
        {
            Configuration  conf    = new HdfsConfiguration();
            MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).Build();
            FileSystem     fs      = cluster.GetFileSystem();
            IPEndPoint     addr    = new IPEndPoint("localhost", cluster.GetNameNodePort());
            DFSClient      client  = new DFSClient(addr, conf);

            try
            {
                // create a new file, write to it and close it.
                //
                Path file1             = new Path("/filestatus.dat");
                FSDataOutputStream stm = AppendTestUtil.CreateFile(fs, file1, 1);
                WriteFile(stm);
                stm.Close();
                // Get a handle to the datanode
                DataNode[] dn = cluster.ListDataNodes();
                NUnit.Framework.Assert.IsTrue("There should be only one datanode but found " + dn
                                              .Length, dn.Length == 1);
                LocatedBlocks locations = client.GetNamenode().GetBlockLocations(file1.ToString()
                                                                                 , 0, long.MaxValue);
                IList <LocatedBlock> blocks = locations.GetLocatedBlocks();
                //
                // Create hard links for a few of the blocks
                //
                for (int i = 0; i < blocks.Count; i = i + 2)
                {
                    ExtendedBlock b = blocks[i].GetBlock();
                    FilePath      f = DataNodeTestUtils.GetFile(dn[0], b.GetBlockPoolId(), b.GetLocalBlock
                                                                    ().GetBlockId());
                    FilePath link = new FilePath(f.ToString() + ".link");
                    System.Console.Out.WriteLine("Creating hardlink for File " + f + " to " + link);
                    HardLink.CreateHardLink(f, link);
                }
                //
                // Detach all blocks. This should remove hardlinks (if any)
                //
                for (int i_1 = 0; i_1 < blocks.Count; i_1++)
                {
                    ExtendedBlock b = blocks[i_1].GetBlock();
                    System.Console.Out.WriteLine("testCopyOnWrite detaching block " + b);
                    NUnit.Framework.Assert.IsTrue("Detaching block " + b + " should have returned true"
                                                  , DataNodeTestUtils.UnlinkBlock(dn[0], b, 1));
                }
                // Since the blocks were already detached earlier, these calls should
                // return false
                //
                for (int i_2 = 0; i_2 < blocks.Count; i_2++)
                {
                    ExtendedBlock b = blocks[i_2].GetBlock();
                    System.Console.Out.WriteLine("testCopyOnWrite detaching block " + b);
                    NUnit.Framework.Assert.IsTrue("Detaching block " + b + " should have returned false"
                                                  , !DataNodeTestUtils.UnlinkBlock(dn[0], b, 1));
                }
            }
            finally
            {
                client.Close();
                fs.Close();
                cluster.Shutdown();
            }
        }