public virtual void TestUpdatePipelineAfterDelete()
        {
            Configuration  conf    = new HdfsConfiguration();
            Path           file    = new Path("/test-file");
            MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).Build();

            try
            {
                FileSystem        fs       = cluster.GetFileSystem();
                NamenodeProtocols namenode = cluster.GetNameNodeRpc();
                DFSOutputStream   @out     = null;
                try
                {
                    // Create a file and make sure a block is allocated for it.
                    @out = (DFSOutputStream)(fs.Create(file).GetWrappedStream());
                    @out.Write(1);
                    @out.Hflush();
                    // Create a snapshot that includes the file.
                    SnapshotTestHelper.CreateSnapshot((DistributedFileSystem)fs, new Path("/"), "s1");
                    // Grab the block info of this file for later use.
                    FSDataInputStream @in      = null;
                    ExtendedBlock     oldBlock = null;
                    try
                    {
                        @in      = fs.Open(file);
                        oldBlock = DFSTestUtil.GetAllBlocks(@in)[0].GetBlock();
                    }
                    finally
                    {
                        IOUtils.CloseStream(@in);
                    }
                    // Allocate a new block ID/gen stamp so we can simulate pipeline
                    // recovery.
                    string       clientName      = ((DistributedFileSystem)fs).GetClient().GetClientName();
                    LocatedBlock newLocatedBlock = namenode.UpdateBlockForPipeline(oldBlock, clientName
                                                                                   );
                    ExtendedBlock newBlock = new ExtendedBlock(oldBlock.GetBlockPoolId(), oldBlock.GetBlockId
                                                                   (), oldBlock.GetNumBytes(), newLocatedBlock.GetBlock().GetGenerationStamp());
                    // Delete the file from the present FS. It will still exist the
                    // previously-created snapshot. This will log an OP_DELETE for the
                    // file in question.
                    fs.Delete(file, true);
                    // Simulate a pipeline recovery, wherein a new block is allocated
                    // for the existing block, resulting in an OP_UPDATE_BLOCKS being
                    // logged for the file in question.
                    try
                    {
                        namenode.UpdatePipeline(clientName, oldBlock, newBlock, newLocatedBlock.GetLocations
                                                    (), newLocatedBlock.GetStorageIDs());
                    }
                    catch (IOException ioe)
                    {
                        // normal
                        GenericTestUtils.AssertExceptionContains("does not exist or it is not under construction"
                                                                 , ioe);
                    }
                    // Make sure the NN can restart with the edit logs as we have them now.
                    cluster.RestartNameNode(true);
                }
                finally
                {
                    IOUtils.CloseStream(@out);
                }
            }
            finally
            {
                cluster.Shutdown();
            }
        }
Example #2
0
        public virtual void TestSortLocatedBlocks()
        {
            // create the DatanodeManager which will be tested
            FSNamesystem fsn = Org.Mockito.Mockito.Mock <FSNamesystem>();

            Org.Mockito.Mockito.When(fsn.HasWriteLock()).ThenReturn(true);
            DatanodeManager dm = new DatanodeManager(Org.Mockito.Mockito.Mock <BlockManager>()
                                                     , fsn, new Configuration());

            // register 5 datanodes, each with different storage ID and type
            DatanodeInfo[] locs         = new DatanodeInfo[5];
            string[]       storageIDs   = new string[5];
            StorageType[]  storageTypes = new StorageType[] { StorageType.Archive, StorageType
                                                              .Default, StorageType.Disk, StorageType.RamDisk, StorageType.Ssd };
            for (int i = 0; i < 5; i++)
            {
                // register new datanode
                string uuid             = "UUID-" + i;
                string ip               = "IP-" + i;
                DatanodeRegistration dr = Org.Mockito.Mockito.Mock <DatanodeRegistration>();
                Org.Mockito.Mockito.When(dr.GetDatanodeUuid()).ThenReturn(uuid);
                Org.Mockito.Mockito.When(dr.GetIpAddr()).ThenReturn(ip);
                Org.Mockito.Mockito.When(dr.GetXferAddr()).ThenReturn(ip + ":9000");
                Org.Mockito.Mockito.When(dr.GetXferPort()).ThenReturn(9000);
                Org.Mockito.Mockito.When(dr.GetSoftwareVersion()).ThenReturn("version1");
                dm.RegisterDatanode(dr);
                // get location and storage information
                locs[i]       = dm.GetDatanode(uuid);
                storageIDs[i] = "storageID-" + i;
            }
            // set first 2 locations as decomissioned
            locs[0].SetDecommissioned();
            locs[1].SetDecommissioned();
            // create LocatedBlock with above locations
            ExtendedBlock        b      = new ExtendedBlock("somePoolID", 1234);
            LocatedBlock         block  = new LocatedBlock(b, locs, storageIDs, storageTypes);
            IList <LocatedBlock> blocks = new AList <LocatedBlock>();

            blocks.AddItem(block);
            string targetIp = locs[4].GetIpAddr();

            // sort block locations
            dm.SortLocatedBlocks(targetIp, blocks);
            // check that storage IDs/types are aligned with datanode locs
            DatanodeInfo[] sortedLocs = block.GetLocations();
            storageIDs   = block.GetStorageIDs();
            storageTypes = block.GetStorageTypes();
            Assert.AssertThat(sortedLocs.Length, IS.Is(5));
            Assert.AssertThat(storageIDs.Length, IS.Is(5));
            Assert.AssertThat(storageTypes.Length, IS.Is(5));
            for (int i_1 = 0; i_1 < sortedLocs.Length; i_1++)
            {
                Assert.AssertThat(((DatanodeInfoWithStorage)sortedLocs[i_1]).GetStorageID(), IS.Is
                                      (storageIDs[i_1]));
                Assert.AssertThat(((DatanodeInfoWithStorage)sortedLocs[i_1]).GetStorageType(), IS.Is
                                      (storageTypes[i_1]));
            }
            // Ensure the local node is first.
            Assert.AssertThat(sortedLocs[0].GetIpAddr(), IS.Is(targetIp));
            // Ensure the two decommissioned DNs were moved to the end.
            Assert.AssertThat(sortedLocs[sortedLocs.Length - 1].GetAdminState(), IS.Is(DatanodeInfo.AdminStates
                                                                                       .Decommissioned));
            Assert.AssertThat(sortedLocs[sortedLocs.Length - 2].GetAdminState(), IS.Is(DatanodeInfo.AdminStates
                                                                                       .Decommissioned));
        }