internal virtual IList <DatanodeStorageInfo> RemoveZombieStorages() { IList <DatanodeStorageInfo> zombies = null; lock (storageMap) { IEnumerator <KeyValuePair <string, DatanodeStorageInfo> > iter = storageMap.GetEnumerator (); while (iter.HasNext()) { KeyValuePair <string, DatanodeStorageInfo> entry = iter.Next(); DatanodeStorageInfo storageInfo = entry.Value; if (storageInfo.GetLastBlockReportId() != curBlockReportId) { Log.Info(storageInfo.GetStorageID() + " had lastBlockReportId 0x" + long.ToHexString (storageInfo.GetLastBlockReportId()) + ", but curBlockReportId = 0x" + long.ToHexString (curBlockReportId)); iter.Remove(); if (zombies == null) { zombies = new List <DatanodeStorageInfo>(); } zombies.AddItem(storageInfo); } storageInfo.SetLastBlockReportId(0); } } return(zombies == null ? EmptyStorageInfoList : zombies); }
public virtual void TestSafeModeIBR() { DatanodeDescriptor node = Org.Mockito.Mockito.Spy(nodes[0]); DatanodeStorageInfo ds = node.GetStorageInfos()[0]; node.isAlive = true; DatanodeRegistration nodeReg = new DatanodeRegistration(node, null, null, string.Empty ); // pretend to be in safemode Org.Mockito.Mockito.DoReturn(true).When(fsn).IsInStartupSafeMode(); // register new node bm.GetDatanodeManager().RegisterDatanode(nodeReg); bm.GetDatanodeManager().AddDatanode(node); // swap in spy NUnit.Framework.Assert.AreEqual(node, bm.GetDatanodeManager().GetDatanode(node)); NUnit.Framework.Assert.AreEqual(0, ds.GetBlockReportCount()); // send block report, should be processed Org.Mockito.Mockito.Reset(node); bm.ProcessReport(node, new DatanodeStorage(ds.GetStorageID()), BlockListAsLongs.Empty , null, false); NUnit.Framework.Assert.AreEqual(1, ds.GetBlockReportCount()); // send block report again, should NOT be processed Org.Mockito.Mockito.Reset(node); bm.ProcessReport(node, new DatanodeStorage(ds.GetStorageID()), BlockListAsLongs.Empty , null, false); NUnit.Framework.Assert.AreEqual(1, ds.GetBlockReportCount()); // re-register as if node restarted, should update existing node bm.GetDatanodeManager().RemoveDatanode(node); Org.Mockito.Mockito.Reset(node); bm.GetDatanodeManager().RegisterDatanode(nodeReg); Org.Mockito.Mockito.Verify(node).UpdateRegInfo(nodeReg); // send block report, should be processed after restart Org.Mockito.Mockito.Reset(node); bm.ProcessReport(node, new DatanodeStorage(ds.GetStorageID()), BlockListAsLongs.Empty , null, false); // Reinitialize as registration with empty storage list pruned // node.storageMap. ds = node.GetStorageInfos()[0]; NUnit.Framework.Assert.AreEqual(1, ds.GetBlockReportCount()); }
public virtual void TestSafeModeIBRAfterIncremental() { DatanodeDescriptor node = Org.Mockito.Mockito.Spy(nodes[0]); DatanodeStorageInfo ds = node.GetStorageInfos()[0]; node.isAlive = true; DatanodeRegistration nodeReg = new DatanodeRegistration(node, null, null, string.Empty ); // pretend to be in safemode Org.Mockito.Mockito.DoReturn(true).When(fsn).IsInStartupSafeMode(); // register new node bm.GetDatanodeManager().RegisterDatanode(nodeReg); bm.GetDatanodeManager().AddDatanode(node); // swap in spy NUnit.Framework.Assert.AreEqual(node, bm.GetDatanodeManager().GetDatanode(node)); NUnit.Framework.Assert.AreEqual(0, ds.GetBlockReportCount()); // send block report while pretending to already have blocks Org.Mockito.Mockito.Reset(node); Org.Mockito.Mockito.DoReturn(1).When(node).NumBlocks(); bm.ProcessReport(node, new DatanodeStorage(ds.GetStorageID()), BlockListAsLongs.Empty , null, false); NUnit.Framework.Assert.AreEqual(1, ds.GetBlockReportCount()); }
/// <summary> /// Regression test for HDFS-7960.<p/> /// Shutting down a datanode, removing a storage directory, and restarting /// the DataNode should not produce zombie storages. /// </summary> /// <exception cref="System.Exception"/> public virtual void TestRemovingStorageDoesNotProduceZombies() { Configuration conf = new HdfsConfiguration(); conf.SetInt(DFSConfigKeys.DfsDatanodeFailedVolumesToleratedKey, 1); int NumStoragesPerDn = 2; MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(3).StoragesPerDatanode (NumStoragesPerDn).Build(); try { cluster.WaitActive(); foreach (DataNode dn in cluster.GetDataNodes()) { NUnit.Framework.Assert.AreEqual(NumStoragesPerDn, cluster.GetNamesystem().GetBlockManager ().GetDatanodeManager().GetDatanode(dn.GetDatanodeId()).GetStorageInfos().Length ); } // Create a file which will end up on all 3 datanodes. Path TestPath = new Path("/foo1"); DistributedFileSystem fs = cluster.GetFileSystem(); DFSTestUtil.CreateFile(fs, TestPath, 1024, (short)3, unchecked ((int)(0xcafecafe)) ); foreach (DataNode dn_1 in cluster.GetDataNodes()) { DataNodeTestUtils.TriggerBlockReport(dn_1); } ExtendedBlock block = DFSTestUtil.GetFirstBlock(fs, new Path("/foo1")); cluster.GetNamesystem().WriteLock(); string storageIdToRemove; string datanodeUuid; // Find the first storage which this block is in. try { IEnumerator <DatanodeStorageInfo> storageInfoIter = cluster.GetNamesystem().GetBlockManager ().GetStorages(block.GetLocalBlock()).GetEnumerator(); NUnit.Framework.Assert.IsTrue(storageInfoIter.HasNext()); DatanodeStorageInfo info = storageInfoIter.Next(); storageIdToRemove = info.GetStorageID(); datanodeUuid = info.GetDatanodeDescriptor().GetDatanodeUuid(); } finally { cluster.GetNamesystem().WriteUnlock(); } // Find the DataNode which holds that first storage. DataNode datanodeToRemoveStorageFrom; int datanodeToRemoveStorageFromIdx = 0; while (true) { if (datanodeToRemoveStorageFromIdx >= cluster.GetDataNodes().Count) { NUnit.Framework.Assert.Fail("failed to find datanode with uuid " + datanodeUuid); datanodeToRemoveStorageFrom = null; break; } DataNode dn_2 = cluster.GetDataNodes()[datanodeToRemoveStorageFromIdx]; if (dn_2.GetDatanodeUuid().Equals(datanodeUuid)) { datanodeToRemoveStorageFrom = dn_2; break; } datanodeToRemoveStorageFromIdx++; } // Find the volume within the datanode which holds that first storage. IList <FsVolumeSpi> volumes = datanodeToRemoveStorageFrom.GetFSDataset().GetVolumes (); NUnit.Framework.Assert.AreEqual(NumStoragesPerDn, volumes.Count); string volumeDirectoryToRemove = null; foreach (FsVolumeSpi volume in volumes) { if (volume.GetStorageID().Equals(storageIdToRemove)) { volumeDirectoryToRemove = volume.GetBasePath(); } } // Shut down the datanode and remove the volume. // Replace the volume directory with a regular file, which will // cause a volume failure. (If we merely removed the directory, // it would be re-initialized with a new storage ID.) NUnit.Framework.Assert.IsNotNull(volumeDirectoryToRemove); datanodeToRemoveStorageFrom.Shutdown(); FileUtil.FullyDelete(new FilePath(volumeDirectoryToRemove)); FileOutputStream fos = new FileOutputStream(volumeDirectoryToRemove); try { fos.Write(1); } finally { fos.Close(); } cluster.RestartDataNode(datanodeToRemoveStorageFromIdx); // Wait for the NameNode to remove the storage. Log.Info("waiting for the datanode to remove " + storageIdToRemove); GenericTestUtils.WaitFor(new _Supplier_227(cluster, datanodeToRemoveStorageFrom, storageIdToRemove, NumStoragesPerDn), 10, 30000); } finally { if (cluster != null) { cluster.Shutdown(); } } }
public virtual void TestSafeModeIBRBeforeFirstFullBR() { // pretend to be in safemode Org.Mockito.Mockito.DoReturn(true).When(fsn).IsInStartupSafeMode(); DatanodeDescriptor node = nodes[0]; DatanodeStorageInfo ds = node.GetStorageInfos()[0]; node.isAlive = true; DatanodeRegistration nodeReg = new DatanodeRegistration(node, null, null, string.Empty ); // register new node bm.GetDatanodeManager().RegisterDatanode(nodeReg); bm.GetDatanodeManager().AddDatanode(node); NUnit.Framework.Assert.AreEqual(node, bm.GetDatanodeManager().GetDatanode(node)); NUnit.Framework.Assert.AreEqual(0, ds.GetBlockReportCount()); // Build a incremental report IList <ReceivedDeletedBlockInfo> rdbiList = new AList <ReceivedDeletedBlockInfo>(); // Build a full report BlockListAsLongs.Builder builder = BlockListAsLongs.Builder(); // blk_42 is finalized. long receivedBlockId = 42; // arbitrary BlockInfoContiguous receivedBlock = AddBlockToBM(receivedBlockId); rdbiList.AddItem(new ReceivedDeletedBlockInfo(new Block(receivedBlock), ReceivedDeletedBlockInfo.BlockStatus .ReceivedBlock, null)); builder.Add(new FinalizedReplica(receivedBlock, null, null)); // blk_43 is under construction. long receivingBlockId = 43; BlockInfoContiguous receivingBlock = AddUcBlockToBM(receivingBlockId); rdbiList.AddItem(new ReceivedDeletedBlockInfo(new Block(receivingBlock), ReceivedDeletedBlockInfo.BlockStatus .ReceivingBlock, null)); builder.Add(new ReplicaBeingWritten(receivingBlock, null, null, null)); // blk_44 has 2 records in IBR. It's finalized. So full BR has 1 record. long receivingReceivedBlockId = 44; BlockInfoContiguous receivingReceivedBlock = AddBlockToBM(receivingReceivedBlockId ); rdbiList.AddItem(new ReceivedDeletedBlockInfo(new Block(receivingReceivedBlock), ReceivedDeletedBlockInfo.BlockStatus.ReceivingBlock, null)); rdbiList.AddItem(new ReceivedDeletedBlockInfo(new Block(receivingReceivedBlock), ReceivedDeletedBlockInfo.BlockStatus.ReceivedBlock, null)); builder.Add(new FinalizedReplica(receivingReceivedBlock, null, null)); // blk_45 is not in full BR, because it's deleted. long ReceivedDeletedBlockId = 45; rdbiList.AddItem(new ReceivedDeletedBlockInfo(new Block(ReceivedDeletedBlockId), ReceivedDeletedBlockInfo.BlockStatus.ReceivedBlock, null)); rdbiList.AddItem(new ReceivedDeletedBlockInfo(new Block(ReceivedDeletedBlockId), ReceivedDeletedBlockInfo.BlockStatus.DeletedBlock, null)); // blk_46 exists in DN for a long time, so it's in full BR, but not in IBR. long existedBlockId = 46; BlockInfoContiguous existedBlock = AddBlockToBM(existedBlockId); builder.Add(new FinalizedReplica(existedBlock, null, null)); // process IBR and full BR StorageReceivedDeletedBlocks srdb = new StorageReceivedDeletedBlocks(new DatanodeStorage (ds.GetStorageID()), Sharpen.Collections.ToArray(rdbiList, new ReceivedDeletedBlockInfo [rdbiList.Count])); bm.ProcessIncrementalBlockReport(node, srdb); // Make sure it's the first full report NUnit.Framework.Assert.AreEqual(0, ds.GetBlockReportCount()); bm.ProcessReport(node, new DatanodeStorage(ds.GetStorageID()), builder.Build(), null , false); NUnit.Framework.Assert.AreEqual(1, ds.GetBlockReportCount()); // verify the storage info is correct NUnit.Framework.Assert.IsTrue(bm.GetStoredBlock(new Block(receivedBlockId)).FindStorageInfo (ds) >= 0); NUnit.Framework.Assert.IsTrue(((BlockInfoContiguousUnderConstruction)bm.GetStoredBlock (new Block(receivingBlockId))).GetNumExpectedLocations() > 0); NUnit.Framework.Assert.IsTrue(bm.GetStoredBlock(new Block(receivingReceivedBlockId )).FindStorageInfo(ds) >= 0); NUnit.Framework.Assert.IsNull(bm.GetStoredBlock(new Block(ReceivedDeletedBlockId) )); NUnit.Framework.Assert.IsTrue(bm.GetStoredBlock(new Block(existedBlock)).FindStorageInfo (ds) >= 0); }