public virtual void TestPendingDeleteUnknownBlocks() { int fileNum = 5; // 5 files Path[] files = new Path[fileNum]; MiniDFSCluster.DataNodeProperties[] dnprops = new MiniDFSCluster.DataNodeProperties [Replication]; // create a group of files, each file contains 1 block for (int i = 0; i < fileNum; i++) { files[i] = new Path("/file" + i); DFSTestUtil.CreateFile(dfs, files[i], Blocksize, Replication, i); } // wait until all DataNodes have replicas WaitForReplication(); for (int i_1 = Replication - 1; i_1 >= 0; i_1--) { dnprops[i_1] = cluster.StopDataNode(i_1); } Sharpen.Thread.Sleep(2000); // delete 2 files, we still have 3 files remaining so that we can cover // every DN storage for (int i_2 = 0; i_2 < 2; i_2++) { dfs.Delete(files[i_2], true); } // restart NameNode cluster.RestartNameNode(false); InvalidateBlocks invalidateBlocks = (InvalidateBlocks)Whitebox.GetInternalState(cluster .GetNamesystem().GetBlockManager(), "invalidateBlocks"); InvalidateBlocks mockIb = Org.Mockito.Mockito.Spy(invalidateBlocks); Org.Mockito.Mockito.DoReturn(1L).When(mockIb).GetInvalidationDelay(); Whitebox.SetInternalState(cluster.GetNamesystem().GetBlockManager(), "invalidateBlocks" , mockIb); NUnit.Framework.Assert.AreEqual(0L, cluster.GetNamesystem().GetPendingDeletionBlocks ()); // restart DataNodes for (int i_3 = 0; i_3 < Replication; i_3++) { cluster.RestartDataNode(dnprops[i_3], true); } cluster.WaitActive(); for (int i_4 = 0; i_4 < Replication; i_4++) { DataNodeTestUtils.TriggerBlockReport(cluster.GetDataNodes()[i_4]); } Sharpen.Thread.Sleep(2000); // make sure we have received block reports by checking the total block # NUnit.Framework.Assert.AreEqual(3, cluster.GetNamesystem().GetBlocksTotal()); NUnit.Framework.Assert.AreEqual(4, cluster.GetNamesystem().GetPendingDeletionBlocks ()); cluster.RestartNameNode(true); Sharpen.Thread.Sleep(6000); NUnit.Framework.Assert.AreEqual(3, cluster.GetNamesystem().GetBlocksTotal()); NUnit.Framework.Assert.AreEqual(0, cluster.GetNamesystem().GetPendingDeletionBlocks ()); }
// let the NN finish deletion /// <summary>Verify block locations after running the migration tool.</summary> /// <exception cref="System.Exception"/> internal virtual void Verify(bool verifyAll) { foreach (DataNode dn in this.cluster.GetDataNodes()) { DataNodeTestUtils.TriggerBlockReport(dn); } if (verifyAll) { this.VerifyNamespace(); } }
/// <exception cref="System.IO.IOException"/> private static void RunTest(string testCaseName, bool createFiles, int numInitialStorages , int expectedStoragesAfterTest) { Configuration conf = new HdfsConfiguration(); MiniDFSCluster cluster = null; try { cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(1).StoragesPerDatanode(numInitialStorages ).Build(); cluster.WaitActive(); DataNode dn0 = cluster.GetDataNodes()[0]; // Ensure NN knows about the storage. DatanodeID dnId = dn0.GetDatanodeId(); DatanodeDescriptor dnDescriptor = cluster.GetNamesystem().GetBlockManager().GetDatanodeManager ().GetDatanode(dnId); Assert.AssertThat(dnDescriptor.GetStorageInfos().Length, IS.Is(numInitialStorages )); string bpid = cluster.GetNamesystem().GetBlockPoolId(); DatanodeRegistration dnReg = dn0.GetDNRegistrationForBP(bpid); DataNodeTestUtils.TriggerBlockReport(dn0); if (createFiles) { Path path = new Path("/", testCaseName); DFSTestUtil.CreateFile(cluster.GetFileSystem(), path, 1024, (short)1, unchecked (( int)(0x1BAD5EED))); DataNodeTestUtils.TriggerBlockReport(dn0); } // Generate a fake StorageReport that is missing one storage. StorageReport[] reports = dn0.GetFSDataset().GetStorageReports(bpid); StorageReport[] prunedReports = new StorageReport[numInitialStorages - 1]; System.Array.Copy(reports, 0, prunedReports, 0, prunedReports.Length); // Stop the DataNode and send fake heartbeat with missing storage. cluster.StopDataNode(0); cluster.GetNameNodeRpc().SendHeartbeat(dnReg, prunedReports, 0L, 0L, 0, 0, 0, null ); // Check that the missing storage was pruned. Assert.AssertThat(dnDescriptor.GetStorageInfos().Length, IS.Is(expectedStoragesAfterTest )); } finally { if (cluster != null) { cluster.Shutdown(); } } }
/// <summary> /// Regression test for HDFS-7960.<p/> /// Shutting down a datanode, removing a storage directory, and restarting /// the DataNode should not produce zombie storages. /// </summary> /// <exception cref="System.Exception"/> public virtual void TestRemovingStorageDoesNotProduceZombies() { Configuration conf = new HdfsConfiguration(); conf.SetInt(DFSConfigKeys.DfsDatanodeFailedVolumesToleratedKey, 1); int NumStoragesPerDn = 2; MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(3).StoragesPerDatanode (NumStoragesPerDn).Build(); try { cluster.WaitActive(); foreach (DataNode dn in cluster.GetDataNodes()) { NUnit.Framework.Assert.AreEqual(NumStoragesPerDn, cluster.GetNamesystem().GetBlockManager ().GetDatanodeManager().GetDatanode(dn.GetDatanodeId()).GetStorageInfos().Length ); } // Create a file which will end up on all 3 datanodes. Path TestPath = new Path("/foo1"); DistributedFileSystem fs = cluster.GetFileSystem(); DFSTestUtil.CreateFile(fs, TestPath, 1024, (short)3, unchecked ((int)(0xcafecafe)) ); foreach (DataNode dn_1 in cluster.GetDataNodes()) { DataNodeTestUtils.TriggerBlockReport(dn_1); } ExtendedBlock block = DFSTestUtil.GetFirstBlock(fs, new Path("/foo1")); cluster.GetNamesystem().WriteLock(); string storageIdToRemove; string datanodeUuid; // Find the first storage which this block is in. try { IEnumerator <DatanodeStorageInfo> storageInfoIter = cluster.GetNamesystem().GetBlockManager ().GetStorages(block.GetLocalBlock()).GetEnumerator(); NUnit.Framework.Assert.IsTrue(storageInfoIter.HasNext()); DatanodeStorageInfo info = storageInfoIter.Next(); storageIdToRemove = info.GetStorageID(); datanodeUuid = info.GetDatanodeDescriptor().GetDatanodeUuid(); } finally { cluster.GetNamesystem().WriteUnlock(); } // Find the DataNode which holds that first storage. DataNode datanodeToRemoveStorageFrom; int datanodeToRemoveStorageFromIdx = 0; while (true) { if (datanodeToRemoveStorageFromIdx >= cluster.GetDataNodes().Count) { NUnit.Framework.Assert.Fail("failed to find datanode with uuid " + datanodeUuid); datanodeToRemoveStorageFrom = null; break; } DataNode dn_2 = cluster.GetDataNodes()[datanodeToRemoveStorageFromIdx]; if (dn_2.GetDatanodeUuid().Equals(datanodeUuid)) { datanodeToRemoveStorageFrom = dn_2; break; } datanodeToRemoveStorageFromIdx++; } // Find the volume within the datanode which holds that first storage. IList <FsVolumeSpi> volumes = datanodeToRemoveStorageFrom.GetFSDataset().GetVolumes (); NUnit.Framework.Assert.AreEqual(NumStoragesPerDn, volumes.Count); string volumeDirectoryToRemove = null; foreach (FsVolumeSpi volume in volumes) { if (volume.GetStorageID().Equals(storageIdToRemove)) { volumeDirectoryToRemove = volume.GetBasePath(); } } // Shut down the datanode and remove the volume. // Replace the volume directory with a regular file, which will // cause a volume failure. (If we merely removed the directory, // it would be re-initialized with a new storage ID.) NUnit.Framework.Assert.IsNotNull(volumeDirectoryToRemove); datanodeToRemoveStorageFrom.Shutdown(); FileUtil.FullyDelete(new FilePath(volumeDirectoryToRemove)); FileOutputStream fos = new FileOutputStream(volumeDirectoryToRemove); try { fos.Write(1); } finally { fos.Close(); } cluster.RestartDataNode(datanodeToRemoveStorageFromIdx); // Wait for the NameNode to remove the storage. Log.Info("waiting for the datanode to remove " + storageIdToRemove); GenericTestUtils.WaitFor(new _Supplier_227(cluster, datanodeToRemoveStorageFrom, storageIdToRemove, NumStoragesPerDn), 10, 30000); } finally { if (cluster != null) { cluster.Shutdown(); } } }
/// <exception cref="System.IO.IOException"/> /// <exception cref="System.Exception"/> protected internal void TriggerBlockReport() { // Trigger block report to NN DataNodeTestUtils.TriggerBlockReport(cluster.GetDataNodes()[0]); Sharpen.Thread.Sleep(10 * 1000); }
public virtual void TestDecommissionStatus() { IPEndPoint addr = new IPEndPoint("localhost", cluster.GetNameNodePort()); DFSClient client = new DFSClient(addr, conf); DatanodeInfo[] info = client.DatanodeReport(HdfsConstants.DatanodeReportType.Live ); NUnit.Framework.Assert.AreEqual("Number of Datanodes ", 2, info.Length); DistributedFileSystem fileSys = cluster.GetFileSystem(); DFSAdmin admin = new DFSAdmin(cluster.GetConfiguration(0)); short replicas = numDatanodes; // // Decommission one node. Verify the decommission status // Path file1 = new Path("decommission.dat"); WriteFile(fileSys, file1, replicas); Path file2 = new Path("decommission1.dat"); FSDataOutputStream st1 = WriteIncompleteFile(fileSys, file2, replicas); foreach (DataNode d in cluster.GetDataNodes()) { DataNodeTestUtils.TriggerBlockReport(d); } FSNamesystem fsn = cluster.GetNamesystem(); DatanodeManager dm = fsn.GetBlockManager().GetDatanodeManager(); for (int iteration = 0; iteration < numDatanodes; iteration++) { string downnode = DecommissionNode(fsn, client, localFileSys, iteration); dm.RefreshNodes(conf); decommissionedNodes.AddItem(downnode); BlockManagerTestUtil.RecheckDecommissionState(dm); IList <DatanodeDescriptor> decommissioningNodes = dm.GetDecommissioningNodes(); if (iteration == 0) { NUnit.Framework.Assert.AreEqual(decommissioningNodes.Count, 1); DatanodeDescriptor decommNode = decommissioningNodes[0]; CheckDecommissionStatus(decommNode, 3, 0, 1); CheckDFSAdminDecommissionStatus(decommissioningNodes.SubList(0, 1), fileSys, admin ); } else { NUnit.Framework.Assert.AreEqual(decommissioningNodes.Count, 2); DatanodeDescriptor decommNode1 = decommissioningNodes[0]; DatanodeDescriptor decommNode2 = decommissioningNodes[1]; // This one is still 3,3,1 since it passed over the UC block // earlier, before node 2 was decommed CheckDecommissionStatus(decommNode1, 3, 3, 1); // This one is 4,4,2 since it has the full state CheckDecommissionStatus(decommNode2, 4, 4, 2); CheckDFSAdminDecommissionStatus(decommissioningNodes.SubList(0, 2), fileSys, admin ); } } // Call refreshNodes on FSNamesystem with empty exclude file. // This will remove the datanodes from decommissioning list and // make them available again. WriteConfigFile(localFileSys, excludeFile, null); dm.RefreshNodes(conf); st1.Close(); CleanupFile(fileSys, file1); CleanupFile(fileSys, file2); }