/// <exception cref="System.IO.IOException"/> public virtual DatanodeCommand CacheReport(DatanodeRegistration registration, string poolId, IList <long> blockIds) { DatanodeProtocolProtos.CacheReportRequestProto.Builder builder = DatanodeProtocolProtos.CacheReportRequestProto .NewBuilder().SetRegistration(PBHelper.Convert(registration)).SetBlockPoolId(poolId ); foreach (long blockId in blockIds) { builder.AddBlocks(blockId); } DatanodeProtocolProtos.CacheReportResponseProto resp; try { resp = rpcProxy.CacheReport(NullController, ((DatanodeProtocolProtos.CacheReportRequestProto )builder.Build())); } catch (ServiceException se) { throw ProtobufHelper.GetRemoteException(se); } if (resp.HasCmd()) { return(PBHelper.Convert(resp.GetCmd())); } return(null); }
public virtual void TestChooseReplicaToDelete() { MiniDFSCluster cluster = null; FileSystem fs = null; try { Configuration conf = new HdfsConfiguration(); conf.SetLong(DFSConfigKeys.DfsBlockSizeKey, SmallBlockSize); cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(3).Build(); fs = cluster.GetFileSystem(); FSNamesystem namesystem = cluster.GetNamesystem(); conf.SetLong(DFSConfigKeys.DfsHeartbeatIntervalKey, 300); cluster.StartDataNodes(conf, 1, true, null, null, null); DataNode lastDN = cluster.GetDataNodes()[3]; DatanodeRegistration dnReg = DataNodeTestUtils.GetDNRegistrationForBP(lastDN, namesystem .GetBlockPoolId()); string lastDNid = dnReg.GetDatanodeUuid(); Path fileName = new Path("/foo2"); DFSTestUtil.CreateFile(fs, fileName, SmallFileLength, (short)4, 0L); DFSTestUtil.WaitReplication(fs, fileName, (short)4); // Wait for tolerable number of heartbeats plus one DatanodeDescriptor nodeInfo = null; long lastHeartbeat = 0; long waitTime = DFSConfigKeys.DfsHeartbeatIntervalDefault * 1000 * (DFSConfigKeys .DfsNamenodeTolerateHeartbeatMultiplierDefault + 1); do { nodeInfo = namesystem.GetBlockManager().GetDatanodeManager().GetDatanode(dnReg); lastHeartbeat = nodeInfo.GetLastUpdateMonotonic(); }while (Time.MonotonicNow() - lastHeartbeat < waitTime); fs.SetReplication(fileName, (short)3); BlockLocation[] locs = fs.GetFileBlockLocations(fs.GetFileStatus(fileName), 0, long.MaxValue ); // All replicas for deletion should be scheduled on lastDN. // And should not actually be deleted, because lastDN does not heartbeat. namesystem.ReadLock(); ICollection <Block> dnBlocks = namesystem.GetBlockManager().excessReplicateMap[lastDNid ]; NUnit.Framework.Assert.AreEqual("Replicas on node " + lastDNid + " should have been deleted" , SmallFileLength / SmallBlockSize, dnBlocks.Count); namesystem.ReadUnlock(); foreach (BlockLocation location in locs) { NUnit.Framework.Assert.AreEqual("Block should still have 4 replicas", 4, location .GetNames().Length); } } finally { if (fs != null) { fs.Close(); } if (cluster != null) { cluster.Shutdown(); } } }
/// <exception cref="System.IO.IOException"/> public virtual void BlockReceivedAndDeleted(DatanodeRegistration registration, string poolId, StorageReceivedDeletedBlocks[] receivedAndDeletedBlocks) { DatanodeProtocolProtos.BlockReceivedAndDeletedRequestProto.Builder builder = DatanodeProtocolProtos.BlockReceivedAndDeletedRequestProto .NewBuilder().SetRegistration(PBHelper.Convert(registration)).SetBlockPoolId(poolId ); foreach (StorageReceivedDeletedBlocks storageBlock in receivedAndDeletedBlocks) { DatanodeProtocolProtos.StorageReceivedDeletedBlocksProto.Builder repBuilder = DatanodeProtocolProtos.StorageReceivedDeletedBlocksProto .NewBuilder(); repBuilder.SetStorageUuid(storageBlock.GetStorage().GetStorageID()); // Set for wire compatibility. repBuilder.SetStorage(PBHelper.Convert(storageBlock.GetStorage())); foreach (ReceivedDeletedBlockInfo rdBlock in storageBlock.GetBlocks()) { repBuilder.AddBlocks(PBHelper.Convert(rdBlock)); } builder.AddBlocks(((DatanodeProtocolProtos.StorageReceivedDeletedBlocksProto)repBuilder .Build())); } try { rpcProxy.BlockReceivedAndDeleted(NullController, ((DatanodeProtocolProtos.BlockReceivedAndDeletedRequestProto )builder.Build())); } catch (ServiceException se) { throw ProtobufHelper.GetRemoteException(se); } }
/// <summary> /// After one of the BPServiceActors registers successfully with the /// NN, it calls this function to verify that the NN it connected to /// is consistent with other NNs serving the block-pool. /// </summary> /// <exception cref="System.IO.IOException"/> internal virtual void RegistrationSucceeded(BPServiceActor bpServiceActor, DatanodeRegistration reg) { WriteLock(); try { if (bpRegistration != null) { CheckNSEquality(bpRegistration.GetStorageInfo().GetNamespaceID(), reg.GetStorageInfo ().GetNamespaceID(), "namespace ID"); CheckNSEquality(bpRegistration.GetStorageInfo().GetClusterID(), reg.GetStorageInfo ().GetClusterID(), "cluster ID"); } bpRegistration = reg; dn.BpRegistrationSucceeded(bpRegistration, GetBlockPoolId()); // Add the initial block token secret keys to the DN's secret manager. if (dn.isBlockTokenEnabled) { dn.blockPoolTokenSecretManager.AddKeys(GetBlockPoolId(), reg.GetExportedKeys()); } } finally { WriteUnlock(); } }
/// <exception cref="System.IO.IOException"/> protected internal override void SendBlockReports(DatanodeRegistration dnR, string poolId, StorageBlockReport[] reports) { Log.Info("Sending combined block reports for " + dnR); cluster.GetNameNodeRpc().BlockReport(dnR, poolId, reports, new BlockReportContext (1, 0, Runtime.NanoTime())); }
/// <summary> /// Similar to BlockReport_03() but works with two DNs /// Test writes a file and closes it. /// </summary> /// <remarks> /// Similar to BlockReport_03() but works with two DNs /// Test writes a file and closes it. /// The second datanode is started in the cluster. /// As soon as the replication process is completed test finds a block from /// the second DN and sets its GS to be < of original one. /// this is the markBlockAsCorrupt case 3 so we expect one pending deletion /// Block report is forced and the check for # of currupted blocks is performed. /// Another block is chosen and its length is set to a lesser than original. /// A check for another corrupted block is performed after yet another /// BlockReport /// </remarks> /// <exception cref="System.IO.IOException">in case of an error</exception> /// <exception cref="System.Exception"/> public virtual void BlockReport_07() { string MethodName = GenericTestUtils.GetMethodName(); Path filePath = new Path("/" + MethodName + ".dat"); int DnN1 = DnN0 + 1; // write file and start second node to be "older" than the original WriteFile(MethodName, FileSize, filePath); StartDNandWait(filePath, true); // all blocks belong to the same file, hence same BP DataNode dn = cluster.GetDataNodes()[DnN1]; string poolId = cluster.GetNamesystem().GetBlockPoolId(); DatanodeRegistration dnR = dn.GetDNRegistrationForBP(poolId); StorageBlockReport[] reports = GetBlockReports(dn, poolId, true, false); SendBlockReports(dnR, poolId, reports); PrintStats(); Assert.AssertThat("Wrong number of corrupt blocks", cluster.GetNamesystem().GetCorruptReplicaBlocks (), IS.Is(0L)); Assert.AssertThat("Wrong number of PendingDeletion blocks", cluster.GetNamesystem ().GetPendingDeletionBlocks(), IS.Is(1L)); Assert.AssertThat("Wrong number of PendingReplication blocks", cluster.GetNamesystem ().GetPendingReplicationBlocks(), IS.Is(0L)); reports = GetBlockReports(dn, poolId, false, true); SendBlockReports(dnR, poolId, reports); PrintStats(); Assert.AssertThat("Wrong number of corrupt blocks", cluster.GetNamesystem().GetCorruptReplicaBlocks (), IS.Is(1L)); Assert.AssertThat("Wrong number of PendingDeletion blocks", cluster.GetNamesystem ().GetPendingDeletionBlocks(), IS.Is(1L)); Assert.AssertThat("Wrong number of PendingReplication blocks", cluster.GetNamesystem ().GetPendingReplicationBlocks(), IS.Is(0L)); PrintStats(); }
public virtual void TestArrayOutOfBoundsException() { MiniDFSCluster cluster = null; try { Configuration conf = new HdfsConfiguration(); cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(2).Build(); cluster.WaitActive(); FileSystem fs = cluster.GetFileSystem(); Path FilePath = new Path("/tmp.txt"); long FileLen = 1L; DFSTestUtil.CreateFile(fs, FilePath, FileLen, (short)2, 1L); // get the block string bpid = cluster.GetNamesystem().GetBlockPoolId(); FilePath storageDir = cluster.GetInstanceStorageDir(0, 0); FilePath dataDir = MiniDFSCluster.GetFinalizedDir(storageDir, bpid); NUnit.Framework.Assert.IsTrue("Data directory does not exist", dataDir.Exists()); ExtendedBlock blk = GetBlock(bpid, dataDir); if (blk == null) { storageDir = cluster.GetInstanceStorageDir(0, 1); dataDir = MiniDFSCluster.GetFinalizedDir(storageDir, bpid); blk = GetBlock(bpid, dataDir); } NUnit.Framework.Assert.IsFalse("Data directory does not contain any blocks or there was an " + "IO error", blk == null); // start a third datanode cluster.StartDataNodes(conf, 1, true, null, null); AList <DataNode> datanodes = cluster.GetDataNodes(); NUnit.Framework.Assert.AreEqual(datanodes.Count, 3); DataNode dataNode = datanodes[2]; // report corrupted block by the third datanode DatanodeRegistration dnR = DataNodeTestUtils.GetDNRegistrationForBP(dataNode, blk .GetBlockPoolId()); FSNamesystem ns = cluster.GetNamesystem(); ns.WriteLock(); try { cluster.GetNamesystem().GetBlockManager().FindAndMarkBlockAsCorrupt(blk, new DatanodeInfo (dnR), "TEST", "STORAGE_ID"); } finally { ns.WriteUnlock(); } // open the file fs.Open(FilePath); //clean up fs.Delete(FilePath, false); } finally { if (cluster != null) { cluster.Shutdown(); } } }
public virtual void TestDeadDatanode() { Configuration conf = new HdfsConfiguration(); conf.SetInt(DFSConfigKeys.DfsNamenodeHeartbeatRecheckIntervalKey, 500); conf.SetLong(DFSConfigKeys.DfsHeartbeatIntervalKey, 1L); cluster = new MiniDFSCluster.Builder(conf).Build(); cluster.WaitActive(); string poolId = cluster.GetNamesystem().GetBlockPoolId(); // wait for datanode to be marked live DataNode dn = cluster.GetDataNodes()[0]; DatanodeRegistration reg = DataNodeTestUtils.GetDNRegistrationForBP(cluster.GetDataNodes ()[0], poolId); DFSTestUtil.WaitForDatanodeState(cluster, reg.GetDatanodeUuid(), true, 20000); // Shutdown and wait for datanode to be marked dead dn.Shutdown(); DFSTestUtil.WaitForDatanodeState(cluster, reg.GetDatanodeUuid(), false, 20000); DatanodeProtocol dnp = cluster.GetNameNodeRpc(); ReceivedDeletedBlockInfo[] blocks = new ReceivedDeletedBlockInfo[] { new ReceivedDeletedBlockInfo (new Block(0), ReceivedDeletedBlockInfo.BlockStatus.ReceivedBlock, null) }; StorageReceivedDeletedBlocks[] storageBlocks = new StorageReceivedDeletedBlocks[] { new StorageReceivedDeletedBlocks(reg.GetDatanodeUuid(), blocks) }; // Ensure blockReceived call from dead datanode is rejected with IOException try { dnp.BlockReceivedAndDeleted(reg, poolId, storageBlocks); NUnit.Framework.Assert.Fail("Expected IOException is not thrown"); } catch (IOException) { } // Expected // Ensure blockReport from dead datanode is rejected with IOException StorageBlockReport[] report = new StorageBlockReport[] { new StorageBlockReport(new DatanodeStorage(reg.GetDatanodeUuid()), BlockListAsLongs.Empty) }; try { dnp.BlockReport(reg, poolId, report, new BlockReportContext(1, 0, Runtime.NanoTime ())); NUnit.Framework.Assert.Fail("Expected IOException is not thrown"); } catch (IOException) { } // Expected // Ensure heartbeat from dead datanode is rejected with a command // that asks datanode to register again StorageReport[] rep = new StorageReport[] { new StorageReport(new DatanodeStorage (reg.GetDatanodeUuid()), false, 0, 0, 0, 0) }; DatanodeCommand[] cmd = dnp.SendHeartbeat(reg, rep, 0L, 0L, 0, 0, 0, null).GetCommands (); NUnit.Framework.Assert.AreEqual(1, cmd.Length); NUnit.Framework.Assert.AreEqual(cmd[0].GetAction(), RegisterCommand.Register.GetAction ()); }
/// <exception cref="System.IO.IOException"/> public virtual HeartbeatResponse SendHeartbeat(DatanodeRegistration registration, StorageReport[] reports, long cacheCapacity, long cacheUsed, int xmitsInProgress , int xceiverCount, int failedVolumes, VolumeFailureSummary volumeFailureSummary ) { DatanodeProtocolProtos.HeartbeatRequestProto.Builder builder = DatanodeProtocolProtos.HeartbeatRequestProto .NewBuilder().SetRegistration(PBHelper.Convert(registration)).SetXmitsInProgress (xmitsInProgress).SetXceiverCount(xceiverCount).SetFailedVolumes(failedVolumes); builder.AddAllReports(PBHelper.ConvertStorageReports(reports)); if (cacheCapacity != 0) { builder.SetCacheCapacity(cacheCapacity); } if (cacheUsed != 0) { builder.SetCacheUsed(cacheUsed); } if (volumeFailureSummary != null) { builder.SetVolumeFailureSummary(PBHelper.ConvertVolumeFailureSummary(volumeFailureSummary )); } DatanodeProtocolProtos.HeartbeatResponseProto resp; try { resp = rpcProxy.SendHeartbeat(NullController, ((DatanodeProtocolProtos.HeartbeatRequestProto )builder.Build())); } catch (ServiceException se) { throw ProtobufHelper.GetRemoteException(se); } DatanodeCommand[] cmds = new DatanodeCommand[resp.GetCmdsList().Count]; int index = 0; foreach (DatanodeProtocolProtos.DatanodeCommandProto p in resp.GetCmdsList()) { cmds[index] = PBHelper.Convert(p); index++; } RollingUpgradeStatus rollingUpdateStatus = null; // Use v2 semantics if available. if (resp.HasRollingUpgradeStatusV2()) { rollingUpdateStatus = PBHelper.Convert(resp.GetRollingUpgradeStatusV2()); } else { if (resp.HasRollingUpgradeStatus()) { rollingUpdateStatus = PBHelper.Convert(resp.GetRollingUpgradeStatus()); } } return(new HeartbeatResponse(cmds, PBHelper.Convert(resp.GetHaStatus()), rollingUpdateStatus )); }
/// <exception cref="System.IO.IOException"/> private IList <DataNode.BlockRecord> InitBlockRecords(DataNode spyDN) { IList <DataNode.BlockRecord> blocks = new AList <DataNode.BlockRecord>(1); DatanodeRegistration dnR = dn.GetDNRegistrationForBP(block.GetBlockPoolId()); DataNode.BlockRecord blockRecord = new DataNode.BlockRecord(new DatanodeID(dnR), spyDN, new ReplicaRecoveryInfo(block.GetBlockId(), block.GetNumBytes(), block.GetGenerationStamp (), HdfsServerConstants.ReplicaState.Finalized)); blocks.AddItem(blockRecord); return(blocks); }
public virtual void StartUpCluster() { conf = new Configuration(); cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(NumDatanodes).Build(); fs = cluster.GetFileSystem(); client = new DFSClient(new IPEndPoint("localhost", cluster.GetNameNodePort()), cluster .GetConfiguration(0)); dn0 = cluster.GetDataNodes()[0]; poolId = cluster.GetNamesystem().GetBlockPoolId(); dn0Reg = dn0.GetDNRegistrationForBP(poolId); }
public virtual void TestRegistrationWithDifferentSoftwareVersionsDuringUpgrade() { Configuration conf = new HdfsConfiguration(); conf.Set(DFSConfigKeys.DfsDatanodeMinSupportedNamenodeVersionKey, "1.0.0"); MiniDFSCluster cluster = null; try { cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(0).Build(); NamenodeProtocols rpcServer = cluster.GetNameNodeRpc(); long nnCTime = cluster.GetNamesystem().GetFSImage().GetStorage().GetCTime(); StorageInfo mockStorageInfo = Org.Mockito.Mockito.Mock <StorageInfo>(); Org.Mockito.Mockito.DoReturn(nnCTime).When(mockStorageInfo).GetCTime(); DatanodeRegistration mockDnReg = Org.Mockito.Mockito.Mock <DatanodeRegistration>(); Org.Mockito.Mockito.DoReturn(HdfsConstants.DatanodeLayoutVersion).When(mockDnReg) .GetVersion(); Org.Mockito.Mockito.DoReturn("fake-storage-id").When(mockDnReg).GetDatanodeUuid(); Org.Mockito.Mockito.DoReturn(mockStorageInfo).When(mockDnReg).GetStorageInfo(); // Should succeed when software versions are the same and CTimes are the // same. Org.Mockito.Mockito.DoReturn(VersionInfo.GetVersion()).When(mockDnReg).GetSoftwareVersion (); Org.Mockito.Mockito.DoReturn("127.0.0.1").When(mockDnReg).GetIpAddr(); Org.Mockito.Mockito.DoReturn(123).When(mockDnReg).GetXferPort(); rpcServer.RegisterDatanode(mockDnReg); // Should succeed when software versions are the same and CTimes are // different. Org.Mockito.Mockito.DoReturn(nnCTime + 1).When(mockStorageInfo).GetCTime(); rpcServer.RegisterDatanode(mockDnReg); // Should fail when software version of DN is different from NN and CTimes // are different. Org.Mockito.Mockito.DoReturn(VersionInfo.GetVersion() + ".1").When(mockDnReg).GetSoftwareVersion (); try { rpcServer.RegisterDatanode(mockDnReg); NUnit.Framework.Assert.Fail("Should not have been able to register DN with different software" + " versions and CTimes"); } catch (IncorrectVersionException ive) { GenericTestUtils.AssertExceptionContains("does not match CTime of NN", ive); Log.Info("Got expected exception", ive); } } finally { if (cluster != null) { cluster.Shutdown(); } } }
/// <summary>Test write a file, verifies and closes it.</summary> /// <remarks> /// Test write a file, verifies and closes it. Then the length of the blocks /// are messed up and BlockReport is forced. /// The modification of blocks' length has to be ignored /// </remarks> /// <exception cref="System.IO.IOException">on an error</exception> public virtual void BlockReport_01() { string MethodName = GenericTestUtils.GetMethodName(); Path filePath = new Path("/" + MethodName + ".dat"); AList <Block> blocks = PrepareForRide(filePath, MethodName, FileSize); if (Log.IsDebugEnabled()) { Log.Debug("Number of blocks allocated " + blocks.Count); } long[] oldLengths = new long[blocks.Count]; int tempLen; for (int i = 0; i < blocks.Count; i++) { Block b = blocks[i]; if (Log.IsDebugEnabled()) { Log.Debug("Block " + b.GetBlockName() + " before\t" + "Size " + b.GetNumBytes()); } oldLengths[i] = b.GetNumBytes(); if (Log.IsDebugEnabled()) { Log.Debug("Setting new length"); } tempLen = rand.Next(BlockSize); b.Set(b.GetBlockId(), tempLen, b.GetGenerationStamp()); if (Log.IsDebugEnabled()) { Log.Debug("Block " + b.GetBlockName() + " after\t " + "Size " + b.GetNumBytes()); } } // all blocks belong to the same file, hence same BP DataNode dn = cluster.GetDataNodes()[DnN0]; string poolId = cluster.GetNamesystem().GetBlockPoolId(); DatanodeRegistration dnR = dn.GetDNRegistrationForBP(poolId); StorageBlockReport[] reports = GetBlockReports(dn, poolId, false, false); SendBlockReports(dnR, poolId, reports); IList <LocatedBlock> blocksAfterReport = DFSTestUtil.GetAllBlocks(fs.Open(filePath )); if (Log.IsDebugEnabled()) { Log.Debug("After mods: Number of blocks allocated " + blocksAfterReport.Count); } for (int i_1 = 0; i_1 < blocksAfterReport.Count; i_1++) { ExtendedBlock b = blocksAfterReport[i_1].GetBlock(); NUnit.Framework.Assert.AreEqual("Length of " + i_1 + "th block is incorrect", oldLengths [i_1], b.GetNumBytes()); } }
public virtual void TestRegistrationWithDifferentSoftwareVersions() { Configuration conf = new HdfsConfiguration(); conf.Set(DFSConfigKeys.DfsDatanodeMinSupportedNamenodeVersionKey, "3.0.0"); conf.Set(DFSConfigKeys.DfsNamenodeMinSupportedDatanodeVersionKey, "3.0.0"); MiniDFSCluster cluster = null; try { cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(0).Build(); NamenodeProtocols rpcServer = cluster.GetNameNodeRpc(); long nnCTime = cluster.GetNamesystem().GetFSImage().GetStorage().GetCTime(); StorageInfo mockStorageInfo = Org.Mockito.Mockito.Mock <StorageInfo>(); Org.Mockito.Mockito.DoReturn(nnCTime).When(mockStorageInfo).GetCTime(); DatanodeRegistration mockDnReg = Org.Mockito.Mockito.Mock <DatanodeRegistration>(); Org.Mockito.Mockito.DoReturn(HdfsConstants.DatanodeLayoutVersion).When(mockDnReg) .GetVersion(); Org.Mockito.Mockito.DoReturn("127.0.0.1").When(mockDnReg).GetIpAddr(); Org.Mockito.Mockito.DoReturn(123).When(mockDnReg).GetXferPort(); Org.Mockito.Mockito.DoReturn("fake-storage-id").When(mockDnReg).GetDatanodeUuid(); Org.Mockito.Mockito.DoReturn(mockStorageInfo).When(mockDnReg).GetStorageInfo(); // Should succeed when software versions are the same. Org.Mockito.Mockito.DoReturn("3.0.0").When(mockDnReg).GetSoftwareVersion(); rpcServer.RegisterDatanode(mockDnReg); // Should succeed when software version of DN is above minimum required by NN. Org.Mockito.Mockito.DoReturn("4.0.0").When(mockDnReg).GetSoftwareVersion(); rpcServer.RegisterDatanode(mockDnReg); // Should fail when software version of DN is below minimum required by NN. Org.Mockito.Mockito.DoReturn("2.0.0").When(mockDnReg).GetSoftwareVersion(); try { rpcServer.RegisterDatanode(mockDnReg); NUnit.Framework.Assert.Fail("Should not have been able to register DN with too-low version." ); } catch (IncorrectVersionException ive) { GenericTestUtils.AssertExceptionContains("The reported DataNode version is too low" , ive); Log.Info("Got expected exception", ive); } } finally { if (cluster != null) { cluster.Shutdown(); } } }
/// <exception cref="System.Exception"/> public virtual void TestDatanodeReRegistration() { // Create a test file DistributedFileSystem dfs = cluster.GetFileSystem(); Path path = new Path("/testRR"); // Create a file and shutdown the DNs, which populates InvalidateBlocks DFSTestUtil.CreateFile(dfs, path, dfs.GetDefaultBlockSize(), (short)NumOfDatanodes , unchecked ((int)(0xED0ED0))); foreach (DataNode dn in cluster.GetDataNodes()) { dn.Shutdown(); } dfs.Delete(path, false); namesystem.WriteLock(); InvalidateBlocks invalidateBlocks; int expected = NumOfDatanodes; try { invalidateBlocks = (InvalidateBlocks)Whitebox.GetInternalState(cluster.GetNamesystem ().GetBlockManager(), "invalidateBlocks"); NUnit.Framework.Assert.AreEqual("Expected invalidate blocks to be the number of DNs" , (long)expected, invalidateBlocks.NumBlocks()); } finally { namesystem.WriteUnlock(); } // Re-register each DN and see that it wipes the invalidation work foreach (DataNode dn_1 in cluster.GetDataNodes()) { DatanodeID did = dn_1.GetDatanodeId(); DatanodeRegistration reg = new DatanodeRegistration(new DatanodeID(UUID.RandomUUID ().ToString(), did), new StorageInfo(HdfsServerConstants.NodeType.DataNode), new ExportedBlockKeys(), VersionInfo.GetVersion()); namesystem.WriteLock(); try { bm.GetDatanodeManager().RegisterDatanode(reg); expected--; NUnit.Framework.Assert.AreEqual("Expected number of invalidate blocks to decrease" , (long)expected, invalidateBlocks.NumBlocks()); } finally { namesystem.WriteUnlock(); } } }
public virtual void TestChangeStorageID() { string DnIpAddr = "127.0.0.1"; string DnHostname = "localhost"; int DnXferPort = 12345; int DnInfoPort = 12346; int DnInfoSecurePort = 12347; int DnIpcPort = 12348; Configuration conf = new HdfsConfiguration(); MiniDFSCluster cluster = null; try { cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(0).Build(); IPEndPoint addr = new IPEndPoint("localhost", cluster.GetNameNodePort()); DFSClient client = new DFSClient(addr, conf); NamenodeProtocols rpcServer = cluster.GetNameNodeRpc(); // register a datanode DatanodeID dnId = new DatanodeID(DnIpAddr, DnHostname, "fake-datanode-id", DnXferPort , DnInfoPort, DnInfoSecurePort, DnIpcPort); long nnCTime = cluster.GetNamesystem().GetFSImage().GetStorage().GetCTime(); StorageInfo mockStorageInfo = Org.Mockito.Mockito.Mock <StorageInfo>(); Org.Mockito.Mockito.DoReturn(nnCTime).When(mockStorageInfo).GetCTime(); Org.Mockito.Mockito.DoReturn(HdfsConstants.DatanodeLayoutVersion).When(mockStorageInfo ).GetLayoutVersion(); DatanodeRegistration dnReg = new DatanodeRegistration(dnId, mockStorageInfo, null , VersionInfo.GetVersion()); rpcServer.RegisterDatanode(dnReg); DatanodeInfo[] report = client.DatanodeReport(HdfsConstants.DatanodeReportType.All ); NUnit.Framework.Assert.AreEqual("Expected a registered datanode", 1, report.Length ); // register the same datanode again with a different storage ID dnId = new DatanodeID(DnIpAddr, DnHostname, "changed-fake-datanode-id", DnXferPort , DnInfoPort, DnInfoSecurePort, DnIpcPort); dnReg = new DatanodeRegistration(dnId, mockStorageInfo, null, VersionInfo.GetVersion ()); rpcServer.RegisterDatanode(dnReg); report = client.DatanodeReport(HdfsConstants.DatanodeReportType.All); NUnit.Framework.Assert.AreEqual("Datanode with changed storage ID not recognized" , 1, report.Length); } finally { if (cluster != null) { cluster.Shutdown(); } } }
/// <exception cref="System.IO.IOException"/> protected internal override void SendBlockReports(DatanodeRegistration dnR, string poolId, StorageBlockReport[] reports) { int i = 0; foreach (StorageBlockReport report in reports) { Log.Info("Sending block report for storage " + report.GetStorage().GetStorageID() ); StorageBlockReport[] singletonReport = new StorageBlockReport[] { report }; cluster.GetNameNodeRpc().BlockReport(dnR, poolId, singletonReport, new BlockReportContext (reports.Length, i, Runtime.NanoTime())); i++; } }
/// <exception cref="System.IO.IOException"/> public virtual void ErrorReport(DatanodeRegistration registration, int errorCode, string msg) { DatanodeProtocolProtos.ErrorReportRequestProto req = ((DatanodeProtocolProtos.ErrorReportRequestProto )DatanodeProtocolProtos.ErrorReportRequestProto.NewBuilder().SetRegistartion(PBHelper .Convert(registration)).SetErrorCode(errorCode).SetMsg(msg).Build()); try { rpcProxy.ErrorReport(NullController, req); } catch (ServiceException se) { throw ProtobufHelper.GetRemoteException(se); } }
/// <exception cref="System.IO.IOException"/> private static void RunTest(string testCaseName, bool createFiles, int numInitialStorages , int expectedStoragesAfterTest) { Configuration conf = new HdfsConfiguration(); MiniDFSCluster cluster = null; try { cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(1).StoragesPerDatanode(numInitialStorages ).Build(); cluster.WaitActive(); DataNode dn0 = cluster.GetDataNodes()[0]; // Ensure NN knows about the storage. DatanodeID dnId = dn0.GetDatanodeId(); DatanodeDescriptor dnDescriptor = cluster.GetNamesystem().GetBlockManager().GetDatanodeManager ().GetDatanode(dnId); Assert.AssertThat(dnDescriptor.GetStorageInfos().Length, IS.Is(numInitialStorages )); string bpid = cluster.GetNamesystem().GetBlockPoolId(); DatanodeRegistration dnReg = dn0.GetDNRegistrationForBP(bpid); DataNodeTestUtils.TriggerBlockReport(dn0); if (createFiles) { Path path = new Path("/", testCaseName); DFSTestUtil.CreateFile(cluster.GetFileSystem(), path, 1024, (short)1, unchecked (( int)(0x1BAD5EED))); DataNodeTestUtils.TriggerBlockReport(dn0); } // Generate a fake StorageReport that is missing one storage. StorageReport[] reports = dn0.GetFSDataset().GetStorageReports(bpid); StorageReport[] prunedReports = new StorageReport[numInitialStorages - 1]; System.Array.Copy(reports, 0, prunedReports, 0, prunedReports.Length); // Stop the DataNode and send fake heartbeat with missing storage. cluster.StopDataNode(0); cluster.GetNameNodeRpc().SendHeartbeat(dnReg, prunedReports, 0L, 0L, 0, 0, 0, null ); // Check that the missing storage was pruned. Assert.AssertThat(dnDescriptor.GetStorageInfos().Length, IS.Is(expectedStoragesAfterTest )); } finally { if (cluster != null) { cluster.Shutdown(); } } }
public virtual void TestBlockHasMultipleReplicasOnSameDN() { string filename = MakeFileName(GenericTestUtils.GetMethodName()); Path filePath = new Path(filename); // Write out a file with a few blocks. DFSTestUtil.CreateFile(fs, filePath, BlockSize, BlockSize * NumBlocks, BlockSize, NumDatanodes, seed); // Get the block list for the file with the block locations. LocatedBlocks locatedBlocks = client.GetLocatedBlocks(filePath.ToString(), 0, BlockSize * NumBlocks); // Generate a fake block report from one of the DataNodes, such // that it reports one copy of each block on either storage. DataNode dn = cluster.GetDataNodes()[0]; DatanodeRegistration dnReg = dn.GetDNRegistrationForBP(bpid); StorageBlockReport[] reports = new StorageBlockReport[cluster.GetStoragesPerDatanode ()]; AList <Replica> blocks = new AList <Replica>(); foreach (LocatedBlock locatedBlock in locatedBlocks.GetLocatedBlocks()) { Block localBlock = locatedBlock.GetBlock().GetLocalBlock(); blocks.AddItem(new FinalizedReplica(localBlock, null, null)); } BlockListAsLongs bll = BlockListAsLongs.Encode(blocks); for (int i = 0; i < cluster.GetStoragesPerDatanode(); ++i) { FsVolumeSpi v = dn.GetFSDataset().GetVolumes()[i]; DatanodeStorage dns = new DatanodeStorage(v.GetStorageID()); reports[i] = new StorageBlockReport(dns, bll); } // Should not assert! cluster.GetNameNodeRpc().BlockReport(dnReg, bpid, reports, new BlockReportContext (1, 0, Runtime.NanoTime())); // Get the block locations once again. locatedBlocks = client.GetLocatedBlocks(filename, 0, BlockSize * NumBlocks); // Make sure that each block has two replicas, one on each DataNode. foreach (LocatedBlock locatedBlock_1 in locatedBlocks.GetLocatedBlocks()) { DatanodeInfo[] locations = locatedBlock_1.GetLocations(); Assert.AssertThat(locations.Length, IS.Is((int)NumDatanodes)); Assert.AssertThat(locations[0].GetDatanodeUuid(), CoreMatchers.Not(locations[1].GetDatanodeUuid ())); } }
/// <exception cref="System.IO.IOException"/> public virtual DatanodeRegistration RegisterDatanode(DatanodeRegistration registration ) { DatanodeProtocolProtos.RegisterDatanodeRequestProto.Builder builder = DatanodeProtocolProtos.RegisterDatanodeRequestProto .NewBuilder().SetRegistration(PBHelper.Convert(registration)); DatanodeProtocolProtos.RegisterDatanodeResponseProto resp; try { resp = rpcProxy.RegisterDatanode(NullController, ((DatanodeProtocolProtos.RegisterDatanodeRequestProto )builder.Build())); } catch (ServiceException se) { throw ProtobufHelper.GetRemoteException(se); } return(PBHelper.Convert(resp.GetRegistration())); }
public virtual void TestStorageWithRemainingCapacity() { Configuration conf = new HdfsConfiguration(); MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).Build(); FileSystem fs = FileSystem.Get(conf); Path file1 = null; try { cluster.WaitActive(); FSNamesystem namesystem = cluster.GetNamesystem(); string poolId = namesystem.GetBlockPoolId(); DatanodeRegistration nodeReg = DataNodeTestUtils.GetDNRegistrationForBP(cluster.GetDataNodes ()[0], poolId); DatanodeDescriptor dd = NameNodeAdapter.GetDatanode(namesystem, nodeReg); // By default, MiniDFSCluster will create 1 datanode with 2 storages. // Assigning 64k for remaining storage capacity and will //create a file with 100k. foreach (DatanodeStorageInfo storage in dd.GetStorageInfos()) { storage.SetUtilizationForTesting(65536, 0, 65536, 0); } //sum of the remaining capacity of both the storages dd.SetRemaining(131072); file1 = new Path("testRemainingStorage.dat"); try { DFSTestUtil.CreateFile(fs, file1, 102400, 102400, 102400, (short)1, unchecked ((int )(0x1BAD5EED))); } catch (RemoteException re) { GenericTestUtils.AssertExceptionContains("nodes instead of " + "minReplication", re); } } finally { // Clean up NUnit.Framework.Assert.IsTrue(fs.Exists(file1)); fs.Delete(file1, true); NUnit.Framework.Assert.IsTrue(!fs.Exists(file1)); cluster.Shutdown(); } }
/// <exception cref="Com.Google.Protobuf.ServiceException"/> public virtual DatanodeProtocolProtos.RegisterDatanodeResponseProto RegisterDatanode (RpcController controller, DatanodeProtocolProtos.RegisterDatanodeRequestProto request ) { DatanodeRegistration registration = PBHelper.Convert(request.GetRegistration()); DatanodeRegistration registrationResp; try { registrationResp = impl.RegisterDatanode(registration); } catch (IOException e) { throw new ServiceException(e); } return((DatanodeProtocolProtos.RegisterDatanodeResponseProto)DatanodeProtocolProtos.RegisterDatanodeResponseProto .NewBuilder().SetRegistration(PBHelper.Convert(registrationResp)).Build()); }
public virtual void TestDatanodeDetect() { AtomicReference <DatanodeProtocolProtos.BlockReportRequestProto> request = new AtomicReference <DatanodeProtocolProtos.BlockReportRequestProto>(); // just capture the outgoing PB DatanodeProtocolPB mockProxy = Org.Mockito.Mockito.Mock <DatanodeProtocolPB>(); Org.Mockito.Mockito.DoAnswer(new _Answer_205(request)).When(mockProxy).BlockReport (Matchers.Any <RpcController>(), Matchers.Any <DatanodeProtocolProtos.BlockReportRequestProto >()); DatanodeProtocolClientSideTranslatorPB nn = new DatanodeProtocolClientSideTranslatorPB (mockProxy); DatanodeRegistration reg = DFSTestUtil.GetLocalDatanodeRegistration(); NamespaceInfo nsInfo = new NamespaceInfo(1, "cluster", "bp", 1); reg.SetNamespaceInfo(nsInfo); Replica r = new FinalizedReplica(new Block(1, 2, 3), null, null); BlockListAsLongs bbl = BlockListAsLongs.Encode(Sharpen.Collections.Singleton(r)); DatanodeStorage storage = new DatanodeStorage("s1"); StorageBlockReport[] sbr = new StorageBlockReport[] { new StorageBlockReport(storage , bbl) }; // check DN sends new-style BR request.Set(null); nsInfo.SetCapabilities(NamespaceInfo.Capability.StorageBlockReportBuffers.GetMask ()); nn.BlockReport(reg, "pool", sbr, new BlockReportContext(1, 0, Runtime.NanoTime()) ); DatanodeProtocolProtos.BlockReportRequestProto proto = request.Get(); NUnit.Framework.Assert.IsNotNull(proto); NUnit.Framework.Assert.IsTrue(proto.GetReports(0).GetBlocksList().IsEmpty()); NUnit.Framework.Assert.IsFalse(proto.GetReports(0).GetBlocksBuffersList().IsEmpty ()); // back up to prior version and check DN sends old-style BR request.Set(null); nsInfo.SetCapabilities(NamespaceInfo.Capability.Unknown.GetMask()); nn.BlockReport(reg, "pool", sbr, new BlockReportContext(1, 0, Runtime.NanoTime()) ); proto = request.Get(); NUnit.Framework.Assert.IsNotNull(proto); NUnit.Framework.Assert.IsFalse(proto.GetReports(0).GetBlocksList().IsEmpty()); NUnit.Framework.Assert.IsTrue(proto.GetReports(0).GetBlocksBuffersList().IsEmpty( )); }
public virtual void TestConvertDatanodeRegistration() { DatanodeID dnId = DFSTestUtil.GetLocalDatanodeID(); BlockKey[] keys = new BlockKey[] { GetBlockKey(2), GetBlockKey(3) }; ExportedBlockKeys expKeys = new ExportedBlockKeys(true, 9, 10, GetBlockKey(1), keys ); DatanodeRegistration reg = new DatanodeRegistration(dnId, new StorageInfo(HdfsServerConstants.NodeType .DataNode), expKeys, "3.0.0"); DatanodeProtocolProtos.DatanodeRegistrationProto proto = PBHelper.Convert(reg); DatanodeRegistration reg2 = PBHelper.Convert(proto); Compare(reg.GetStorageInfo(), reg2.GetStorageInfo()); Compare(reg.GetExportedKeys(), reg2.GetExportedKeys()); Compare(reg, reg2); NUnit.Framework.Assert.AreEqual(reg.GetSoftwareVersion(), reg2.GetSoftwareVersion ()); }
/// <summary>Test creates a file and closes it.</summary> /// <remarks> /// Test creates a file and closes it. /// The second datanode is started in the cluster. /// As soon as the replication process is completed test runs /// Block report and checks that no underreplicated blocks are left /// </remarks> /// <exception cref="System.IO.IOException">in case of an error</exception> /// <exception cref="System.Exception"/> public virtual void BlockReport_06() { string MethodName = GenericTestUtils.GetMethodName(); Path filePath = new Path("/" + MethodName + ".dat"); int DnN1 = DnN0 + 1; WriteFile(MethodName, FileSize, filePath); StartDNandWait(filePath, true); // all blocks belong to the same file, hence same BP DataNode dn = cluster.GetDataNodes()[DnN1]; string poolId = cluster.GetNamesystem().GetBlockPoolId(); DatanodeRegistration dnR = dn.GetDNRegistrationForBP(poolId); StorageBlockReport[] reports = GetBlockReports(dn, poolId, false, false); SendBlockReports(dnR, poolId, reports); PrintStats(); NUnit.Framework.Assert.AreEqual("Wrong number of PendingReplication Blocks", 0, cluster .GetNamesystem().GetUnderReplicatedBlocks()); }
/// <exception cref="System.IO.IOException"/> public virtual DatanodeCommand BlockReport(DatanodeRegistration registration, string poolId, StorageBlockReport[] reports, BlockReportContext context) { DatanodeProtocolProtos.BlockReportRequestProto.Builder builder = DatanodeProtocolProtos.BlockReportRequestProto .NewBuilder().SetRegistration(PBHelper.Convert(registration)).SetBlockPoolId(poolId ); bool useBlocksBuffer = registration.GetNamespaceInfo().IsCapabilitySupported(NamespaceInfo.Capability .StorageBlockReportBuffers); foreach (StorageBlockReport r in reports) { DatanodeProtocolProtos.StorageBlockReportProto.Builder reportBuilder = DatanodeProtocolProtos.StorageBlockReportProto .NewBuilder().SetStorage(PBHelper.Convert(r.GetStorage())); BlockListAsLongs blocks = r.GetBlocks(); if (useBlocksBuffer) { reportBuilder.SetNumberOfBlocks(blocks.GetNumberOfBlocks()); reportBuilder.AddAllBlocksBuffers(blocks.GetBlocksBuffers()); } else { foreach (long value in blocks.GetBlockListAsLongs()) { reportBuilder.AddBlocks(value); } } builder.AddReports(((DatanodeProtocolProtos.StorageBlockReportProto)reportBuilder .Build())); } builder.SetContext(PBHelper.Convert(context)); DatanodeProtocolProtos.BlockReportResponseProto resp; try { resp = rpcProxy.BlockReport(NullController, ((DatanodeProtocolProtos.BlockReportRequestProto )builder.Build())); } catch (ServiceException se) { throw ProtobufHelper.GetRemoteException(se); } return(resp.HasCmd() ? PBHelper.Convert(resp.GetCmd()) : null); }
// return the initial state of the configuration // Similar to BlockReport_08 but corrupts GS and len of the TEMPORARY's // replica block. Expect the same behaviour: NN should simply ignore this // block /// <exception cref="System.IO.IOException"/> public virtual void BlockReport_09() { string MethodName = GenericTestUtils.GetMethodName(); Path filePath = new Path("/" + MethodName + ".dat"); int DnN1 = DnN0 + 1; int bytesChkSum = 1024 * 1000; conf.SetInt(DFSConfigKeys.DfsBytesPerChecksumKey, bytesChkSum); conf.SetLong(DFSConfigKeys.DfsBlockSizeKey, 6 * bytesChkSum); ShutDownCluster(); StartUpCluster(); // write file and start second node to be "older" than the original try { WriteFile(MethodName, 12 * bytesChkSum, filePath); Block bl = FindBlock(filePath, 12 * bytesChkSum); BlockReportTestBase.BlockChecker bc = new BlockReportTestBase.BlockChecker(this, filePath); bc.Start(); WaitForTempReplica(bl, DnN1); // all blocks belong to the same file, hence same BP DataNode dn = cluster.GetDataNodes()[DnN1]; string poolId = cluster.GetNamesystem().GetBlockPoolId(); DatanodeRegistration dnR = dn.GetDNRegistrationForBP(poolId); StorageBlockReport[] reports = GetBlockReports(dn, poolId, true, true); SendBlockReports(dnR, poolId, reports); PrintStats(); NUnit.Framework.Assert.AreEqual("Wrong number of PendingReplication blocks", 2, cluster .GetNamesystem().GetPendingReplicationBlocks()); try { bc.Join(); } catch (Exception) { } } finally { ResetConfiguration(); } }
public virtual void TestSafeModeIBR() { DatanodeDescriptor node = Org.Mockito.Mockito.Spy(nodes[0]); DatanodeStorageInfo ds = node.GetStorageInfos()[0]; node.isAlive = true; DatanodeRegistration nodeReg = new DatanodeRegistration(node, null, null, string.Empty ); // pretend to be in safemode Org.Mockito.Mockito.DoReturn(true).When(fsn).IsInStartupSafeMode(); // register new node bm.GetDatanodeManager().RegisterDatanode(nodeReg); bm.GetDatanodeManager().AddDatanode(node); // swap in spy NUnit.Framework.Assert.AreEqual(node, bm.GetDatanodeManager().GetDatanode(node)); NUnit.Framework.Assert.AreEqual(0, ds.GetBlockReportCount()); // send block report, should be processed Org.Mockito.Mockito.Reset(node); bm.ProcessReport(node, new DatanodeStorage(ds.GetStorageID()), BlockListAsLongs.Empty , null, false); NUnit.Framework.Assert.AreEqual(1, ds.GetBlockReportCount()); // send block report again, should NOT be processed Org.Mockito.Mockito.Reset(node); bm.ProcessReport(node, new DatanodeStorage(ds.GetStorageID()), BlockListAsLongs.Empty , null, false); NUnit.Framework.Assert.AreEqual(1, ds.GetBlockReportCount()); // re-register as if node restarted, should update existing node bm.GetDatanodeManager().RemoveDatanode(node); Org.Mockito.Mockito.Reset(node); bm.GetDatanodeManager().RegisterDatanode(nodeReg); Org.Mockito.Mockito.Verify(node).UpdateRegInfo(nodeReg); // send block report, should be processed after restart Org.Mockito.Mockito.Reset(node); bm.ProcessReport(node, new DatanodeStorage(ds.GetStorageID()), BlockListAsLongs.Empty , null, false); // Reinitialize as registration with empty storage list pruned // node.storageMap. ds = node.GetStorageInfos()[0]; NUnit.Framework.Assert.AreEqual(1, ds.GetBlockReportCount()); }
/// <exception cref="System.IO.IOException"/> private void TestDataNodeRedirect(Path path) { // Create the file if (hdfs.Exists(path)) { hdfs.Delete(path, true); } FSDataOutputStream @out = hdfs.Create(path, (short)1); @out.WriteBytes("0123456789"); @out.Close(); // Get the path's block location so we can determine // if we were redirected to the right DN. BlockLocation[] locations = hdfs.GetFileBlockLocations(path, 0, 10); string xferAddr = locations[0].GetNames()[0]; // Connect to the NN to get redirected Uri u = hftpFs.GetNamenodeURL("/data" + ServletUtil.EncodePath(path.ToUri().GetPath ()), "ugi=userx,groupy"); HttpURLConnection conn = (HttpURLConnection)u.OpenConnection(); HttpURLConnection.SetFollowRedirects(true); conn.Connect(); conn.GetInputStream(); bool @checked = false; // Find the datanode that has the block according to locations // and check that the URL was redirected to this DN's info port foreach (DataNode node in cluster.GetDataNodes()) { DatanodeRegistration dnR = DataNodeTestUtils.GetDNRegistrationForBP(node, blockPoolId ); if (dnR.GetXferAddr().Equals(xferAddr)) { @checked = true; NUnit.Framework.Assert.AreEqual(dnR.GetInfoPort(), conn.GetURL().Port); } } NUnit.Framework.Assert.IsTrue("The test never checked that location of " + "the block and hftp desitnation are the same" , @checked); }