public virtual void TestBlocksCounter() { DatanodeDescriptor dd = BlockManagerTestUtil.GetLocalDatanodeDescriptor(true); NUnit.Framework.Assert.AreEqual(0, dd.NumBlocks()); BlockInfoContiguous blk = new BlockInfoContiguous(new Block(1L), (short)1); BlockInfoContiguous blk1 = new BlockInfoContiguous(new Block(2L), (short)2); DatanodeStorageInfo[] storages = dd.GetStorageInfos(); NUnit.Framework.Assert.IsTrue(storages.Length > 0); // add first block NUnit.Framework.Assert.IsTrue(storages[0].AddBlock(blk) == DatanodeStorageInfo.AddBlockResult .Added); NUnit.Framework.Assert.AreEqual(1, dd.NumBlocks()); // remove a non-existent block NUnit.Framework.Assert.IsFalse(dd.RemoveBlock(blk1)); NUnit.Framework.Assert.AreEqual(1, dd.NumBlocks()); // add an existent block NUnit.Framework.Assert.IsFalse(storages[0].AddBlock(blk) == DatanodeStorageInfo.AddBlockResult .Added); NUnit.Framework.Assert.AreEqual(1, dd.NumBlocks()); // add second block NUnit.Framework.Assert.IsTrue(storages[0].AddBlock(blk1) == DatanodeStorageInfo.AddBlockResult .Added); NUnit.Framework.Assert.AreEqual(2, dd.NumBlocks()); // remove first block NUnit.Framework.Assert.IsTrue(dd.RemoveBlock(blk)); NUnit.Framework.Assert.AreEqual(1, dd.NumBlocks()); // remove second block NUnit.Framework.Assert.IsTrue(dd.RemoveBlock(blk1)); NUnit.Framework.Assert.AreEqual(0, dd.NumBlocks()); }
public virtual void TestQueues() { DatanodeDescriptor fakeDN = DFSTestUtil.GetLocalDatanodeDescriptor(); DatanodeStorage storage = new DatanodeStorage("STORAGE_ID"); DatanodeStorageInfo storageInfo = new DatanodeStorageInfo(fakeDN, storage); msgs.EnqueueReportedBlock(storageInfo, block1Gs1, HdfsServerConstants.ReplicaState .Finalized); msgs.EnqueueReportedBlock(storageInfo, block1Gs2, HdfsServerConstants.ReplicaState .Finalized); NUnit.Framework.Assert.AreEqual(2, msgs.Count()); // Nothing queued yet for block 2 NUnit.Framework.Assert.IsNull(msgs.TakeBlockQueue(block2Gs1)); NUnit.Framework.Assert.AreEqual(2, msgs.Count()); Queue <PendingDataNodeMessages.ReportedBlockInfo> q = msgs.TakeBlockQueue(block1Gs2DifferentInstance ); NUnit.Framework.Assert.AreEqual("ReportedBlockInfo [block=blk_1_1, dn=127.0.0.1:50010, reportedState=FINALIZED]," + "ReportedBlockInfo [block=blk_1_2, dn=127.0.0.1:50010, reportedState=FINALIZED]" , Joiner.On(",").Join(q)); NUnit.Framework.Assert.AreEqual(0, msgs.Count()); // Should be null if we pull again NUnit.Framework.Assert.IsNull(msgs.TakeBlockQueue(block1Gs1)); NUnit.Framework.Assert.AreEqual(0, msgs.Count()); }
/// <summary>Check if node is already in the map.</summary> internal virtual bool Contains(DatanodeDescriptor node) { if (node == null) { return(false); } string ipAddr = node.GetIpAddr(); hostmapLock.ReadLock().Lock(); try { DatanodeDescriptor[] nodes = map[ipAddr]; if (nodes != null) { foreach (DatanodeDescriptor containedNode in nodes) { if (node == containedNode) { return(true); } } } } finally { hostmapLock.ReadLock().Unlock(); } return(false); }
/// <summary> /// Ensure that the given NameNode marks the specified DataNode as /// entirely dead/expired. /// </summary> /// <param name="nn">the NameNode to manipulate</param> /// <param name="dnName">the name of the DataNode</param> public static void NoticeDeadDatanode(NameNode nn, string dnName) { FSNamesystem namesystem = nn.GetNamesystem(); namesystem.WriteLock(); try { DatanodeManager dnm = namesystem.GetBlockManager().GetDatanodeManager(); HeartbeatManager hbm = dnm.GetHeartbeatManager(); DatanodeDescriptor[] dnds = hbm.GetDatanodes(); DatanodeDescriptor theDND = null; foreach (DatanodeDescriptor dnd in dnds) { if (dnd.GetXferAddr().Equals(dnName)) { theDND = dnd; } } NUnit.Framework.Assert.IsNotNull("Could not find DN with name: " + dnName, theDND ); lock (hbm) { DFSTestUtil.SetDatanodeDead(theDND); hbm.HeartbeatCheck(); } } finally { namesystem.WriteUnlock(); } }
/// <summary> /// Choose a single datanode from the provided list of possible /// targets, weighted by the percentage of free space remaining on the node. /// </summary> /// <returns>The chosen datanode</returns> private static DatanodeDescriptor ChooseRandomDatanodeByRemainingCapacity(IList <DatanodeDescriptor > targets) { // Use a weighted probability to choose the target datanode float total = 0; foreach (DatanodeDescriptor d in targets) { total += d.GetCacheRemainingPercent(); } // Give each datanode a portion of keyspace equal to its relative weight // [0, w1) selects d1, [w1, w2) selects d2, etc. SortedDictionary <int, DatanodeDescriptor> lottery = new SortedDictionary <int, DatanodeDescriptor >(); int offset = 0; foreach (DatanodeDescriptor d_1 in targets) { // Since we're using floats, be paranoid about negative values int weight = Math.Max(1, (int)((d_1.GetCacheRemainingPercent() / total) * 1000000 )); offset += weight; lottery[offset] = d_1; } // Choose a number from [0, offset), which is the total amount of weight, // to select the winner DatanodeDescriptor winner = lottery.HigherEntry(random.Next(offset)).Value; return(winner); }
public virtual void TestChooseReplicaToDelete() { MiniDFSCluster cluster = null; FileSystem fs = null; try { Configuration conf = new HdfsConfiguration(); conf.SetLong(DFSConfigKeys.DfsBlockSizeKey, SmallBlockSize); cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(3).Build(); fs = cluster.GetFileSystem(); FSNamesystem namesystem = cluster.GetNamesystem(); conf.SetLong(DFSConfigKeys.DfsHeartbeatIntervalKey, 300); cluster.StartDataNodes(conf, 1, true, null, null, null); DataNode lastDN = cluster.GetDataNodes()[3]; DatanodeRegistration dnReg = DataNodeTestUtils.GetDNRegistrationForBP(lastDN, namesystem .GetBlockPoolId()); string lastDNid = dnReg.GetDatanodeUuid(); Path fileName = new Path("/foo2"); DFSTestUtil.CreateFile(fs, fileName, SmallFileLength, (short)4, 0L); DFSTestUtil.WaitReplication(fs, fileName, (short)4); // Wait for tolerable number of heartbeats plus one DatanodeDescriptor nodeInfo = null; long lastHeartbeat = 0; long waitTime = DFSConfigKeys.DfsHeartbeatIntervalDefault * 1000 * (DFSConfigKeys .DfsNamenodeTolerateHeartbeatMultiplierDefault + 1); do { nodeInfo = namesystem.GetBlockManager().GetDatanodeManager().GetDatanode(dnReg); lastHeartbeat = nodeInfo.GetLastUpdateMonotonic(); }while (Time.MonotonicNow() - lastHeartbeat < waitTime); fs.SetReplication(fileName, (short)3); BlockLocation[] locs = fs.GetFileBlockLocations(fs.GetFileStatus(fileName), 0, long.MaxValue ); // All replicas for deletion should be scheduled on lastDN. // And should not actually be deleted, because lastDN does not heartbeat. namesystem.ReadLock(); ICollection <Block> dnBlocks = namesystem.GetBlockManager().excessReplicateMap[lastDNid ]; NUnit.Framework.Assert.AreEqual("Replicas on node " + lastDNid + " should have been deleted" , SmallFileLength / SmallBlockSize, dnBlocks.Count); namesystem.ReadUnlock(); foreach (BlockLocation location in locs) { NUnit.Framework.Assert.AreEqual("Block should still have 4 replicas", 4, location .GetNames().Length); } } finally { if (fs != null) { fs.Close(); } if (cluster != null) { cluster.Shutdown(); } } }
public virtual void TestChooseMoreTargetsThanNodeGroups() { for (int i = 0; i < NumOfDatanodes; i++) { cluster.Remove(dataNodes[i]); } for (int i_1 = 0; i_1 < NumOfDatanodesBoundary; i_1++) { DatanodeDescriptor node = dataNodesInBoundaryCase[i_1]; if (cluster.Contains(node)) { cluster.Remove(node); } } for (int i_2 = 0; i_2 < NumOfDatanodesMoreTargets; i_2++) { cluster.Add(dataNodesInMoreTargetsCase[i_2]); } for (int i_3 = 0; i_3 < NumOfDatanodesMoreTargets; i_3++) { UpdateHeartbeatWithUsage(dataNodesInMoreTargetsCase[i_3], 2 * HdfsConstants.MinBlocksForWrite * BlockSize, 0L, 2 * HdfsConstants.MinBlocksForWrite * BlockSize, 0L, 0L, 0L, 0 , 0); } DatanodeStorageInfo[] targets; // Test normal case -- 3 replicas targets = ChooseTarget(3, dataNodesInMoreTargetsCase[0]); NUnit.Framework.Assert.AreEqual(targets.Length, 3); NUnit.Framework.Assert.IsTrue(CheckTargetsOnDifferentNodeGroup(targets)); // Test special case -- replica number over node groups. targets = ChooseTarget(10, dataNodesInMoreTargetsCase[0]); NUnit.Framework.Assert.IsTrue(CheckTargetsOnDifferentNodeGroup(targets)); // Verify it only can find 6 targets for placing replicas. NUnit.Framework.Assert.AreEqual(targets.Length, 6); }
public virtual void TestMultipleLists() { DatanodeDescriptor[] datanodes = new DatanodeDescriptor[] { new DatanodeDescriptor (new DatanodeID("127.0.0.1", "localhost", "abcd", 5000, 5001, 5002, 5003)), new DatanodeDescriptor(new DatanodeID("127.0.1.1", "localhost", "efgh", 6000, 6001, 6002, 6003)) }; DatanodeDescriptor.CachedBlocksList[] lists = new DatanodeDescriptor.CachedBlocksList [] { datanodes[0].GetPendingCached(), datanodes[0].GetCached(), datanodes[1].GetPendingCached (), datanodes[1].GetCached(), datanodes[1].GetPendingUncached() }; int NumBlocks = 8000; CachedBlock[] blocks = new CachedBlock[NumBlocks]; for (int i = 0; i < NumBlocks; i++) { blocks[i] = new CachedBlock(i, (short)i, true); } Random r = new Random(654); foreach (DatanodeDescriptor.CachedBlocksList list in lists) { TestAddElementsToList(list, blocks); } foreach (DatanodeDescriptor.CachedBlocksList list_1 in lists) { TestRemoveElementsFromList(r, list_1, blocks); } }
internal virtual bool RemoveFromCorruptReplicasMap(Block blk, DatanodeDescriptor datanode, CorruptReplicasMap.Reason reason) { IDictionary <DatanodeDescriptor, CorruptReplicasMap.Reason> datanodes = corruptReplicasMap [blk]; if (datanodes == null) { return(false); } // if reasons can be compared but don't match, return false. CorruptReplicasMap.Reason storedReason = datanodes[datanode]; if (reason != CorruptReplicasMap.Reason.Any && storedReason != null && reason != storedReason) { return(false); } if (Sharpen.Collections.Remove(datanodes, datanode) != null) { // remove the replicas if (datanodes.IsEmpty()) { // remove the block if there is no more corrupted replicas Sharpen.Collections.Remove(corruptReplicasMap, blk); } return(true); } return(false); }
internal virtual IList <Block> InvalidateWork(DatanodeDescriptor dn) { lock (this) { long delay = GetInvalidationDelay(); if (delay > 0) { if (BlockManager.Log.IsDebugEnabled()) { BlockManager.Log.Debug("Block deletion is delayed during NameNode startup. " + "The deletion will start after " + delay + " ms."); } return(null); } LightWeightHashSet <Block> set = node2blocks[dn]; if (set == null) { return(null); } // # blocks that can be sent in one message is limited int limit = blockInvalidateLimit; IList <Block> toInvalidate = set.PollN(limit); // If we send everything in this message, remove this node entry if (set.IsEmpty()) { Remove(dn); } dn.AddBlocksToBeInvalidated(toInvalidate); numBlocks -= toInvalidate.Count; return(toInvalidate); } }
public virtual void TestRemove() { DatanodeDescriptor nodeNotInMap = DFSTestUtil.GetDatanodeDescriptor("3.3.3.3", "/d1/r4" ); NUnit.Framework.Assert.IsFalse(map.Remove(nodeNotInMap)); NUnit.Framework.Assert.IsTrue(map.Remove(dataNodes[0])); NUnit.Framework.Assert.IsTrue(map.GetDatanodeByHost("1.1.1.1.") == null); NUnit.Framework.Assert.IsTrue(map.GetDatanodeByHost("2.2.2.2") == dataNodes[1]); DatanodeDescriptor node = map.GetDatanodeByHost("3.3.3.3"); NUnit.Framework.Assert.IsTrue(node == dataNodes[2] || node == dataNodes[3]); NUnit.Framework.Assert.IsNull(map.GetDatanodeByHost("4.4.4.4")); NUnit.Framework.Assert.IsTrue(map.Remove(dataNodes[2])); NUnit.Framework.Assert.IsNull(map.GetDatanodeByHost("1.1.1.1")); NUnit.Framework.Assert.AreEqual(map.GetDatanodeByHost("2.2.2.2"), dataNodes[1]); NUnit.Framework.Assert.AreEqual(map.GetDatanodeByHost("3.3.3.3"), dataNodes[3]); NUnit.Framework.Assert.IsTrue(map.Remove(dataNodes[3])); NUnit.Framework.Assert.IsNull(map.GetDatanodeByHost("1.1.1.1")); NUnit.Framework.Assert.AreEqual(map.GetDatanodeByHost("2.2.2.2"), dataNodes[1]); NUnit.Framework.Assert.IsNull(map.GetDatanodeByHost("3.3.3.3")); NUnit.Framework.Assert.IsFalse(map.Remove(null)); NUnit.Framework.Assert.IsTrue(map.Remove(dataNodes[1])); NUnit.Framework.Assert.IsFalse(map.Remove(dataNodes[1])); }
/// <exception cref="System.Exception"/> public virtual void TestSetrepIncWithUnderReplicatedBlocks() { // 1 min timeout Configuration conf = new HdfsConfiguration(); short ReplicationFactor = 2; string FileName = "/testFile"; Path FilePath = new Path(FileName); MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(ReplicationFactor + 1).Build(); try { // create a file with one block with a replication factor of 2 FileSystem fs = cluster.GetFileSystem(); DFSTestUtil.CreateFile(fs, FilePath, 1L, ReplicationFactor, 1L); DFSTestUtil.WaitReplication(fs, FilePath, ReplicationFactor); // remove one replica from the blocksMap so block becomes under-replicated // but the block does not get put into the under-replicated blocks queue BlockManager bm = cluster.GetNamesystem().GetBlockManager(); ExtendedBlock b = DFSTestUtil.GetFirstBlock(fs, FilePath); DatanodeDescriptor dn = bm.blocksMap.GetStorages(b.GetLocalBlock()).GetEnumerator ().Next().GetDatanodeDescriptor(); bm.AddToInvalidates(b.GetLocalBlock(), dn); Sharpen.Thread.Sleep(5000); bm.blocksMap.RemoveNode(b.GetLocalBlock(), dn); // increment this file's replication factor FsShell shell = new FsShell(conf); NUnit.Framework.Assert.AreEqual(0, shell.Run(new string[] { "-setrep", "-w", Sharpen.Extensions.ToString (1 + ReplicationFactor), FileName })); } finally { cluster.Shutdown(); } }
/// <summary>Add new entries to the PendingUncached list.</summary> /// <param name="neededUncached">The number of replicas that need to be uncached.</param> /// <param name="cachedBlock">The block which needs to be uncached.</param> /// <param name="cached">A list of DataNodes currently caching the block.</param> /// <param name="pendingUncached"> /// A list of DataNodes that will soon uncache the /// block. /// </param> private void AddNewPendingUncached(int neededUncached, CachedBlock cachedBlock, IList <DatanodeDescriptor> cached, IList <DatanodeDescriptor> pendingUncached) { // Figure out which replicas can be uncached. List <DatanodeDescriptor> possibilities = new List <DatanodeDescriptor>(); foreach (DatanodeDescriptor datanode in cached) { if (!pendingUncached.Contains(datanode)) { possibilities.AddItem(datanode); } } while (neededUncached > 0) { if (possibilities.IsEmpty()) { Log.Warn("Logic error: we're trying to uncache more replicas than " + "actually exist for " + cachedBlock); return; } DatanodeDescriptor datanode_1 = possibilities.Remove(random.Next(possibilities.Count )); pendingUncached.AddItem(datanode_1); bool added = datanode_1.GetPendingUncached().AddItem(cachedBlock); System.Diagnostics.Debug.Assert(added); neededUncached--; } }
/// <summary>Add all nodes from a dependent nodes list to excludedNodes.</summary> /// <returns>number of new excluded nodes</returns> private int AddDependentNodesToExcludedNodes(DatanodeDescriptor chosenNode, ICollection <Node> excludedNodes) { if (this.host2datanodeMap == null) { return(0); } int countOfExcludedNodes = 0; foreach (string hostname in chosenNode.GetDependentHostNames()) { DatanodeDescriptor node = this.host2datanodeMap.GetDataNodeByHostName(hostname); if (node != null) { if (excludedNodes.AddItem(node)) { countOfExcludedNodes++; } } else { Log.Warn("Not able to find datanode " + hostname + " which has dependency with datanode " + chosenNode.GetHostName()); } } return(countOfExcludedNodes); }
/// <summary> /// add node to the map /// return true if the node is added; false otherwise. /// </summary> internal virtual bool Add(DatanodeDescriptor node) { hostmapLock.WriteLock().Lock(); try { if (node == null || Contains(node)) { return(false); } string ipAddr = node.GetIpAddr(); string hostname = node.GetHostName(); mapHost[hostname] = ipAddr; DatanodeDescriptor[] nodes = map[ipAddr]; DatanodeDescriptor[] newNodes; if (nodes == null) { newNodes = new DatanodeDescriptor[1]; newNodes[0] = node; } else { // rare case: more than one datanode on the host newNodes = new DatanodeDescriptor[nodes.Length + 1]; System.Array.Copy(nodes, 0, newNodes, 0, nodes.Length); newNodes[nodes.Length] = node; } map[ipAddr] = newNodes; return(true); } finally { hostmapLock.WriteLock().Unlock(); } }
private void RemoveNode(DatanodeDescriptor deadNode) { NetworkTopology cluster = bm.GetDatanodeManager().GetNetworkTopology(); cluster.Remove(deadNode); bm.RemoveBlocksAssociatedTo(deadNode); }
/// <exception cref="System.Exception"/> private void DoTestTwoOfThreeNodesDecommissioned(int testIndex) { // Block originally on A1, A2, B1 IList <DatanodeStorageInfo> origStorages = GetStorages(0, 1, 3); IList <DatanodeDescriptor> origNodes = GetNodes(origStorages); BlockInfoContiguous blockInfo = AddBlockOnNodes(testIndex, origNodes); // Decommission two of the nodes (A1, A2) IList <DatanodeDescriptor> decomNodes = StartDecommission(0, 1); DatanodeStorageInfo[] pipeline = ScheduleSingleReplication(blockInfo); NUnit.Framework.Assert.IsTrue("Source of replication should be one of the nodes the block " + "was on. Was: " + pipeline[0], origStorages.Contains(pipeline[0])); NUnit.Framework.Assert.AreEqual("Should have three targets", 3, pipeline.Length); bool foundOneOnRackA = false; for (int i = 1; i < pipeline.Length; i++) { DatanodeDescriptor target = pipeline[i].GetDatanodeDescriptor(); if (rackA.Contains(target)) { foundOneOnRackA = true; } NUnit.Framework.Assert.IsFalse(decomNodes.Contains(target)); NUnit.Framework.Assert.IsFalse(origNodes.Contains(target)); } NUnit.Framework.Assert.IsTrue("Should have at least one target on rack A. Pipeline: " + Joiner.On(",").Join(pipeline), foundOneOnRackA); }
/// <summary> /// remove node from the map /// return true if the node is removed; false otherwise. /// </summary> internal virtual bool Remove(DatanodeDescriptor node) { if (node == null) { return(false); } string ipAddr = node.GetIpAddr(); string hostname = node.GetHostName(); hostmapLock.WriteLock().Lock(); try { DatanodeDescriptor[] nodes = map[ipAddr]; if (nodes == null) { return(false); } if (nodes.Length == 1) { if (nodes[0] == node) { Sharpen.Collections.Remove(map, ipAddr); //remove hostname key since last datanode is removed Sharpen.Collections.Remove(mapHost, hostname); return(true); } else { return(false); } } //rare case int i = 0; for (; i < nodes.Length; i++) { if (nodes[i] == node) { break; } } if (i == nodes.Length) { return(false); } else { DatanodeDescriptor[] newNodes; newNodes = new DatanodeDescriptor[nodes.Length - 1]; System.Array.Copy(nodes, 0, newNodes, 0, i); System.Array.Copy(nodes, i + 1, newNodes, i, nodes.Length - i - 1); map[ipAddr] = newNodes; return(true); } } finally { hostmapLock.WriteLock().Unlock(); } }
internal DatanodeStorageInfo(DatanodeDescriptor dn, DatanodeStorage s) { // The ID of the last full block report which updated this storage. this.dn = dn; this.storageID = s.GetStorageID(); this.storageType = s.GetStorageType(); this.state = s.GetState(); }
public virtual void TestChooseTargetWithDecomNodes() { namenode.GetNamesystem().WriteLock(); try { string blockPoolId = namenode.GetNamesystem().GetBlockPoolId(); dnManager.HandleHeartbeat(dnrList[3], BlockManagerTestUtil.GetStorageReportsForDatanode (dataNodes[3]), blockPoolId, dataNodes[3].GetCacheCapacity(), dataNodes[3].GetCacheRemaining (), 2, 0, 0, null); dnManager.HandleHeartbeat(dnrList[4], BlockManagerTestUtil.GetStorageReportsForDatanode (dataNodes[4]), blockPoolId, dataNodes[4].GetCacheCapacity(), dataNodes[4].GetCacheRemaining (), 4, 0, 0, null); dnManager.HandleHeartbeat(dnrList[5], BlockManagerTestUtil.GetStorageReportsForDatanode (dataNodes[5]), blockPoolId, dataNodes[5].GetCacheCapacity(), dataNodes[5].GetCacheRemaining (), 4, 0, 0, null); // value in the above heartbeats int load = 2 + 4 + 4; FSNamesystem fsn = namenode.GetNamesystem(); NUnit.Framework.Assert.AreEqual((double)load / 6, dnManager.GetFSClusterStats().GetInServiceXceiverAverage (), Epsilon); // Decommission DNs so BlockPlacementPolicyDefault.isGoodTarget() // returns false for (int i = 0; i < 3; i++) { DatanodeDescriptor d = dnManager.GetDatanode(dnrList[i]); dnManager.GetDecomManager().StartDecommission(d); d.SetDecommissioned(); } NUnit.Framework.Assert.AreEqual((double)load / 3, dnManager.GetFSClusterStats().GetInServiceXceiverAverage (), Epsilon); // update references of writer DN to update the de-commissioned state IList <DatanodeDescriptor> liveNodes = new AList <DatanodeDescriptor>(); dnManager.FetchDatanodes(liveNodes, null, false); DatanodeDescriptor writerDn = null; if (liveNodes.Contains(dataNodes[0])) { writerDn = liveNodes[liveNodes.IndexOf(dataNodes[0])]; } // Call chooseTarget() DatanodeStorageInfo[] targets = namenode.GetNamesystem().GetBlockManager().GetBlockPlacementPolicy ().ChooseTarget("testFile.txt", 3, writerDn, new AList <DatanodeStorageInfo>(), false , null, 1024, TestBlockStoragePolicy.DefaultStoragePolicy); NUnit.Framework.Assert.AreEqual(3, targets.Length); ICollection <DatanodeStorageInfo> targetSet = new HashSet <DatanodeStorageInfo>(Arrays .AsList(targets)); for (int i_1 = 3; i_1 < storages.Length; i_1++) { NUnit.Framework.Assert.IsTrue(targetSet.Contains(storages[i_1])); } } finally { dataNodes[0].StopDecommission(); dataNodes[1].StopDecommission(); dataNodes[2].StopDecommission(); namenode.GetNamesystem().WriteUnlock(); } }
public virtual void TestGetDatanodeByHost() { NUnit.Framework.Assert.AreEqual(map.GetDatanodeByHost("1.1.1.1"), dataNodes[0]); NUnit.Framework.Assert.AreEqual(map.GetDatanodeByHost("2.2.2.2"), dataNodes[1]); DatanodeDescriptor node = map.GetDatanodeByHost("3.3.3.3"); NUnit.Framework.Assert.IsTrue(node == dataNodes[2] || node == dataNodes[3]); NUnit.Framework.Assert.IsNull(map.GetDatanodeByHost("4.4.4.4")); }
private static void UpdateHeartbeatWithUsage(DatanodeDescriptor dn, long capacity , long dfsUsed, long remaining, long blockPoolUsed, long dnCacheCapacity, long dnCacheUsed , int xceiverCount, int volFailures) { dn.GetStorageInfos()[0].SetUtilizationForTesting(capacity, dfsUsed, remaining, blockPoolUsed ); dn.UpdateHeartbeat(BlockManagerTestUtil.GetStorageReportsForDatanode(dn), dnCacheCapacity , dnCacheUsed, xceiverCount, volFailures, null); }
internal virtual void AddDatanode(DatanodeDescriptor d) { lock (this) { // update in-service node count stats.Add(d); datanodes.AddItem(d); d.isAlive = true; } }
internal static DatanodeDescriptor[] ToDatanodeDescriptors(Org.Apache.Hadoop.Hdfs.Server.Blockmanagement.DatanodeStorageInfo [] storages) { DatanodeDescriptor[] datanodes = new DatanodeDescriptor[storages.Length]; for (int i = 0; i < storages.Length; ++i) { datanodes[i] = storages[i].GetDatanodeDescriptor(); } return(datanodes); }
public virtual void TestCorruptReplicaInfo() { CorruptReplicasMap crm = new CorruptReplicasMap(); // Make sure initial values are returned correctly NUnit.Framework.Assert.AreEqual("Number of corrupt blocks must initially be 0", 0 , crm.Size()); NUnit.Framework.Assert.IsNull("Param n cannot be less than 0", crm.GetCorruptReplicaBlockIds (-1, null)); NUnit.Framework.Assert.IsNull("Param n cannot be greater than 100", crm.GetCorruptReplicaBlockIds (101, null)); long[] l = crm.GetCorruptReplicaBlockIds(0, null); NUnit.Framework.Assert.IsNotNull("n = 0 must return non-null", l); NUnit.Framework.Assert.AreEqual("n = 0 must return an empty list", 0, l.Length); // create a list of block_ids. A list is used to allow easy validation of the // output of getCorruptReplicaBlockIds int NumBlockIds = 140; IList <long> block_ids = new List <long>(); for (int i = 0; i < NumBlockIds; i++) { block_ids.AddItem((long)i); } DatanodeDescriptor dn1 = DFSTestUtil.GetLocalDatanodeDescriptor(); DatanodeDescriptor dn2 = DFSTestUtil.GetLocalDatanodeDescriptor(); AddToCorruptReplicasMap(crm, GetBlock(0), dn1); NUnit.Framework.Assert.AreEqual("Number of corrupt blocks not returning correctly" , 1, crm.Size()); AddToCorruptReplicasMap(crm, GetBlock(1), dn1); NUnit.Framework.Assert.AreEqual("Number of corrupt blocks not returning correctly" , 2, crm.Size()); AddToCorruptReplicasMap(crm, GetBlock(1), dn2); NUnit.Framework.Assert.AreEqual("Number of corrupt blocks not returning correctly" , 2, crm.Size()); crm.RemoveFromCorruptReplicasMap(GetBlock(1)); NUnit.Framework.Assert.AreEqual("Number of corrupt blocks not returning correctly" , 1, crm.Size()); crm.RemoveFromCorruptReplicasMap(GetBlock(0)); NUnit.Framework.Assert.AreEqual("Number of corrupt blocks not returning correctly" , 0, crm.Size()); foreach (long block_id in block_ids) { AddToCorruptReplicasMap(crm, GetBlock(block_id), dn1); } NUnit.Framework.Assert.AreEqual("Number of corrupt blocks not returning correctly" , NumBlockIds, crm.Size()); NUnit.Framework.Assert.IsTrue("First five block ids not returned correctly ", Arrays .Equals(new long[] { 0, 1, 2, 3, 4 }, crm.GetCorruptReplicaBlockIds(5, null))); Log.Info(crm.GetCorruptReplicaBlockIds(10, 7L)); Log.Info(block_ids.SubList(7, 18)); NUnit.Framework.Assert.IsTrue("10 blocks after 7 not returned correctly ", Arrays .Equals(new long[] { 8, 9, 10, 11, 12, 13, 14, 15, 16, 17 }, crm.GetCorruptReplicaBlockIds (10, 7L))); }
public static DatanodeDescriptor GetLocalDatanodeDescriptor(bool initializeStorage ) { DatanodeDescriptor dn = new DatanodeDescriptor(DFSTestUtil.GetLocalDatanodeID()); if (initializeStorage) { dn.UpdateStorage(new DatanodeStorage(DatanodeStorage.GenerateUuid())); } return(dn); }
public virtual void TestContains() { DatanodeDescriptor nodeNotInMap = DFSTestUtil.GetDatanodeDescriptor("3.3.3.3", "/d1/r4" ); for (int i = 0; i < dataNodes.Length; i++) { NUnit.Framework.Assert.IsTrue(map.Contains(dataNodes[i])); } NUnit.Framework.Assert.IsFalse(map.Contains(null)); NUnit.Framework.Assert.IsFalse(map.Contains(nodeNotInMap)); }
public static DatanodeDescriptor GetDatanodeDescriptor(string ipAddr, string rackLocation , DatanodeStorage storage, string hostname) { DatanodeDescriptor dn = DFSTestUtil.GetDatanodeDescriptor(ipAddr, DFSConfigKeys.DfsDatanodeDefaultPort , rackLocation, hostname); if (storage != null) { dn.UpdateStorage(storage); } return(dn); }
internal virtual void UpdateHeartbeat(DatanodeDescriptor node, StorageReport[] reports , long cacheCapacity, long cacheUsed, int xceiverCount, int failedVolumes, VolumeFailureSummary volumeFailureSummary) { lock (this) { stats.Subtract(node); node.UpdateHeartbeat(reports, cacheCapacity, cacheUsed, xceiverCount, failedVolumes , volumeFailureSummary); stats.Add(node); } }
public virtual void TestIncludeExcludeLists() { BlockManager bm = Org.Mockito.Mockito.Mock <BlockManager>(); FSNamesystem fsn = Org.Mockito.Mockito.Mock <FSNamesystem>(); Configuration conf = new Configuration(); HostFileManager hm = new HostFileManager(); HostFileManager.HostSet includedNodes = new HostFileManager.HostSet(); HostFileManager.HostSet excludedNodes = new HostFileManager.HostSet(); includedNodes.Add(Entry("127.0.0.1:12345")); includedNodes.Add(Entry("localhost:12345")); includedNodes.Add(Entry("127.0.0.1:12345")); includedNodes.Add(Entry("127.0.0.2")); excludedNodes.Add(Entry("127.0.0.1:12346")); excludedNodes.Add(Entry("127.0.30.1:12346")); NUnit.Framework.Assert.AreEqual(2, includedNodes.Size()); NUnit.Framework.Assert.AreEqual(2, excludedNodes.Size()); hm.Refresh(includedNodes, excludedNodes); DatanodeManager dm = new DatanodeManager(bm, fsn, conf); Whitebox.SetInternalState(dm, "hostFileManager", hm); IDictionary <string, DatanodeDescriptor> dnMap = (IDictionary <string, DatanodeDescriptor >)Whitebox.GetInternalState(dm, "datanodeMap"); // After the de-duplication, there should be only one DN from the included // nodes declared as dead. NUnit.Framework.Assert.AreEqual(2, dm.GetDatanodeListForReport(HdfsConstants.DatanodeReportType .All).Count); NUnit.Framework.Assert.AreEqual(2, dm.GetDatanodeListForReport(HdfsConstants.DatanodeReportType .Dead).Count); dnMap["uuid-foo"] = new DatanodeDescriptor(new DatanodeID("127.0.0.1", "localhost" , "uuid-foo", 12345, 1020, 1021, 1022)); NUnit.Framework.Assert.AreEqual(1, dm.GetDatanodeListForReport(HdfsConstants.DatanodeReportType .Dead).Count); dnMap["uuid-bar"] = new DatanodeDescriptor(new DatanodeID("127.0.0.2", "127.0.0.2" , "uuid-bar", 12345, 1020, 1021, 1022)); NUnit.Framework.Assert.AreEqual(0, dm.GetDatanodeListForReport(HdfsConstants.DatanodeReportType .Dead).Count); DatanodeDescriptor spam = new DatanodeDescriptor(new DatanodeID("127.0.0" + ".3", "127.0.0.3", "uuid-spam", 12345, 1020, 1021, 1022)); DFSTestUtil.SetDatanodeDead(spam); includedNodes.Add(Entry("127.0.0.3:12345")); dnMap["uuid-spam"] = spam; NUnit.Framework.Assert.AreEqual(1, dm.GetDatanodeListForReport(HdfsConstants.DatanodeReportType .Dead).Count); Sharpen.Collections.Remove(dnMap, "uuid-spam"); NUnit.Framework.Assert.AreEqual(1, dm.GetDatanodeListForReport(HdfsConstants.DatanodeReportType .Dead).Count); excludedNodes.Add(Entry("127.0.0.3")); NUnit.Framework.Assert.AreEqual(0, dm.GetDatanodeListForReport(HdfsConstants.DatanodeReportType .Dead).Count); }