public virtual void TestBlocksCounter() { DatanodeDescriptor dd = BlockManagerTestUtil.GetLocalDatanodeDescriptor(true); NUnit.Framework.Assert.AreEqual(0, dd.NumBlocks()); BlockInfoContiguous blk = new BlockInfoContiguous(new Block(1L), (short)1); BlockInfoContiguous blk1 = new BlockInfoContiguous(new Block(2L), (short)2); DatanodeStorageInfo[] storages = dd.GetStorageInfos(); NUnit.Framework.Assert.IsTrue(storages.Length > 0); // add first block NUnit.Framework.Assert.IsTrue(storages[0].AddBlock(blk) == DatanodeStorageInfo.AddBlockResult .Added); NUnit.Framework.Assert.AreEqual(1, dd.NumBlocks()); // remove a non-existent block NUnit.Framework.Assert.IsFalse(dd.RemoveBlock(blk1)); NUnit.Framework.Assert.AreEqual(1, dd.NumBlocks()); // add an existent block NUnit.Framework.Assert.IsFalse(storages[0].AddBlock(blk) == DatanodeStorageInfo.AddBlockResult .Added); NUnit.Framework.Assert.AreEqual(1, dd.NumBlocks()); // add second block NUnit.Framework.Assert.IsTrue(storages[0].AddBlock(blk1) == DatanodeStorageInfo.AddBlockResult .Added); NUnit.Framework.Assert.AreEqual(2, dd.NumBlocks()); // remove first block NUnit.Framework.Assert.IsTrue(dd.RemoveBlock(blk)); NUnit.Framework.Assert.AreEqual(1, dd.NumBlocks()); // remove second block NUnit.Framework.Assert.IsTrue(dd.RemoveBlock(blk1)); NUnit.Framework.Assert.AreEqual(0, dd.NumBlocks()); }
public virtual DatanodeStorageInfo.AddBlockResult AddBlock(BlockInfoContiguous b) { // First check whether the block belongs to a different storage // on the same DN. DatanodeStorageInfo.AddBlockResult result = DatanodeStorageInfo.AddBlockResult.Added; DatanodeStorageInfo otherStorage = b.FindStorageInfo(GetDatanodeDescriptor()); if (otherStorage != null) { if (otherStorage != this) { // The block belongs to a different storage. Remove it first. otherStorage.RemoveBlock(b); result = DatanodeStorageInfo.AddBlockResult.Replaced; } else { // The block is already associated with this storage. return(DatanodeStorageInfo.AddBlockResult.AlreadyExist); } } // add to the head of the data-node list b.AddStorage(this); blockList = b.ListInsert(blockList, this); numBlocks++; return(result); }
/// <exception cref="System.Exception"/> private void DoTestTwoOfThreeNodesDecommissioned(int testIndex) { // Block originally on A1, A2, B1 IList <DatanodeStorageInfo> origStorages = GetStorages(0, 1, 3); IList <DatanodeDescriptor> origNodes = GetNodes(origStorages); BlockInfoContiguous blockInfo = AddBlockOnNodes(testIndex, origNodes); // Decommission two of the nodes (A1, A2) IList <DatanodeDescriptor> decomNodes = StartDecommission(0, 1); DatanodeStorageInfo[] pipeline = ScheduleSingleReplication(blockInfo); NUnit.Framework.Assert.IsTrue("Source of replication should be one of the nodes the block " + "was on. Was: " + pipeline[0], origStorages.Contains(pipeline[0])); NUnit.Framework.Assert.AreEqual("Should have three targets", 3, pipeline.Length); bool foundOneOnRackA = false; for (int i = 1; i < pipeline.Length; i++) { DatanodeDescriptor target = pipeline[i].GetDatanodeDescriptor(); if (rackA.Contains(target)) { foundOneOnRackA = true; } NUnit.Framework.Assert.IsFalse(decomNodes.Contains(target)); NUnit.Framework.Assert.IsFalse(origNodes.Contains(target)); } NUnit.Framework.Assert.IsTrue("Should have at least one target on rack A. Pipeline: " + Joiner.On(",").Join(pipeline), foundOneOnRackA); }
/// <summary>Remove block from the list of blocks belonging to the data-node.</summary> /// <remarks> /// Remove block from the list of blocks belonging to the data-node. Remove /// data-node from the block. /// </remarks> internal virtual bool RemoveBlock(string storageID, BlockInfoContiguous b) { DatanodeStorageInfo s = GetStorageInfo(storageID); if (s != null) { return(s.RemoveBlock(b)); } return(false); }
private BlockInfoContiguous AddBlockToBM(long blkId) { Block block = new Block(blkId); BlockInfoContiguous blockInfo = new BlockInfoContiguous(block, (short)3); BlockCollection bc = Org.Mockito.Mockito.Mock <BlockCollection>(); Org.Mockito.Mockito.DoReturn((short)3).When(bc).GetBlockReplication(); bm.blocksMap.AddBlockCollection(blockInfo, bc); return(blockInfo); }
public virtual void TestAddStorage() { BlockInfoContiguous blockInfo = new BlockInfoContiguous((short)3); DatanodeStorageInfo storage = DFSTestUtil.CreateDatanodeStorageInfo("storageID", "127.0.0.1"); bool added = blockInfo.AddStorage(storage); NUnit.Framework.Assert.IsTrue(added); NUnit.Framework.Assert.AreEqual(storage, blockInfo.GetStorageInfo(0)); }
/// <summary> /// Tell the block manager that replication is completed for the given /// pipeline. /// </summary> /// <exception cref="System.IO.IOException"/> private void FulfillPipeline(BlockInfoContiguous blockInfo, DatanodeStorageInfo[] pipeline) { for (int i = 1; i < pipeline.Length; i++) { DatanodeStorageInfo storage = pipeline[i]; bm.AddBlock(storage, blockInfo, null); blockInfo.AddStorage(storage); } }
/// <summary>Remove block from the list of blocks belonging to the data-node.</summary> /// <remarks> /// Remove block from the list of blocks belonging to the data-node. Remove /// data-node from the block. /// </remarks> internal virtual bool RemoveBlock(BlockInfoContiguous b) { DatanodeStorageInfo s = b.FindStorageInfo(this); // if block exists on this datanode if (s != null) { return(s.RemoveBlock(b)); } return(false); }
private BlockInfoContiguous AddBlockOnNodes(long blockId, IList <DatanodeDescriptor > nodes) { BlockCollection bc = Org.Mockito.Mockito.Mock <BlockCollection>(); Org.Mockito.Mockito.DoReturn((short)3).When(bc).GetBlockReplication(); BlockInfoContiguous blockInfo = BlockOnNodes(blockId, nodes); bm.blocksMap.AddBlockCollection(blockInfo, bc); return(blockInfo); }
public virtual bool RemoveBlock(BlockInfoContiguous b) { blockList = b.ListRemove(blockList, this); if (b.RemoveStorage(this)) { numBlocks--; return(true); } else { return(false); } }
private void DoBasicTest(int testIndex) { IList <DatanodeStorageInfo> origStorages = GetStorages(0, 1); IList <DatanodeDescriptor> origNodes = GetNodes(origStorages); BlockInfoContiguous blockInfo = AddBlockOnNodes(testIndex, origNodes); DatanodeStorageInfo[] pipeline = ScheduleSingleReplication(blockInfo); NUnit.Framework.Assert.AreEqual(2, pipeline.Length); NUnit.Framework.Assert.IsTrue("Source of replication should be one of the nodes the block " + "was on. Was: " + pipeline[0], origStorages.Contains(pipeline[0])); NUnit.Framework.Assert.IsTrue("Destination of replication should be on the other rack. " + "Was: " + pipeline[1], rackB.Contains(pipeline[1].GetDatanodeDescriptor())); }
private string FindReasonForNotCaching(CachedBlock cblock, BlockInfoContiguous blockInfo ) { if (blockInfo == null) { // Somehow, a cache report with the block arrived, but the block // reports from the DataNode haven't (yet?) described such a block. // Alternately, the NameNode might have invalidated the block, but the // DataNode hasn't caught up. In any case, we want to tell the DN // to uncache this. return("not tracked by the BlockManager"); } else { if (!blockInfo.IsComplete()) { // When a cached block changes state from complete to some other state // on the DataNode (perhaps because of append), it will begin the // uncaching process. However, the uncaching process is not // instantaneous, especially if clients have pinned the block. So // there may be a period of time when incomplete blocks remain cached // on the DataNodes. return("not complete"); } else { if (cblock.GetReplication() == 0) { // Since 0 is not a valid value for a cache directive's replication // field, seeing a replication of 0 on a CacheBlock means that it // has never been reached by any sweep. return("not needed by any directives"); } else { if (cblock.GetMark() != mark) { // Although the block was needed in the past, we didn't reach it during // the current sweep. Therefore, it doesn't need to be cached any more. // Need to set the replication to 0 so it doesn't flip back to cached // when the mark flips on the next scan cblock.SetReplicationAndMark((short)0, mark); return("no longer needed by any directives"); } } } } return(null); }
private void DoTestSufficientlyReplBlocksUsesNewRack(int testIndex) { // Originally on only nodes in rack A. IList <DatanodeDescriptor> origNodes = rackA; BlockInfoContiguous blockInfo = AddBlockOnNodes(testIndex, origNodes); DatanodeStorageInfo[] pipeline = ScheduleSingleReplication(blockInfo); NUnit.Framework.Assert.AreEqual(2, pipeline.Length); // single new copy NUnit.Framework.Assert.IsTrue("Source of replication should be one of the nodes the block " + "was on. Was: " + pipeline[0], origNodes.Contains(pipeline[0].GetDatanodeDescriptor ())); NUnit.Framework.Assert.IsTrue("Destination of replication should be on the other rack. " + "Was: " + pipeline[1], rackB.Contains(pipeline[1].GetDatanodeDescriptor())); }
private BlockInfoContiguous BlockOnNodes(long blkId, IList <DatanodeDescriptor> nodes ) { Block block = new Block(blkId); BlockInfoContiguous blockInfo = new BlockInfoContiguous(block, (short)3); foreach (DatanodeDescriptor dn in nodes) { foreach (DatanodeStorageInfo storage in dn.GetStorageInfos()) { blockInfo.AddStorage(storage); } } return(blockInfo); }
/// <exception cref="System.Exception"/> private void DoTestOneOfTwoRacksDecommissioned(int testIndex) { // Block originally on A1, A2, B1 IList <DatanodeStorageInfo> origStorages = GetStorages(0, 1, 3); IList <DatanodeDescriptor> origNodes = GetNodes(origStorages); BlockInfoContiguous blockInfo = AddBlockOnNodes(testIndex, origNodes); // Decommission all of the nodes in rack A IList <DatanodeDescriptor> decomNodes = StartDecommission(0, 1, 2); DatanodeStorageInfo[] pipeline = ScheduleSingleReplication(blockInfo); NUnit.Framework.Assert.IsTrue("Source of replication should be one of the nodes the block " + "was on. Was: " + pipeline[0], origStorages.Contains(pipeline[0])); // Only up to two nodes can be picked per rack when there are two racks. NUnit.Framework.Assert.AreEqual("Should have two targets", 2, pipeline.Length); bool foundOneOnRackB = false; for (int i = 1; i < pipeline.Length; i++) { DatanodeDescriptor target = pipeline[i].GetDatanodeDescriptor(); if (rackB.Contains(target)) { foundOneOnRackB = true; } NUnit.Framework.Assert.IsFalse(decomNodes.Contains(target)); NUnit.Framework.Assert.IsFalse(origNodes.Contains(target)); } NUnit.Framework.Assert.IsTrue("Should have at least one target on rack B. Pipeline: " + Joiner.On(",").Join(pipeline), foundOneOnRackB); // Mark the block as received on the target nodes in the pipeline FulfillPipeline(blockInfo, pipeline); // the block is still under-replicated. Add a new node. This should allow // the third off-rack replica. DatanodeDescriptor rackCNode = DFSTestUtil.GetDatanodeDescriptor("7.7.7.7", "/rackC" ); rackCNode.UpdateStorage(new DatanodeStorage(DatanodeStorage.GenerateUuid())); AddNodes(ImmutableList.Of(rackCNode)); try { DatanodeStorageInfo[] pipeline2 = ScheduleSingleReplication(blockInfo); NUnit.Framework.Assert.AreEqual(2, pipeline2.Length); NUnit.Framework.Assert.AreEqual(rackCNode, pipeline2[1].GetDatanodeDescriptor()); } finally { RemoveNode(rackCNode); } }
public virtual void TestAllReplicasOnSameRack() { Configuration conf = new HdfsConfiguration(); conf.Unset(DFSConfigKeys.NetTopologyScriptFileNameKey); fsn = Org.Mockito.Mockito.Mock <FSNamesystem>(); Org.Mockito.Mockito.DoReturn(true).When(fsn).HasWriteLock(); Org.Mockito.Mockito.DoReturn(true).When(fsn).HasReadLock(); bm = new BlockManager(fsn, conf); // Add nodes on two racks AddNodes(nodes); // Added a new block in blocksMap and all the replicas are on the same rack BlockInfoContiguous blockInfo = AddBlockOnNodes(1, rackA); // Since the network toppolgy is multi-rack, the blockHasEnoughRacks // should return false. NUnit.Framework.Assert.IsFalse("Replicas for block is not stored on enough racks" , bm.BlockHasEnoughRacks(blockInfo)); }
/// <summary>Checks whether a block is sufficiently replicated for decommissioning.</summary> /// <remarks> /// Checks whether a block is sufficiently replicated for decommissioning. /// Full-strength replication is not always necessary, hence "sufficient". /// </remarks> /// <returns>true if sufficient, else false.</returns> private bool IsSufficientlyReplicated(BlockInfoContiguous block, BlockCollection bc, NumberReplicas numberReplicas) { int numExpected = bc.GetBlockReplication(); int numLive = numberReplicas.LiveReplicas(); if (!blockManager.IsNeededReplication(block, numExpected, numLive)) { // Block doesn't need replication. Skip. Log.Trace("Block {} does not need replication.", block); return(true); } // Block is under-replicated Log.Trace("Block {} numExpected={}, numLive={}", block, numExpected, numLive); if (numExpected > numLive) { if (bc.IsUnderConstruction() && block.Equals(bc.GetLastBlock())) { // Can decom a UC block as long as there will still be minReplicas if (numLive >= blockManager.minReplication) { Log.Trace("UC block {} sufficiently-replicated since numLive ({}) " + ">= minR ({})" , block, numLive, blockManager.minReplication); return(true); } else { Log.Trace("UC block {} insufficiently-replicated since numLive " + "({}) < minR ({})" , block, numLive, blockManager.minReplication); } } else { // Can decom a non-UC as long as the default replication is met if (numLive >= blockManager.defaultReplication) { return(true); } } } return(false); }
public virtual void TestReplaceStorage() { // Create two dummy storages. DatanodeStorageInfo storage1 = DFSTestUtil.CreateDatanodeStorageInfo("storageID1" , "127.0.0.1"); DatanodeStorageInfo storage2 = new DatanodeStorageInfo(storage1.GetDatanodeDescriptor (), new DatanodeStorage("storageID2")); int NumBlocks = 10; BlockInfoContiguous[] blockInfos = new BlockInfoContiguous[NumBlocks]; // Create a few dummy blocks and add them to the first storage. for (int i = 0; i < NumBlocks; ++i) { blockInfos[i] = new BlockInfoContiguous((short)3); storage1.AddBlock(blockInfos[i]); } // Try to move one of the blocks to a different storage. bool added = storage2.AddBlock(blockInfos[NumBlocks / 2]) == DatanodeStorageInfo.AddBlockResult .Added; Assert.AssertThat(added, IS.Is(false)); Assert.AssertThat(blockInfos[NumBlocks / 2].GetStorageInfo(0), IS.Is(storage2)); }
/// <summary>Move block to the head of the list of blocks belonging to the data-node. /// </summary> /// <returns>the index of the head of the blockList</returns> internal virtual int MoveBlockToHead(BlockInfoContiguous b, int curIndex, int headIndex ) { blockList = b.MoveBlockToHead(blockList, this, curIndex, headIndex); return(curIndex); }
public virtual void TestProcessPendingReplications() { Configuration conf = new HdfsConfiguration(); conf.SetLong(DFSConfigKeys.DfsNamenodeReplicationPendingTimeoutSecKey, Timeout); MiniDFSCluster cluster = null; Block block; BlockInfoContiguous blockInfo; try { cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(DatanodeCount).Build(); cluster.WaitActive(); FSNamesystem fsn = cluster.GetNamesystem(); BlockManager blkManager = fsn.GetBlockManager(); PendingReplicationBlocks pendingReplications = blkManager.pendingReplications; UnderReplicatedBlocks neededReplications = blkManager.neededReplications; BlocksMap blocksMap = blkManager.blocksMap; // // Add 1 block to pendingReplications with GenerationStamp = 0. // block = new Block(1, 1, 0); blockInfo = new BlockInfoContiguous(block, (short)3); pendingReplications.Increment(block, DatanodeStorageInfo.ToDatanodeDescriptors(DFSTestUtil .CreateDatanodeStorageInfos(1))); BlockCollection bc = Org.Mockito.Mockito.Mock <BlockCollection>(); Org.Mockito.Mockito.DoReturn((short)3).When(bc).GetBlockReplication(); // Place into blocksmap with GenerationStamp = 1 blockInfo.SetGenerationStamp(1); blocksMap.AddBlockCollection(blockInfo, bc); NUnit.Framework.Assert.AreEqual("Size of pendingReplications ", 1, pendingReplications .Size()); // Add a second block to pendingReplications that has no // corresponding entry in blocksmap block = new Block(2, 2, 0); pendingReplications.Increment(block, DatanodeStorageInfo.ToDatanodeDescriptors(DFSTestUtil .CreateDatanodeStorageInfos(1))); // verify 2 blocks in pendingReplications NUnit.Framework.Assert.AreEqual("Size of pendingReplications ", 2, pendingReplications .Size()); // // Wait for everything to timeout. // while (pendingReplications.Size() > 0) { try { Sharpen.Thread.Sleep(100); } catch (Exception) { } } // // Verify that block moves to neededReplications // while (neededReplications.Size() == 0) { try { Sharpen.Thread.Sleep(100); } catch (Exception) { } } // Verify that the generation stamp we will try to replicate // is now 1 foreach (Block b in neededReplications) { NUnit.Framework.Assert.AreEqual("Generation stamp is 1 ", 1, b.GetGenerationStamp ()); } // Verify size of neededReplications is exactly 1. NUnit.Framework.Assert.AreEqual("size of neededReplications is 1 ", 1, neededReplications .Size()); } finally { if (cluster != null) { cluster.Shutdown(); } } }
public virtual void TestBlockListMoveToHead() { Log.Info("BlockInfo moveToHead tests..."); int MaxBlocks = 10; DatanodeStorageInfo dd = DFSTestUtil.CreateDatanodeStorageInfo("s1", "1.1.1.1"); AList <Block> blockList = new AList <Block>(MaxBlocks); AList <BlockInfoContiguous> blockInfoList = new AList <BlockInfoContiguous>(); int headIndex; int curIndex; Log.Info("Building block list..."); for (int i = 0; i < MaxBlocks; i++) { blockList.AddItem(new Block(i, 0, GenerationStamp.LastReservedStamp)); blockInfoList.AddItem(new BlockInfoContiguous(blockList[i], (short)3)); dd.AddBlock(blockInfoList[i]); // index of the datanode should be 0 NUnit.Framework.Assert.AreEqual("Find datanode should be 0", 0, blockInfoList[i]. FindStorageInfo(dd)); } // list length should be equal to the number of blocks we inserted Log.Info("Checking list length..."); NUnit.Framework.Assert.AreEqual("Length should be MAX_BLOCK", MaxBlocks, dd.NumBlocks ()); IEnumerator <BlockInfoContiguous> it = dd.GetBlockIterator(); int len = 0; while (it.HasNext()) { it.Next(); len++; } NUnit.Framework.Assert.AreEqual("There should be MAX_BLOCK blockInfo's", MaxBlocks , len); headIndex = dd.GetBlockListHeadForTesting().FindStorageInfo(dd); Log.Info("Moving each block to the head of the list..."); for (int i_1 = 0; i_1 < MaxBlocks; i_1++) { curIndex = blockInfoList[i_1].FindStorageInfo(dd); headIndex = dd.MoveBlockToHead(blockInfoList[i_1], curIndex, headIndex); // the moved element must be at the head of the list NUnit.Framework.Assert.AreEqual("Block should be at the head of the list now.", blockInfoList [i_1], dd.GetBlockListHeadForTesting()); } // move head of the list to the head - this should not change the list Log.Info("Moving head to the head..."); BlockInfoContiguous temp = dd.GetBlockListHeadForTesting(); curIndex = 0; headIndex = 0; dd.MoveBlockToHead(temp, curIndex, headIndex); NUnit.Framework.Assert.AreEqual("Moving head to the head of the list shopuld not change the list" , temp, dd.GetBlockListHeadForTesting()); // check all elements of the list against the original blockInfoList Log.Info("Checking elements of the list..."); temp = dd.GetBlockListHeadForTesting(); NUnit.Framework.Assert.IsNotNull("Head should not be null", temp); int c = MaxBlocks - 1; while (temp != null) { NUnit.Framework.Assert.AreEqual("Expected element is not on the list", blockInfoList [c--], temp); temp = temp.GetNext(0); } Log.Info("Moving random blocks to the head of the list..."); headIndex = dd.GetBlockListHeadForTesting().FindStorageInfo(dd); Random rand = new Random(); for (int i_2 = 0; i_2 < MaxBlocks; i_2++) { int j = rand.Next(MaxBlocks); curIndex = blockInfoList[j].FindStorageInfo(dd); headIndex = dd.MoveBlockToHead(blockInfoList[j], curIndex, headIndex); // the moved element must be at the head of the list NUnit.Framework.Assert.AreEqual("Block should be at the head of the list now.", blockInfoList [j], dd.GetBlockListHeadForTesting()); } }
/// <summary> /// Used while checking if decommission-in-progress datanodes can be marked /// as decommissioned. /// </summary> /// <remarks> /// Used while checking if decommission-in-progress datanodes can be marked /// as decommissioned. Combines shared logic of /// pruneSufficientlyReplicated and handleInsufficientlyReplicated. /// </remarks> /// <param name="datanode">Datanode</param> /// <param name="it"> /// Iterator over the blocks on the /// datanode /// </param> /// <param name="insufficientlyReplicated"> /// Return parameter. If it's not null, /// will contain the insufficiently /// replicated-blocks from the list. /// </param> /// <param name="pruneSufficientlyReplicated"> /// whether to remove sufficiently /// replicated blocks from the iterator /// </param> /// <returns> /// true if there are under-replicated blocks in the provided block /// iterator, else false. /// </returns> private void ProcessBlocksForDecomInternal(DatanodeDescriptor datanode, IEnumerator <BlockInfoContiguous> it, IList <BlockInfoContiguous> insufficientlyReplicated, bool pruneSufficientlyReplicated) { bool firstReplicationLog = true; int underReplicatedBlocks = 0; int decommissionOnlyReplicas = 0; int underReplicatedInOpenFiles = 0; while (it.HasNext()) { this.numBlocksChecked++; BlockInfoContiguous block = it.Next(); // Remove the block from the list if it's no longer in the block map, // e.g. the containing file has been deleted if (this._enclosing.blockManager.blocksMap.GetStoredBlock(block) == null) { DecommissionManager.Log.Trace("Removing unknown block {}", block); it.Remove(); continue; } BlockCollection bc = this._enclosing.blockManager.blocksMap.GetBlockCollection(block ); if (bc == null) { // Orphan block, will be invalidated eventually. Skip. continue; } NumberReplicas num = this._enclosing.blockManager.CountNodes(block); int liveReplicas = num.LiveReplicas(); int curReplicas = liveReplicas; // Schedule under-replicated blocks for replication if not already // pending if (this._enclosing.blockManager.IsNeededReplication(block, bc.GetBlockReplication (), liveReplicas)) { if (!this._enclosing.blockManager.neededReplications.Contains(block) && this._enclosing .blockManager.pendingReplications.GetNumReplicas(block) == 0 && this._enclosing. namesystem.IsPopulatingReplQueues()) { // Process these blocks only when active NN is out of safe mode. this._enclosing.blockManager.neededReplications.Add(block, curReplicas, num.DecommissionedReplicas (), bc.GetBlockReplication()); } } // Even if the block is under-replicated, // it doesn't block decommission if it's sufficiently replicated if (this._enclosing.IsSufficientlyReplicated(block, bc, num)) { if (pruneSufficientlyReplicated) { it.Remove(); } continue; } // We've found an insufficiently replicated block. if (insufficientlyReplicated != null) { insufficientlyReplicated.AddItem(block); } // Log if this is our first time through if (firstReplicationLog) { DecommissionManager.LogBlockReplicationInfo(block, bc, datanode, num, this._enclosing .blockManager.blocksMap.GetStorages(block)); firstReplicationLog = false; } // Update various counts underReplicatedBlocks++; if (bc.IsUnderConstruction()) { underReplicatedInOpenFiles++; } if ((curReplicas == 0) && (num.DecommissionedReplicas() > 0)) { decommissionOnlyReplicas++; } } datanode.decommissioningStatus.Set(underReplicatedBlocks, decommissionOnlyReplicas , underReplicatedInOpenFiles); }
/// <summary>Add new entries to the PendingCached list.</summary> /// <param name="neededCached">The number of replicas that need to be cached.</param> /// <param name="cachedBlock">The block which needs to be cached.</param> /// <param name="cached">A list of DataNodes currently caching the block.</param> /// <param name="pendingCached"> /// A list of DataNodes that will soon cache the /// block. /// </param> private void AddNewPendingCached(int neededCached, CachedBlock cachedBlock, IList <DatanodeDescriptor> cached, IList <DatanodeDescriptor> pendingCached) { // To figure out which replicas can be cached, we consult the // blocksMap. We don't want to try to cache a corrupt replica, though. BlockInfoContiguous blockInfo = blockManager.GetStoredBlock(new Block(cachedBlock .GetBlockId())); if (blockInfo == null) { Log.Debug("Block {}: can't add new cached replicas," + " because there is no record of this block " + "on the NameNode.", cachedBlock.GetBlockId()); return; } if (!blockInfo.IsComplete()) { Log.Debug("Block {}: can't cache this block, because it is not yet" + " complete." , cachedBlock.GetBlockId()); return; } // Filter the list of replicas to only the valid targets IList <DatanodeDescriptor> possibilities = new List <DatanodeDescriptor>(); int numReplicas = blockInfo.GetCapacity(); ICollection <DatanodeDescriptor> corrupt = blockManager.GetCorruptReplicas(blockInfo ); int outOfCapacity = 0; for (int i = 0; i < numReplicas; i++) { DatanodeDescriptor datanode = blockInfo.GetDatanode(i); if (datanode == null) { continue; } if (datanode.IsDecommissioned() || datanode.IsDecommissionInProgress()) { continue; } if (corrupt != null && corrupt.Contains(datanode)) { continue; } if (pendingCached.Contains(datanode) || cached.Contains(datanode)) { continue; } long pendingBytes = 0; // Subtract pending cached blocks from effective capacity IEnumerator <CachedBlock> it = datanode.GetPendingCached().GetEnumerator(); while (it.HasNext()) { CachedBlock cBlock = it.Next(); BlockInfoContiguous info = blockManager.GetStoredBlock(new Block(cBlock.GetBlockId ())); if (info != null) { pendingBytes -= info.GetNumBytes(); } } it = datanode.GetPendingUncached().GetEnumerator(); // Add pending uncached blocks from effective capacity while (it.HasNext()) { CachedBlock cBlock = it.Next(); BlockInfoContiguous info = blockManager.GetStoredBlock(new Block(cBlock.GetBlockId ())); if (info != null) { pendingBytes += info.GetNumBytes(); } } long pendingCapacity = pendingBytes + datanode.GetCacheRemaining(); if (pendingCapacity < blockInfo.GetNumBytes()) { Log.Trace("Block {}: DataNode {} is not a valid possibility " + "because the block has size {}, but the DataNode only has {}" + "bytes of cache remaining ({} pending bytes, {} already cached.", blockInfo.GetBlockId (), datanode.GetDatanodeUuid(), blockInfo.GetNumBytes(), pendingCapacity, pendingBytes , datanode.GetCacheRemaining()); outOfCapacity++; continue; } possibilities.AddItem(datanode); } IList <DatanodeDescriptor> chosen = ChooseDatanodesForCaching(possibilities, neededCached , blockManager.GetDatanodeManager().GetStaleInterval()); foreach (DatanodeDescriptor datanode_1 in chosen) { Log.Trace("Block {}: added to PENDING_CACHED on DataNode {}", blockInfo.GetBlockId (), datanode_1.GetDatanodeUuid()); pendingCached.AddItem(datanode_1); bool added = datanode_1.GetPendingCached().AddItem(cachedBlock); System.Diagnostics.Debug.Assert(added); } // We were unable to satisfy the requested replication factor if (neededCached > chosen.Count) { Log.Debug("Block {}: we only have {} of {} cached replicas." + " {} DataNodes have insufficient cache capacity." , blockInfo.GetBlockId(), (cachedBlock.GetReplication() - neededCached + chosen. Count), cachedBlock.GetReplication(), outOfCapacity); } }
public virtual void TestSafeModeIBRBeforeFirstFullBR() { // pretend to be in safemode Org.Mockito.Mockito.DoReturn(true).When(fsn).IsInStartupSafeMode(); DatanodeDescriptor node = nodes[0]; DatanodeStorageInfo ds = node.GetStorageInfos()[0]; node.isAlive = true; DatanodeRegistration nodeReg = new DatanodeRegistration(node, null, null, string.Empty ); // register new node bm.GetDatanodeManager().RegisterDatanode(nodeReg); bm.GetDatanodeManager().AddDatanode(node); NUnit.Framework.Assert.AreEqual(node, bm.GetDatanodeManager().GetDatanode(node)); NUnit.Framework.Assert.AreEqual(0, ds.GetBlockReportCount()); // Build a incremental report IList <ReceivedDeletedBlockInfo> rdbiList = new AList <ReceivedDeletedBlockInfo>(); // Build a full report BlockListAsLongs.Builder builder = BlockListAsLongs.Builder(); // blk_42 is finalized. long receivedBlockId = 42; // arbitrary BlockInfoContiguous receivedBlock = AddBlockToBM(receivedBlockId); rdbiList.AddItem(new ReceivedDeletedBlockInfo(new Block(receivedBlock), ReceivedDeletedBlockInfo.BlockStatus .ReceivedBlock, null)); builder.Add(new FinalizedReplica(receivedBlock, null, null)); // blk_43 is under construction. long receivingBlockId = 43; BlockInfoContiguous receivingBlock = AddUcBlockToBM(receivingBlockId); rdbiList.AddItem(new ReceivedDeletedBlockInfo(new Block(receivingBlock), ReceivedDeletedBlockInfo.BlockStatus .ReceivingBlock, null)); builder.Add(new ReplicaBeingWritten(receivingBlock, null, null, null)); // blk_44 has 2 records in IBR. It's finalized. So full BR has 1 record. long receivingReceivedBlockId = 44; BlockInfoContiguous receivingReceivedBlock = AddBlockToBM(receivingReceivedBlockId ); rdbiList.AddItem(new ReceivedDeletedBlockInfo(new Block(receivingReceivedBlock), ReceivedDeletedBlockInfo.BlockStatus.ReceivingBlock, null)); rdbiList.AddItem(new ReceivedDeletedBlockInfo(new Block(receivingReceivedBlock), ReceivedDeletedBlockInfo.BlockStatus.ReceivedBlock, null)); builder.Add(new FinalizedReplica(receivingReceivedBlock, null, null)); // blk_45 is not in full BR, because it's deleted. long ReceivedDeletedBlockId = 45; rdbiList.AddItem(new ReceivedDeletedBlockInfo(new Block(ReceivedDeletedBlockId), ReceivedDeletedBlockInfo.BlockStatus.ReceivedBlock, null)); rdbiList.AddItem(new ReceivedDeletedBlockInfo(new Block(ReceivedDeletedBlockId), ReceivedDeletedBlockInfo.BlockStatus.DeletedBlock, null)); // blk_46 exists in DN for a long time, so it's in full BR, but not in IBR. long existedBlockId = 46; BlockInfoContiguous existedBlock = AddBlockToBM(existedBlockId); builder.Add(new FinalizedReplica(existedBlock, null, null)); // process IBR and full BR StorageReceivedDeletedBlocks srdb = new StorageReceivedDeletedBlocks(new DatanodeStorage (ds.GetStorageID()), Sharpen.Collections.ToArray(rdbiList, new ReceivedDeletedBlockInfo [rdbiList.Count])); bm.ProcessIncrementalBlockReport(node, srdb); // Make sure it's the first full report NUnit.Framework.Assert.AreEqual(0, ds.GetBlockReportCount()); bm.ProcessReport(node, new DatanodeStorage(ds.GetStorageID()), builder.Build(), null , false); NUnit.Framework.Assert.AreEqual(1, ds.GetBlockReportCount()); // verify the storage info is correct NUnit.Framework.Assert.IsTrue(bm.GetStoredBlock(new Block(receivedBlockId)).FindStorageInfo (ds) >= 0); NUnit.Framework.Assert.IsTrue(((BlockInfoContiguousUnderConstruction)bm.GetStoredBlock (new Block(receivingBlockId))).GetNumExpectedLocations() > 0); NUnit.Framework.Assert.IsTrue(bm.GetStoredBlock(new Block(receivingReceivedBlockId )).FindStorageInfo(ds) >= 0); NUnit.Framework.Assert.IsNull(bm.GetStoredBlock(new Block(ReceivedDeletedBlockId) )); NUnit.Framework.Assert.IsTrue(bm.GetStoredBlock(new Block(existedBlock)).FindStorageInfo (ds) >= 0); }
/// <summary>Scan through the cached block map.</summary> /// <remarks> /// Scan through the cached block map. /// Any blocks which are under-replicated should be assigned new Datanodes. /// Blocks that are over-replicated should be removed from Datanodes. /// </remarks> private void RescanCachedBlockMap() { for (IEnumerator <CachedBlock> cbIter = cachedBlocks.GetEnumerator(); cbIter.HasNext ();) { scannedBlocks++; CachedBlock cblock = cbIter.Next(); IList <DatanodeDescriptor> pendingCached = cblock.GetDatanodes(DatanodeDescriptor.CachedBlocksList.Type .PendingCached); IList <DatanodeDescriptor> cached = cblock.GetDatanodes(DatanodeDescriptor.CachedBlocksList.Type .Cached); IList <DatanodeDescriptor> pendingUncached = cblock.GetDatanodes(DatanodeDescriptor.CachedBlocksList.Type .PendingUncached); // Remove nodes from PENDING_UNCACHED if they were actually uncached. for (IEnumerator <DatanodeDescriptor> iter = pendingUncached.GetEnumerator(); iter .HasNext();) { DatanodeDescriptor datanode = iter.Next(); if (!cblock.IsInList(datanode.GetCached())) { Log.Trace("Block {}: removing from PENDING_UNCACHED for node {} " + "because the DataNode uncached it." , cblock.GetBlockId(), datanode.GetDatanodeUuid()); datanode.GetPendingUncached().Remove(cblock); iter.Remove(); } } BlockInfoContiguous blockInfo = blockManager.GetStoredBlock(new Block(cblock.GetBlockId ())); string reason = FindReasonForNotCaching(cblock, blockInfo); int neededCached = 0; if (reason != null) { Log.Trace("Block {}: can't cache block because it is {}", cblock.GetBlockId(), reason ); } else { neededCached = cblock.GetReplication(); } int numCached = cached.Count; if (numCached >= neededCached) { // If we have enough replicas, drop all pending cached. for (IEnumerator <DatanodeDescriptor> iter_1 = pendingCached.GetEnumerator(); iter_1 .HasNext();) { DatanodeDescriptor datanode = iter_1.Next(); datanode.GetPendingCached().Remove(cblock); iter_1.Remove(); Log.Trace("Block {}: removing from PENDING_CACHED for node {}" + "because we already have {} cached replicas and we only" + " need {}", cblock.GetBlockId(), datanode.GetDatanodeUuid(), numCached, neededCached ); } } if (numCached < neededCached) { // If we don't have enough replicas, drop all pending uncached. for (IEnumerator <DatanodeDescriptor> iter_1 = pendingUncached.GetEnumerator(); iter_1 .HasNext();) { DatanodeDescriptor datanode = iter_1.Next(); datanode.GetPendingUncached().Remove(cblock); iter_1.Remove(); Log.Trace("Block {}: removing from PENDING_UNCACHED for node {} " + "because we only have {} cached replicas and we need " + "{}", cblock.GetBlockId(), datanode.GetDatanodeUuid(), numCached, neededCached ); } } int neededUncached = numCached - (pendingUncached.Count + neededCached); if (neededUncached > 0) { AddNewPendingUncached(neededUncached, cblock, cached, pendingUncached); } else { int additionalCachedNeeded = neededCached - (numCached + pendingCached.Count); if (additionalCachedNeeded > 0) { AddNewPendingCached(additionalCachedNeeded, cblock, cached, pendingCached); } } if ((neededCached == 0) && pendingUncached.IsEmpty() && pendingCached.IsEmpty()) { // we have nothing more to do with this block. Log.Trace("Block {}: removing from cachedBlocks, since neededCached " + "== 0, and pendingUncached and pendingCached are empty." , cblock.GetBlockId()); cbIter.Remove(); } } }