Esempio n. 1
0
        /// <summary>Checks whether a block is sufficiently replicated for decommissioning.</summary>
        /// <remarks>
        /// Checks whether a block is sufficiently replicated for decommissioning.
        /// Full-strength replication is not always necessary, hence "sufficient".
        /// </remarks>
        /// <returns>true if sufficient, else false.</returns>
        private bool IsSufficientlyReplicated(BlockInfoContiguous block, BlockCollection
                                              bc, NumberReplicas numberReplicas)
        {
            int numExpected = bc.GetBlockReplication();
            int numLive     = numberReplicas.LiveReplicas();

            if (!blockManager.IsNeededReplication(block, numExpected, numLive))
            {
                // Block doesn't need replication. Skip.
                Log.Trace("Block {} does not need replication.", block);
                return(true);
            }
            // Block is under-replicated
            Log.Trace("Block {} numExpected={}, numLive={}", block, numExpected, numLive);
            if (numExpected > numLive)
            {
                if (bc.IsUnderConstruction() && block.Equals(bc.GetLastBlock()))
                {
                    // Can decom a UC block as long as there will still be minReplicas
                    if (numLive >= blockManager.minReplication)
                    {
                        Log.Trace("UC block {} sufficiently-replicated since numLive ({}) " + ">= minR ({})"
                                  , block, numLive, blockManager.minReplication);
                        return(true);
                    }
                    else
                    {
                        Log.Trace("UC block {} insufficiently-replicated since numLive " + "({}) < minR ({})"
                                  , block, numLive, blockManager.minReplication);
                    }
                }
                else
                {
                    // Can decom a non-UC as long as the default replication is met
                    if (numLive >= blockManager.defaultReplication)
                    {
                        return(true);
                    }
                }
            }
            return(false);
        }
Esempio n. 2
0
        private static void LogBlockReplicationInfo(Block block, BlockCollection bc, DatanodeDescriptor
                                                    srcNode, NumberReplicas num, IEnumerable <DatanodeStorageInfo> storages)
        {
            int           curReplicas         = num.LiveReplicas();
            int           curExpectedReplicas = bc.GetBlockReplication();
            StringBuilder nodeList            = new StringBuilder();

            foreach (DatanodeStorageInfo storage in storages)
            {
                DatanodeDescriptor node = storage.GetDatanodeDescriptor();
                nodeList.Append(node);
                nodeList.Append(" ");
            }
            Log.Info("Block: " + block + ", Expected Replicas: " + curExpectedReplicas + ", live replicas: "
                     + curReplicas + ", corrupt replicas: " + num.CorruptReplicas() + ", decommissioned replicas: "
                     + num.DecommissionedReplicas() + ", excess replicas: " + num.ExcessReplicas() +
                     ", Is Open File: " + bc.IsUnderConstruction() + ", Datanodes having this block: "
                     + nodeList + ", Current Datanode: " + srcNode + ", Is current datanode decommissioning: "
                     + srcNode.IsDecommissionInProgress());
        }
Esempio n. 3
0
            /// <summary>
            /// Used while checking if decommission-in-progress datanodes can be marked
            /// as decommissioned.
            /// </summary>
            /// <remarks>
            /// Used while checking if decommission-in-progress datanodes can be marked
            /// as decommissioned. Combines shared logic of
            /// pruneSufficientlyReplicated and handleInsufficientlyReplicated.
            /// </remarks>
            /// <param name="datanode">Datanode</param>
            /// <param name="it">
            /// Iterator over the blocks on the
            /// datanode
            /// </param>
            /// <param name="insufficientlyReplicated">
            /// Return parameter. If it's not null,
            /// will contain the insufficiently
            /// replicated-blocks from the list.
            /// </param>
            /// <param name="pruneSufficientlyReplicated">
            /// whether to remove sufficiently
            /// replicated blocks from the iterator
            /// </param>
            /// <returns>
            /// true if there are under-replicated blocks in the provided block
            /// iterator, else false.
            /// </returns>
            private void ProcessBlocksForDecomInternal(DatanodeDescriptor datanode, IEnumerator
                                                       <BlockInfoContiguous> it, IList <BlockInfoContiguous> insufficientlyReplicated, bool
                                                       pruneSufficientlyReplicated)
            {
                bool firstReplicationLog        = true;
                int  underReplicatedBlocks      = 0;
                int  decommissionOnlyReplicas   = 0;
                int  underReplicatedInOpenFiles = 0;

                while (it.HasNext())
                {
                    this.numBlocksChecked++;
                    BlockInfoContiguous block = it.Next();
                    // Remove the block from the list if it's no longer in the block map,
                    // e.g. the containing file has been deleted
                    if (this._enclosing.blockManager.blocksMap.GetStoredBlock(block) == null)
                    {
                        DecommissionManager.Log.Trace("Removing unknown block {}", block);
                        it.Remove();
                        continue;
                    }
                    BlockCollection bc = this._enclosing.blockManager.blocksMap.GetBlockCollection(block
                                                                                                   );
                    if (bc == null)
                    {
                        // Orphan block, will be invalidated eventually. Skip.
                        continue;
                    }
                    NumberReplicas num          = this._enclosing.blockManager.CountNodes(block);
                    int            liveReplicas = num.LiveReplicas();
                    int            curReplicas  = liveReplicas;
                    // Schedule under-replicated blocks for replication if not already
                    // pending
                    if (this._enclosing.blockManager.IsNeededReplication(block, bc.GetBlockReplication
                                                                             (), liveReplicas))
                    {
                        if (!this._enclosing.blockManager.neededReplications.Contains(block) && this._enclosing
                            .blockManager.pendingReplications.GetNumReplicas(block) == 0 && this._enclosing.
                            namesystem.IsPopulatingReplQueues())
                        {
                            // Process these blocks only when active NN is out of safe mode.
                            this._enclosing.blockManager.neededReplications.Add(block, curReplicas, num.DecommissionedReplicas
                                                                                    (), bc.GetBlockReplication());
                        }
                    }
                    // Even if the block is under-replicated,
                    // it doesn't block decommission if it's sufficiently replicated
                    if (this._enclosing.IsSufficientlyReplicated(block, bc, num))
                    {
                        if (pruneSufficientlyReplicated)
                        {
                            it.Remove();
                        }
                        continue;
                    }
                    // We've found an insufficiently replicated block.
                    if (insufficientlyReplicated != null)
                    {
                        insufficientlyReplicated.AddItem(block);
                    }
                    // Log if this is our first time through
                    if (firstReplicationLog)
                    {
                        DecommissionManager.LogBlockReplicationInfo(block, bc, datanode, num, this._enclosing
                                                                    .blockManager.blocksMap.GetStorages(block));
                        firstReplicationLog = false;
                    }
                    // Update various counts
                    underReplicatedBlocks++;
                    if (bc.IsUnderConstruction())
                    {
                        underReplicatedInOpenFiles++;
                    }
                    if ((curReplicas == 0) && (num.DecommissionedReplicas() > 0))
                    {
                        decommissionOnlyReplicas++;
                    }
                }
                datanode.decommissioningStatus.Set(underReplicatedBlocks, decommissionOnlyReplicas
                                                   , underReplicatedInOpenFiles);
            }