Ejemplo n.º 1
0
        /// <exception cref="System.IO.IOException"/>
        public virtual HeartbeatResponse SendHeartbeat(DatanodeRegistration registration,
                                                       StorageReport[] reports, long cacheCapacity, long cacheUsed, int xmitsInProgress
                                                       , int xceiverCount, int failedVolumes, VolumeFailureSummary volumeFailureSummary
                                                       )
        {
            DatanodeProtocolProtos.HeartbeatRequestProto.Builder builder = DatanodeProtocolProtos.HeartbeatRequestProto
                                                                           .NewBuilder().SetRegistration(PBHelper.Convert(registration)).SetXmitsInProgress
                                                                               (xmitsInProgress).SetXceiverCount(xceiverCount).SetFailedVolumes(failedVolumes);
            builder.AddAllReports(PBHelper.ConvertStorageReports(reports));
            if (cacheCapacity != 0)
            {
                builder.SetCacheCapacity(cacheCapacity);
            }
            if (cacheUsed != 0)
            {
                builder.SetCacheUsed(cacheUsed);
            }
            if (volumeFailureSummary != null)
            {
                builder.SetVolumeFailureSummary(PBHelper.ConvertVolumeFailureSummary(volumeFailureSummary
                                                                                     ));
            }
            DatanodeProtocolProtos.HeartbeatResponseProto resp;
            try
            {
                resp = rpcProxy.SendHeartbeat(NullController, ((DatanodeProtocolProtos.HeartbeatRequestProto
                                                                )builder.Build()));
            }
            catch (ServiceException se)
            {
                throw ProtobufHelper.GetRemoteException(se);
            }
            DatanodeCommand[] cmds = new DatanodeCommand[resp.GetCmdsList().Count];
            int index = 0;

            foreach (DatanodeProtocolProtos.DatanodeCommandProto p in resp.GetCmdsList())
            {
                cmds[index] = PBHelper.Convert(p);
                index++;
            }
            RollingUpgradeStatus rollingUpdateStatus = null;

            // Use v2 semantics if available.
            if (resp.HasRollingUpgradeStatusV2())
            {
                rollingUpdateStatus = PBHelper.Convert(resp.GetRollingUpgradeStatusV2());
            }
            else
            {
                if (resp.HasRollingUpgradeStatus())
                {
                    rollingUpdateStatus = PBHelper.Convert(resp.GetRollingUpgradeStatus());
                }
            }
            return(new HeartbeatResponse(cmds, PBHelper.Convert(resp.GetHaStatus()), rollingUpdateStatus
                                         ));
        }
        /// <exception cref="Com.Google.Protobuf.ServiceException"/>
        public virtual DatanodeProtocolProtos.BlockReportResponseProto BlockReport(RpcController
                                                                                   controller, DatanodeProtocolProtos.BlockReportRequestProto request)
        {
            DatanodeCommand cmd = null;

            StorageBlockReport[] report = new StorageBlockReport[request.GetReportsCount()];
            int index = 0;

            foreach (DatanodeProtocolProtos.StorageBlockReportProto s in request.GetReportsList
                         ())
            {
                BlockListAsLongs blocks;
                if (s.HasNumberOfBlocks())
                {
                    // new style buffer based reports
                    int num = (int)s.GetNumberOfBlocks();
                    Preconditions.CheckState(s.GetBlocksCount() == 0, "cannot send both blocks list and buffers"
                                             );
                    blocks = BlockListAsLongs.DecodeBuffers(num, s.GetBlocksBuffersList());
                }
                else
                {
                    blocks = BlockListAsLongs.DecodeLongs(s.GetBlocksList());
                }
                report[index++] = new StorageBlockReport(PBHelper.Convert(s.GetStorage()), blocks
                                                         );
            }
            try
            {
                cmd = impl.BlockReport(PBHelper.Convert(request.GetRegistration()), request.GetBlockPoolId
                                           (), report, request.HasContext() ? PBHelper.Convert(request.GetContext()) : null
                                       );
            }
            catch (IOException e)
            {
                throw new ServiceException(e);
            }
            DatanodeProtocolProtos.BlockReportResponseProto.Builder builder = DatanodeProtocolProtos.BlockReportResponseProto
                                                                              .NewBuilder();
            if (cmd != null)
            {
                builder.SetCmd(PBHelper.Convert(cmd));
            }
            return((DatanodeProtocolProtos.BlockReportResponseProto)builder.Build());
        }
Ejemplo n.º 3
0
        /// <exception cref="System.IO.IOException"/>
        internal virtual bool ProcessCommandFromActor(DatanodeCommand cmd, BPServiceActor
                                                      actor)
        {
            System.Diagnostics.Debug.Assert(bpServices.Contains(actor));
            if (cmd == null)
            {
                return(true);
            }

            /*
             * Datanode Registration can be done asynchronously here. No need to hold
             * the lock. for more info refer HDFS-5014
             */
            if (DatanodeProtocol.DnaRegister == cmd.GetAction())
            {
                // namenode requested a registration - at start or if NN lost contact
                // Just logging the claiming state is OK here instead of checking the
                // actor state by obtaining the lock
                Log.Info("DatanodeCommand action : DNA_REGISTER from " + actor.nnAddr + " with "
                         + actor.state + " state");
                actor.ReRegister();
                return(false);
            }
            WriteLock();
            try
            {
                if (actor == bpServiceToActive)
                {
                    return(ProcessCommandFromActive(cmd, actor));
                }
                else
                {
                    return(ProcessCommandFromStandby(cmd, actor));
                }
            }
            finally
            {
                WriteUnlock();
            }
        }
        /// <exception cref="Com.Google.Protobuf.ServiceException"/>
        public virtual DatanodeProtocolProtos.CacheReportResponseProto CacheReport(RpcController
                                                                                   controller, DatanodeProtocolProtos.CacheReportRequestProto request)
        {
            DatanodeCommand cmd = null;

            try
            {
                cmd = impl.CacheReport(PBHelper.Convert(request.GetRegistration()), request.GetBlockPoolId
                                           (), request.GetBlocksList());
            }
            catch (IOException e)
            {
                throw new ServiceException(e);
            }
            DatanodeProtocolProtos.CacheReportResponseProto.Builder builder = DatanodeProtocolProtos.CacheReportResponseProto
                                                                              .NewBuilder();
            if (cmd != null)
            {
                builder.SetCmd(PBHelper.Convert(cmd));
            }
            return((DatanodeProtocolProtos.CacheReportResponseProto)builder.Build());
        }
Ejemplo n.º 5
0
        /// <summary>
        /// This method should handle commands from Standby namenode except
        /// DNA_REGISTER which should be handled earlier itself.
        /// </summary>
        /// <exception cref="System.IO.IOException"/>
        private bool ProcessCommandFromStandby(DatanodeCommand cmd, BPServiceActor actor)
        {
            switch (cmd.GetAction())
            {
            case DatanodeProtocol.DnaAccesskeyupdate:
            {
                Log.Info("DatanodeCommand action from standby: DNA_ACCESSKEYUPDATE");
                if (dn.isBlockTokenEnabled)
                {
                    dn.blockPoolTokenSecretManager.AddKeys(GetBlockPoolId(), ((KeyUpdateCommand)cmd).
                                                           GetExportedKeys());
                }
                break;
            }

            case DatanodeProtocol.DnaTransfer:
            case DatanodeProtocol.DnaInvalidate:
            case DatanodeProtocol.DnaShutdown:
            case DatanodeProtocol.DnaFinalize:
            case DatanodeProtocol.DnaRecoverblock:
            case DatanodeProtocol.DnaBalancerbandwidthupdate:
            case DatanodeProtocol.DnaCache:
            case DatanodeProtocol.DnaUncache:
            {
                Log.Warn("Got a command from standby NN - ignoring command:" + cmd.GetAction());
                break;
            }

            default:
            {
                Log.Warn("Unknown DatanodeCommand action: " + cmd.GetAction());
                break;
            }
            }
            return(true);
        }
Ejemplo n.º 6
0
        /// <summary>
        /// This method should handle all commands from Active namenode except
        /// DNA_REGISTER which should be handled earlier itself.
        /// </summary>
        /// <param name="cmd"/>
        /// <returns>true if further processing may be required or false otherwise.</returns>
        /// <exception cref="System.IO.IOException"/>
        private bool ProcessCommandFromActive(DatanodeCommand cmd, BPServiceActor actor)
        {
            BlockCommand   bcmd       = cmd is BlockCommand ? (BlockCommand)cmd : null;
            BlockIdCommand blockIdCmd = cmd is BlockIdCommand ? (BlockIdCommand)cmd : null;

            switch (cmd.GetAction())
            {
            case DatanodeProtocol.DnaTransfer:
            {
                // Send a copy of a block to another datanode
                dn.TransferBlocks(bcmd.GetBlockPoolId(), bcmd.GetBlocks(), bcmd.GetTargets(), bcmd
                                  .GetTargetStorageTypes());
                dn.metrics.IncrBlocksReplicated(bcmd.GetBlocks().Length);
                break;
            }

            case DatanodeProtocol.DnaInvalidate:
            {
                //
                // Some local block(s) are obsolete and can be
                // safely garbage-collected.
                //
                Block[] toDelete = bcmd.GetBlocks();
                try
                {
                    // using global fsdataset
                    dn.GetFSDataset().Invalidate(bcmd.GetBlockPoolId(), toDelete);
                }
                catch (IOException e)
                {
                    // Exceptions caught here are not expected to be disk-related.
                    throw;
                }
                dn.metrics.IncrBlocksRemoved(toDelete.Length);
                break;
            }

            case DatanodeProtocol.DnaCache:
            {
                Log.Info("DatanodeCommand action: DNA_CACHE for " + blockIdCmd.GetBlockPoolId() +
                         " of [" + BlockIdArrayToString(blockIdCmd.GetBlockIds()) + "]");
                dn.GetFSDataset().Cache(blockIdCmd.GetBlockPoolId(), blockIdCmd.GetBlockIds());
                break;
            }

            case DatanodeProtocol.DnaUncache:
            {
                Log.Info("DatanodeCommand action: DNA_UNCACHE for " + blockIdCmd.GetBlockPoolId()
                         + " of [" + BlockIdArrayToString(blockIdCmd.GetBlockIds()) + "]");
                dn.GetFSDataset().Uncache(blockIdCmd.GetBlockPoolId(), blockIdCmd.GetBlockIds());
                break;
            }

            case DatanodeProtocol.DnaShutdown:
            {
                // TODO: DNA_SHUTDOWN appears to be unused - the NN never sends this command
                // See HDFS-2987.
                throw new NotSupportedException("Received unimplemented DNA_SHUTDOWN");
            }

            case DatanodeProtocol.DnaFinalize:
            {
                string bp = ((FinalizeCommand)cmd).GetBlockPoolId();
                Log.Info("Got finalize command for block pool " + bp);
                System.Diagnostics.Debug.Assert(GetBlockPoolId().Equals(bp), "BP " + GetBlockPoolId
                                                    () + " received DNA_FINALIZE " + "for other block pool " + bp);
                dn.FinalizeUpgradeForPool(bp);
                break;
            }

            case DatanodeProtocol.DnaRecoverblock:
            {
                string who = "NameNode at " + actor.GetNNSocketAddress();
                dn.RecoverBlocks(who, ((BlockRecoveryCommand)cmd).GetRecoveringBlocks());
                break;
            }

            case DatanodeProtocol.DnaAccesskeyupdate:
            {
                Log.Info("DatanodeCommand action: DNA_ACCESSKEYUPDATE");
                if (dn.isBlockTokenEnabled)
                {
                    dn.blockPoolTokenSecretManager.AddKeys(GetBlockPoolId(), ((KeyUpdateCommand)cmd).
                                                           GetExportedKeys());
                }
                break;
            }

            case DatanodeProtocol.DnaBalancerbandwidthupdate:
            {
                Log.Info("DatanodeCommand action: DNA_BALANCERBANDWIDTHUPDATE");
                long bandwidth = ((BalancerBandwidthCommand)cmd).GetBalancerBandwidthValue();
                if (bandwidth > 0)
                {
                    DataXceiverServer dxcs = (DataXceiverServer)dn.dataXceiverServer.GetRunnable();
                    Log.Info("Updating balance throttler bandwidth from " + dxcs.balanceThrottler.GetBandwidth
                                 () + " bytes/s " + "to: " + bandwidth + " bytes/s.");
                    dxcs.balanceThrottler.SetBandwidth(bandwidth);
                }
                break;
            }

            default:
            {
                Log.Warn("Unknown DatanodeCommand action: " + cmd.GetAction());
                break;
            }
            }
            return(true);
        }