/// <exception cref="System.IO.IOException"/> public virtual HeartbeatResponse SendHeartbeat(DatanodeRegistration registration, StorageReport[] reports, long cacheCapacity, long cacheUsed, int xmitsInProgress , int xceiverCount, int failedVolumes, VolumeFailureSummary volumeFailureSummary ) { DatanodeProtocolProtos.HeartbeatRequestProto.Builder builder = DatanodeProtocolProtos.HeartbeatRequestProto .NewBuilder().SetRegistration(PBHelper.Convert(registration)).SetXmitsInProgress (xmitsInProgress).SetXceiverCount(xceiverCount).SetFailedVolumes(failedVolumes); builder.AddAllReports(PBHelper.ConvertStorageReports(reports)); if (cacheCapacity != 0) { builder.SetCacheCapacity(cacheCapacity); } if (cacheUsed != 0) { builder.SetCacheUsed(cacheUsed); } if (volumeFailureSummary != null) { builder.SetVolumeFailureSummary(PBHelper.ConvertVolumeFailureSummary(volumeFailureSummary )); } DatanodeProtocolProtos.HeartbeatResponseProto resp; try { resp = rpcProxy.SendHeartbeat(NullController, ((DatanodeProtocolProtos.HeartbeatRequestProto )builder.Build())); } catch (ServiceException se) { throw ProtobufHelper.GetRemoteException(se); } DatanodeCommand[] cmds = new DatanodeCommand[resp.GetCmdsList().Count]; int index = 0; foreach (DatanodeProtocolProtos.DatanodeCommandProto p in resp.GetCmdsList()) { cmds[index] = PBHelper.Convert(p); index++; } RollingUpgradeStatus rollingUpdateStatus = null; // Use v2 semantics if available. if (resp.HasRollingUpgradeStatusV2()) { rollingUpdateStatus = PBHelper.Convert(resp.GetRollingUpgradeStatusV2()); } else { if (resp.HasRollingUpgradeStatus()) { rollingUpdateStatus = PBHelper.Convert(resp.GetRollingUpgradeStatus()); } } return(new HeartbeatResponse(cmds, PBHelper.Convert(resp.GetHaStatus()), rollingUpdateStatus )); }
/// <exception cref="Com.Google.Protobuf.ServiceException"/> public virtual DatanodeProtocolProtos.BlockReportResponseProto BlockReport(RpcController controller, DatanodeProtocolProtos.BlockReportRequestProto request) { DatanodeCommand cmd = null; StorageBlockReport[] report = new StorageBlockReport[request.GetReportsCount()]; int index = 0; foreach (DatanodeProtocolProtos.StorageBlockReportProto s in request.GetReportsList ()) { BlockListAsLongs blocks; if (s.HasNumberOfBlocks()) { // new style buffer based reports int num = (int)s.GetNumberOfBlocks(); Preconditions.CheckState(s.GetBlocksCount() == 0, "cannot send both blocks list and buffers" ); blocks = BlockListAsLongs.DecodeBuffers(num, s.GetBlocksBuffersList()); } else { blocks = BlockListAsLongs.DecodeLongs(s.GetBlocksList()); } report[index++] = new StorageBlockReport(PBHelper.Convert(s.GetStorage()), blocks ); } try { cmd = impl.BlockReport(PBHelper.Convert(request.GetRegistration()), request.GetBlockPoolId (), report, request.HasContext() ? PBHelper.Convert(request.GetContext()) : null ); } catch (IOException e) { throw new ServiceException(e); } DatanodeProtocolProtos.BlockReportResponseProto.Builder builder = DatanodeProtocolProtos.BlockReportResponseProto .NewBuilder(); if (cmd != null) { builder.SetCmd(PBHelper.Convert(cmd)); } return((DatanodeProtocolProtos.BlockReportResponseProto)builder.Build()); }
/// <exception cref="System.IO.IOException"/> internal virtual bool ProcessCommandFromActor(DatanodeCommand cmd, BPServiceActor actor) { System.Diagnostics.Debug.Assert(bpServices.Contains(actor)); if (cmd == null) { return(true); } /* * Datanode Registration can be done asynchronously here. No need to hold * the lock. for more info refer HDFS-5014 */ if (DatanodeProtocol.DnaRegister == cmd.GetAction()) { // namenode requested a registration - at start or if NN lost contact // Just logging the claiming state is OK here instead of checking the // actor state by obtaining the lock Log.Info("DatanodeCommand action : DNA_REGISTER from " + actor.nnAddr + " with " + actor.state + " state"); actor.ReRegister(); return(false); } WriteLock(); try { if (actor == bpServiceToActive) { return(ProcessCommandFromActive(cmd, actor)); } else { return(ProcessCommandFromStandby(cmd, actor)); } } finally { WriteUnlock(); } }
/// <exception cref="Com.Google.Protobuf.ServiceException"/> public virtual DatanodeProtocolProtos.CacheReportResponseProto CacheReport(RpcController controller, DatanodeProtocolProtos.CacheReportRequestProto request) { DatanodeCommand cmd = null; try { cmd = impl.CacheReport(PBHelper.Convert(request.GetRegistration()), request.GetBlockPoolId (), request.GetBlocksList()); } catch (IOException e) { throw new ServiceException(e); } DatanodeProtocolProtos.CacheReportResponseProto.Builder builder = DatanodeProtocolProtos.CacheReportResponseProto .NewBuilder(); if (cmd != null) { builder.SetCmd(PBHelper.Convert(cmd)); } return((DatanodeProtocolProtos.CacheReportResponseProto)builder.Build()); }
/// <summary> /// This method should handle commands from Standby namenode except /// DNA_REGISTER which should be handled earlier itself. /// </summary> /// <exception cref="System.IO.IOException"/> private bool ProcessCommandFromStandby(DatanodeCommand cmd, BPServiceActor actor) { switch (cmd.GetAction()) { case DatanodeProtocol.DnaAccesskeyupdate: { Log.Info("DatanodeCommand action from standby: DNA_ACCESSKEYUPDATE"); if (dn.isBlockTokenEnabled) { dn.blockPoolTokenSecretManager.AddKeys(GetBlockPoolId(), ((KeyUpdateCommand)cmd). GetExportedKeys()); } break; } case DatanodeProtocol.DnaTransfer: case DatanodeProtocol.DnaInvalidate: case DatanodeProtocol.DnaShutdown: case DatanodeProtocol.DnaFinalize: case DatanodeProtocol.DnaRecoverblock: case DatanodeProtocol.DnaBalancerbandwidthupdate: case DatanodeProtocol.DnaCache: case DatanodeProtocol.DnaUncache: { Log.Warn("Got a command from standby NN - ignoring command:" + cmd.GetAction()); break; } default: { Log.Warn("Unknown DatanodeCommand action: " + cmd.GetAction()); break; } } return(true); }
/// <summary> /// This method should handle all commands from Active namenode except /// DNA_REGISTER which should be handled earlier itself. /// </summary> /// <param name="cmd"/> /// <returns>true if further processing may be required or false otherwise.</returns> /// <exception cref="System.IO.IOException"/> private bool ProcessCommandFromActive(DatanodeCommand cmd, BPServiceActor actor) { BlockCommand bcmd = cmd is BlockCommand ? (BlockCommand)cmd : null; BlockIdCommand blockIdCmd = cmd is BlockIdCommand ? (BlockIdCommand)cmd : null; switch (cmd.GetAction()) { case DatanodeProtocol.DnaTransfer: { // Send a copy of a block to another datanode dn.TransferBlocks(bcmd.GetBlockPoolId(), bcmd.GetBlocks(), bcmd.GetTargets(), bcmd .GetTargetStorageTypes()); dn.metrics.IncrBlocksReplicated(bcmd.GetBlocks().Length); break; } case DatanodeProtocol.DnaInvalidate: { // // Some local block(s) are obsolete and can be // safely garbage-collected. // Block[] toDelete = bcmd.GetBlocks(); try { // using global fsdataset dn.GetFSDataset().Invalidate(bcmd.GetBlockPoolId(), toDelete); } catch (IOException e) { // Exceptions caught here are not expected to be disk-related. throw; } dn.metrics.IncrBlocksRemoved(toDelete.Length); break; } case DatanodeProtocol.DnaCache: { Log.Info("DatanodeCommand action: DNA_CACHE for " + blockIdCmd.GetBlockPoolId() + " of [" + BlockIdArrayToString(blockIdCmd.GetBlockIds()) + "]"); dn.GetFSDataset().Cache(blockIdCmd.GetBlockPoolId(), blockIdCmd.GetBlockIds()); break; } case DatanodeProtocol.DnaUncache: { Log.Info("DatanodeCommand action: DNA_UNCACHE for " + blockIdCmd.GetBlockPoolId() + " of [" + BlockIdArrayToString(blockIdCmd.GetBlockIds()) + "]"); dn.GetFSDataset().Uncache(blockIdCmd.GetBlockPoolId(), blockIdCmd.GetBlockIds()); break; } case DatanodeProtocol.DnaShutdown: { // TODO: DNA_SHUTDOWN appears to be unused - the NN never sends this command // See HDFS-2987. throw new NotSupportedException("Received unimplemented DNA_SHUTDOWN"); } case DatanodeProtocol.DnaFinalize: { string bp = ((FinalizeCommand)cmd).GetBlockPoolId(); Log.Info("Got finalize command for block pool " + bp); System.Diagnostics.Debug.Assert(GetBlockPoolId().Equals(bp), "BP " + GetBlockPoolId () + " received DNA_FINALIZE " + "for other block pool " + bp); dn.FinalizeUpgradeForPool(bp); break; } case DatanodeProtocol.DnaRecoverblock: { string who = "NameNode at " + actor.GetNNSocketAddress(); dn.RecoverBlocks(who, ((BlockRecoveryCommand)cmd).GetRecoveringBlocks()); break; } case DatanodeProtocol.DnaAccesskeyupdate: { Log.Info("DatanodeCommand action: DNA_ACCESSKEYUPDATE"); if (dn.isBlockTokenEnabled) { dn.blockPoolTokenSecretManager.AddKeys(GetBlockPoolId(), ((KeyUpdateCommand)cmd). GetExportedKeys()); } break; } case DatanodeProtocol.DnaBalancerbandwidthupdate: { Log.Info("DatanodeCommand action: DNA_BALANCERBANDWIDTHUPDATE"); long bandwidth = ((BalancerBandwidthCommand)cmd).GetBalancerBandwidthValue(); if (bandwidth > 0) { DataXceiverServer dxcs = (DataXceiverServer)dn.dataXceiverServer.GetRunnable(); Log.Info("Updating balance throttler bandwidth from " + dxcs.balanceThrottler.GetBandwidth () + " bytes/s " + "to: " + bandwidth + " bytes/s."); dxcs.balanceThrottler.SetBandwidth(bandwidth); } break; } default: { Log.Warn("Unknown DatanodeCommand action: " + cmd.GetAction()); break; } } return(true); }