/// <exception cref="System.IO.IOException"/>
public virtual HeartbeatResponse SendHeartbeat(DatanodeRegistration registration,
StorageReport[] reports, long cacheCapacity, long cacheUsed, int xmitsInProgress
, int xceiverCount, int failedVolumes, VolumeFailureSummary volumeFailureSummary
)
{
DatanodeProtocolProtos.HeartbeatRequestProto.Builder builder = DatanodeProtocolProtos.HeartbeatRequestProto
.NewBuilder().SetRegistration(PBHelper.Convert(registration)).SetXmitsInProgress
(xmitsInProgress).SetXceiverCount(xceiverCount).SetFailedVolumes(failedVolumes);
builder.AddAllReports(PBHelper.ConvertStorageReports(reports));
if (cacheCapacity != 0)
{
builder.SetCacheCapacity(cacheCapacity);
}
if (cacheUsed != 0)
{
builder.SetCacheUsed(cacheUsed);
}
if (volumeFailureSummary != null)
{
builder.SetVolumeFailureSummary(PBHelper.ConvertVolumeFailureSummary(volumeFailureSummary
));
}
DatanodeProtocolProtos.HeartbeatResponseProto resp;
try
{
resp = rpcProxy.SendHeartbeat(NullController, ((DatanodeProtocolProtos.HeartbeatRequestProto
)builder.Build()));
}
catch (ServiceException se)
{
throw ProtobufHelper.GetRemoteException(se);
}
DatanodeCommand[] cmds = new DatanodeCommand[resp.GetCmdsList().Count];
int index = 0;
foreach (DatanodeProtocolProtos.DatanodeCommandProto p in resp.GetCmdsList())
{
cmds[index] = PBHelper.Convert(p);
index++;
}
RollingUpgradeStatus rollingUpdateStatus = null;
// Use v2 semantics if available.
if (resp.HasRollingUpgradeStatusV2())
{
rollingUpdateStatus = PBHelper.Convert(resp.GetRollingUpgradeStatusV2());
}
else
{
if (resp.HasRollingUpgradeStatus())
{
rollingUpdateStatus = PBHelper.Convert(resp.GetRollingUpgradeStatus());
}
}
return(new HeartbeatResponse(cmds, PBHelper.Convert(resp.GetHaStatus()), rollingUpdateStatus
));
}
/// <exception cref="Com.Google.Protobuf.ServiceException"/>
public virtual DatanodeProtocolProtos.BlockReportResponseProto BlockReport(RpcController
controller, DatanodeProtocolProtos.BlockReportRequestProto request)
{
DatanodeCommand cmd = null;
StorageBlockReport[] report = new StorageBlockReport[request.GetReportsCount()];
int index = 0;
foreach (DatanodeProtocolProtos.StorageBlockReportProto s in request.GetReportsList
())
{
BlockListAsLongs blocks;
if (s.HasNumberOfBlocks())
{
// new style buffer based reports
int num = (int)s.GetNumberOfBlocks();
Preconditions.CheckState(s.GetBlocksCount() == 0, "cannot send both blocks list and buffers"
);
blocks = BlockListAsLongs.DecodeBuffers(num, s.GetBlocksBuffersList());
}
else
{
blocks = BlockListAsLongs.DecodeLongs(s.GetBlocksList());
}
report[index++] = new StorageBlockReport(PBHelper.Convert(s.GetStorage()), blocks
);
}
try
{
cmd = impl.BlockReport(PBHelper.Convert(request.GetRegistration()), request.GetBlockPoolId
(), report, request.HasContext() ? PBHelper.Convert(request.GetContext()) : null
);
}
catch (IOException e)
{
throw new ServiceException(e);
}
DatanodeProtocolProtos.BlockReportResponseProto.Builder builder = DatanodeProtocolProtos.BlockReportResponseProto
.NewBuilder();
if (cmd != null)
{
builder.SetCmd(PBHelper.Convert(cmd));
}
return((DatanodeProtocolProtos.BlockReportResponseProto)builder.Build());
}
/// <exception cref="System.IO.IOException"/>
internal virtual bool ProcessCommandFromActor(DatanodeCommand cmd, BPServiceActor
actor)
{
System.Diagnostics.Debug.Assert(bpServices.Contains(actor));
if (cmd == null)
{
return(true);
}
/*
* Datanode Registration can be done asynchronously here. No need to hold
* the lock. for more info refer HDFS-5014
*/
if (DatanodeProtocol.DnaRegister == cmd.GetAction())
{
// namenode requested a registration - at start or if NN lost contact
// Just logging the claiming state is OK here instead of checking the
// actor state by obtaining the lock
Log.Info("DatanodeCommand action : DNA_REGISTER from " + actor.nnAddr + " with "
+ actor.state + " state");
actor.ReRegister();
return(false);
}
WriteLock();
try
{
if (actor == bpServiceToActive)
{
return(ProcessCommandFromActive(cmd, actor));
}
else
{
return(ProcessCommandFromStandby(cmd, actor));
}
}
finally
{
WriteUnlock();
}
}
/// <exception cref="Com.Google.Protobuf.ServiceException"/>
public virtual DatanodeProtocolProtos.CacheReportResponseProto CacheReport(RpcController
controller, DatanodeProtocolProtos.CacheReportRequestProto request)
{
DatanodeCommand cmd = null;
try
{
cmd = impl.CacheReport(PBHelper.Convert(request.GetRegistration()), request.GetBlockPoolId
(), request.GetBlocksList());
}
catch (IOException e)
{
throw new ServiceException(e);
}
DatanodeProtocolProtos.CacheReportResponseProto.Builder builder = DatanodeProtocolProtos.CacheReportResponseProto
.NewBuilder();
if (cmd != null)
{
builder.SetCmd(PBHelper.Convert(cmd));
}
return((DatanodeProtocolProtos.CacheReportResponseProto)builder.Build());
}
/// <summary>
/// This method should handle commands from Standby namenode except
/// DNA_REGISTER which should be handled earlier itself.
/// </summary>
/// <exception cref="System.IO.IOException"/>
private bool ProcessCommandFromStandby(DatanodeCommand cmd, BPServiceActor actor)
{
switch (cmd.GetAction())
{
case DatanodeProtocol.DnaAccesskeyupdate:
{
Log.Info("DatanodeCommand action from standby: DNA_ACCESSKEYUPDATE");
if (dn.isBlockTokenEnabled)
{
dn.blockPoolTokenSecretManager.AddKeys(GetBlockPoolId(), ((KeyUpdateCommand)cmd).
GetExportedKeys());
}
break;
}
case DatanodeProtocol.DnaTransfer:
case DatanodeProtocol.DnaInvalidate:
case DatanodeProtocol.DnaShutdown:
case DatanodeProtocol.DnaFinalize:
case DatanodeProtocol.DnaRecoverblock:
case DatanodeProtocol.DnaBalancerbandwidthupdate:
case DatanodeProtocol.DnaCache:
case DatanodeProtocol.DnaUncache:
{
Log.Warn("Got a command from standby NN - ignoring command:" + cmd.GetAction());
break;
}
default:
{
Log.Warn("Unknown DatanodeCommand action: " + cmd.GetAction());
break;
}
}
return(true);
}
/// <summary>
/// This method should handle all commands from Active namenode except
/// DNA_REGISTER which should be handled earlier itself.
/// </summary>
/// <param name="cmd"/>
/// <returns>true if further processing may be required or false otherwise.</returns>
/// <exception cref="System.IO.IOException"/>
private bool ProcessCommandFromActive(DatanodeCommand cmd, BPServiceActor actor)
{
BlockCommand bcmd = cmd is BlockCommand ? (BlockCommand)cmd : null;
BlockIdCommand blockIdCmd = cmd is BlockIdCommand ? (BlockIdCommand)cmd : null;
switch (cmd.GetAction())
{
case DatanodeProtocol.DnaTransfer:
{
// Send a copy of a block to another datanode
dn.TransferBlocks(bcmd.GetBlockPoolId(), bcmd.GetBlocks(), bcmd.GetTargets(), bcmd
.GetTargetStorageTypes());
dn.metrics.IncrBlocksReplicated(bcmd.GetBlocks().Length);
break;
}
case DatanodeProtocol.DnaInvalidate:
{
//
// Some local block(s) are obsolete and can be
// safely garbage-collected.
//
Block[] toDelete = bcmd.GetBlocks();
try
{
// using global fsdataset
dn.GetFSDataset().Invalidate(bcmd.GetBlockPoolId(), toDelete);
}
catch (IOException e)
{
// Exceptions caught here are not expected to be disk-related.
throw;
}
dn.metrics.IncrBlocksRemoved(toDelete.Length);
break;
}
case DatanodeProtocol.DnaCache:
{
Log.Info("DatanodeCommand action: DNA_CACHE for " + blockIdCmd.GetBlockPoolId() +
" of [" + BlockIdArrayToString(blockIdCmd.GetBlockIds()) + "]");
dn.GetFSDataset().Cache(blockIdCmd.GetBlockPoolId(), blockIdCmd.GetBlockIds());
break;
}
case DatanodeProtocol.DnaUncache:
{
Log.Info("DatanodeCommand action: DNA_UNCACHE for " + blockIdCmd.GetBlockPoolId()
+ " of [" + BlockIdArrayToString(blockIdCmd.GetBlockIds()) + "]");
dn.GetFSDataset().Uncache(blockIdCmd.GetBlockPoolId(), blockIdCmd.GetBlockIds());
break;
}
case DatanodeProtocol.DnaShutdown:
{
// TODO: DNA_SHUTDOWN appears to be unused - the NN never sends this command
// See HDFS-2987.
throw new NotSupportedException("Received unimplemented DNA_SHUTDOWN");
}
case DatanodeProtocol.DnaFinalize:
{
string bp = ((FinalizeCommand)cmd).GetBlockPoolId();
Log.Info("Got finalize command for block pool " + bp);
System.Diagnostics.Debug.Assert(GetBlockPoolId().Equals(bp), "BP " + GetBlockPoolId
() + " received DNA_FINALIZE " + "for other block pool " + bp);
dn.FinalizeUpgradeForPool(bp);
break;
}
case DatanodeProtocol.DnaRecoverblock:
{
string who = "NameNode at " + actor.GetNNSocketAddress();
dn.RecoverBlocks(who, ((BlockRecoveryCommand)cmd).GetRecoveringBlocks());
break;
}
case DatanodeProtocol.DnaAccesskeyupdate:
{
Log.Info("DatanodeCommand action: DNA_ACCESSKEYUPDATE");
if (dn.isBlockTokenEnabled)
{
dn.blockPoolTokenSecretManager.AddKeys(GetBlockPoolId(), ((KeyUpdateCommand)cmd).
GetExportedKeys());
}
break;
}
case DatanodeProtocol.DnaBalancerbandwidthupdate:
{
Log.Info("DatanodeCommand action: DNA_BALANCERBANDWIDTHUPDATE");
long bandwidth = ((BalancerBandwidthCommand)cmd).GetBalancerBandwidthValue();
if (bandwidth > 0)
{
DataXceiverServer dxcs = (DataXceiverServer)dn.dataXceiverServer.GetRunnable();
Log.Info("Updating balance throttler bandwidth from " + dxcs.balanceThrottler.GetBandwidth
() + " bytes/s " + "to: " + bandwidth + " bytes/s.");
dxcs.balanceThrottler.SetBandwidth(bandwidth);
}
break;
}
default:
{
Log.Warn("Unknown DatanodeCommand action: " + cmd.GetAction());
break;
}
}
return(true);
}
