// Server side handler of the SendHeartBeat RPC
public override Task <DataNodeProto.HeartBeatResponse> SendHeartBeat(DataNodeProto.HeartBeatRequest request, ServerCallContext context)
{
DataNodeProto.BlockCommand blockCommands = DataNodeManager.Instance.UpdateDataNodes(request.NodeInfo);
DataNodeProto.HeartBeatResponse response = new DataNodeProto.HeartBeatResponse
{
Commands = blockCommands
};
return(Task.FromResult(response));
}
/// <summary>
/// Send blockreport
/// </summary>
/// <param name="client">Datanode grpc client</param>
public static async Task SendHeartBeat(DataNodeProto.DataNodeProto.DataNodeProtoClient client)
{
while (true)
{
DataNodeProto.HeartBeatRequest heartBeatRequest = CreateHeartBeatRequest();
try
{
DataNodeProto.HeartBeatResponse response = client.SendHeartBeat(heartBeatRequest);
DataNodeProto.BlockCommand nameNodeCommands = response.Commands;
switch (nameNodeCommands.Action)
{
case DataNodeProto.BlockCommand.Types.Action.Transfer:
foreach (var block in nameNodeCommands.DataBlock.ToList())
{
// Get block data
byte[] blockData = BlockStorage.Instance.ReadBlock(Guid.Parse(block.BlockId.Value));
if (blockData != null)
{
Metadata metaData = new Metadata {
new Metadata.Entry("blockid", block.BlockId.Value),
new Metadata.Entry("blocksize", blockData.Length.ToString())
};
// Send data to each block
foreach (var dataNode in block.DataNodes)
{
Task task = ForwardBlock(dataNode, blockData, metaData);
}
}
}
break;
case DataNodeProto.BlockCommand.Types.Action.Delete:
InvalidateBlocks(nameNodeCommands.BlockList);
break;
}
}
catch (RpcException e)
{
Console.WriteLine("HeartBeat failed: " + e.Message);
}
await Task.Delay(Constants.HeartBeatInterval); // This is an HDFS default
}
}
/// <summary>
/// Updates the list of Datanodes
/// </summary>
/// <param name="nodeInfo">A single DataNodes info</param>
public DataNodeProto.BlockCommand UpdateDataNodes(DataNodeProto.DataNodeInfo nodeInfo)
{
// Update list of datanodes
int index = FindNodeIndexFromIP(nodeInfo.DataNode.IpAddress);
// Not found -> add to list
if (index < 0)
{
DataNode node = new DataNode(nodeInfo.DataNode.IpAddress, nodeInfo.DiskSpace, DateTime.UtcNow);
NodeList.Add(node);
index = FindNodeIndexFromIP(nodeInfo.DataNode.IpAddress);
if (NodeList.Count <= Constants.ReplicationFactor)
{
BelowReplicationFactorNewDataNodeRedistribute(node.IpAddress);
}
else
{
CheckIfRedistributeNeeded();
}
}
else // Found, update lastHeartBeat timestamp
{
NodeList[index].DiskSpace = nodeInfo.DiskSpace;
NodeList[index].LastHeartBeat = DateTime.UtcNow;
}
DataNodeProto.BlockCommand returnCommand;
if (NodeList[index].Requests.Count > 0)
{
returnCommand = NodeList[index].Requests[0];
NodeList[index].Requests.Remove(returnCommand);
}
else
{
returnCommand = new DataNodeProto.BlockCommand();
}
return(returnCommand);
}
/// <summary>
/// Data block redistribution log. Chooses a DataNode to forward to that does not currently contain the block
/// </summary>
/// <param name="currentBlock">The data block infomation, which contains list of addresses the block is stored on</param>
/// <param name="currentNodeIP">The DataNode's IP Address</param>
/// <param name="blockID">Unique ID of block</param>
/// <returns>Command for the DataNode to run</returns>
public void Redistribute(List <string> currentBlock, string currentNodeIP, Guid blockID)
{
List <DataNodeProto.DataNode> nodes = new List <DataNodeProto.DataNode>();
for (int i = 0; i < Constants.ReplicationFactor - currentBlock.Count; i++)
{
string ipAddress = GrabNextDataNode().IpAddress;
// While it isn't a node that contains this block, choose the next in the node list
while (ipAddress == currentNodeIP || currentBlock.Contains(ipAddress))
{
ipAddress = GrabNextDataNode().IpAddress;
}
Console.WriteLine("Node to be added for redistribution: " + ipAddress + " for blockid = " + blockID.ToString());
nodes.Add(
new DataNodeProto.DataNode
{
IpAddress = ipAddress
});
}
DataNodeProto.DataBlock dataBlock = new DataNodeProto.DataBlock
{
BlockId = new DataNodeProto.UUID {
Value = blockID.ToString()
},
DataNodes = { nodes }
};
// Tell the node where to send a copy of the current block
DataNodeProto.BlockCommand blockCommand = new DataNodeProto.BlockCommand
{
Action = DataNodeProto.BlockCommand.Types.Action.Transfer,
DataBlock = { dataBlock }
};
AddRequestToNode(currentNodeIP, blockCommand);
}
/// <summary>
/// Adds a request that will be sent to a DataNode to perform
/// </summary>
/// <param name="ipAddress">Address of DataNode</param>
/// <param name="blockCommand">Command to be added</param>
public void AddRequestToNode(string ipAddress, DataNodeProto.BlockCommand blockCommand)
{
int index = FindNodeIndexFromIP(ipAddress);
NodeList[index].Requests.Add(blockCommand);
}
