// Server side handler of the SendHeartBeat RPC public override Task <DataNodeProto.HeartBeatResponse> SendHeartBeat(DataNodeProto.HeartBeatRequest request, ServerCallContext context) { DataNodeProto.BlockCommand blockCommands = DataNodeManager.Instance.UpdateDataNodes(request.NodeInfo); DataNodeProto.HeartBeatResponse response = new DataNodeProto.HeartBeatResponse { Commands = blockCommands }; return(Task.FromResult(response)); }
/// <summary> /// Send blockreport /// </summary> /// <param name="client">Datanode grpc client</param> public static async Task SendHeartBeat(DataNodeProto.DataNodeProto.DataNodeProtoClient client) { while (true) { DataNodeProto.HeartBeatRequest heartBeatRequest = CreateHeartBeatRequest(); try { DataNodeProto.HeartBeatResponse response = client.SendHeartBeat(heartBeatRequest); DataNodeProto.BlockCommand nameNodeCommands = response.Commands; switch (nameNodeCommands.Action) { case DataNodeProto.BlockCommand.Types.Action.Transfer: foreach (var block in nameNodeCommands.DataBlock.ToList()) { // Get block data byte[] blockData = BlockStorage.Instance.ReadBlock(Guid.Parse(block.BlockId.Value)); if (blockData != null) { Metadata metaData = new Metadata { new Metadata.Entry("blockid", block.BlockId.Value), new Metadata.Entry("blocksize", blockData.Length.ToString()) }; // Send data to each block foreach (var dataNode in block.DataNodes) { Task task = ForwardBlock(dataNode, blockData, metaData); } } } break; case DataNodeProto.BlockCommand.Types.Action.Delete: InvalidateBlocks(nameNodeCommands.BlockList); break; } } catch (RpcException e) { Console.WriteLine("HeartBeat failed: " + e.Message); } await Task.Delay(Constants.HeartBeatInterval); // This is an HDFS default } }
/// <summary> /// Updates the list of Datanodes /// </summary> /// <param name="nodeInfo">A single DataNodes info</param> public DataNodeProto.BlockCommand UpdateDataNodes(DataNodeProto.DataNodeInfo nodeInfo) { // Update list of datanodes int index = FindNodeIndexFromIP(nodeInfo.DataNode.IpAddress); // Not found -> add to list if (index < 0) { DataNode node = new DataNode(nodeInfo.DataNode.IpAddress, nodeInfo.DiskSpace, DateTime.UtcNow); NodeList.Add(node); index = FindNodeIndexFromIP(nodeInfo.DataNode.IpAddress); if (NodeList.Count <= Constants.ReplicationFactor) { BelowReplicationFactorNewDataNodeRedistribute(node.IpAddress); } else { CheckIfRedistributeNeeded(); } } else // Found, update lastHeartBeat timestamp { NodeList[index].DiskSpace = nodeInfo.DiskSpace; NodeList[index].LastHeartBeat = DateTime.UtcNow; } DataNodeProto.BlockCommand returnCommand; if (NodeList[index].Requests.Count > 0) { returnCommand = NodeList[index].Requests[0]; NodeList[index].Requests.Remove(returnCommand); } else { returnCommand = new DataNodeProto.BlockCommand(); } return(returnCommand); }
/// <summary> /// Data block redistribution log. Chooses a DataNode to forward to that does not currently contain the block /// </summary> /// <param name="currentBlock">The data block infomation, which contains list of addresses the block is stored on</param> /// <param name="currentNodeIP">The DataNode's IP Address</param> /// <param name="blockID">Unique ID of block</param> /// <returns>Command for the DataNode to run</returns> public void Redistribute(List <string> currentBlock, string currentNodeIP, Guid blockID) { List <DataNodeProto.DataNode> nodes = new List <DataNodeProto.DataNode>(); for (int i = 0; i < Constants.ReplicationFactor - currentBlock.Count; i++) { string ipAddress = GrabNextDataNode().IpAddress; // While it isn't a node that contains this block, choose the next in the node list while (ipAddress == currentNodeIP || currentBlock.Contains(ipAddress)) { ipAddress = GrabNextDataNode().IpAddress; } Console.WriteLine("Node to be added for redistribution: " + ipAddress + " for blockid = " + blockID.ToString()); nodes.Add( new DataNodeProto.DataNode { IpAddress = ipAddress }); } DataNodeProto.DataBlock dataBlock = new DataNodeProto.DataBlock { BlockId = new DataNodeProto.UUID { Value = blockID.ToString() }, DataNodes = { nodes } }; // Tell the node where to send a copy of the current block DataNodeProto.BlockCommand blockCommand = new DataNodeProto.BlockCommand { Action = DataNodeProto.BlockCommand.Types.Action.Transfer, DataBlock = { dataBlock } }; AddRequestToNode(currentNodeIP, blockCommand); }
/// <summary> /// Adds a request that will be sent to a DataNode to perform /// </summary> /// <param name="ipAddress">Address of DataNode</param> /// <param name="blockCommand">Command to be added</param> public void AddRequestToNode(string ipAddress, DataNodeProto.BlockCommand blockCommand) { int index = FindNodeIndexFromIP(ipAddress); NodeList[index].Requests.Add(blockCommand); }