Пример #1
0
        public override Task <ClientProto.StatusResponse> GetReady(ClientProto.BlockInfo blockInfo, ServerCallContext context)
        {
            ClientProto.StatusResponse response = new ClientProto.StatusResponse {
                Type = ClientProto.StatusResponse.Types.StatusType.Success
            };
            if (blockInfo.IpAddress.Count() == 0)
            {
                response.Type = ClientProto.StatusResponse.Types.StatusType.Ready;
                return(Task.FromResult(response));
            }
            else
            {
                Guid    blockId   = Guid.Parse(blockInfo.BlockId.Value);
                Channel channel   = null;
                string  ipAddress = "";
                try
                {
                    ipAddress = blockInfo.IpAddress[0];
                    blockInfo.IpAddress.RemoveAt(0);
                    channel = ConnectionManager.Instance.CreateChannel(blockId, ipAddress, Constants.Port.ToString());
                    var client = new ClientProto.ClientProto.ClientProtoClient(channel);

                    client.GetReady(blockInfo);
                }
                catch (RpcException e)
                {
                    ConnectionManager.Instance.ShutDownChannel(blockId, channel);
                    response.Message = "Failed to get ready for " + ipAddress + ": " + e.Message;
                }

                // We will always return ready as if it reaches this far there is at least 1 node in the pipeline, which is good enough!
                return(Task.FromResult(response));
            }
        }
Пример #2
0
        /// <summary>
        /// Server side handler of the WriteDataBlock RPC. DataNode writes block to a file and then returns success status of write.
        /// </summary>
        /// <param name="requestStream">Stream of bytes</param>
        /// <param name="context">Context of call. Contains metadata containing blockid and list of addresses</param>
        /// <returns>Status of task</returns>
        public override async Task <ClientProto.StatusResponse> WriteBlock(Grpc.Core.IAsyncStreamReader <ClientProto.BlockData> requestStream, ServerCallContext context)
        {
            List <Metadata.Entry> metaData = context.RequestHeaders.ToList();

            // Get blockID
            Guid blockId   = Util.GetBlockID(metaData);
            int  blockSize = Util.GetBlockSize(metaData);

            string filePath = BlockStorage.Instance.CreateFile(blockId);

            Channel channel = ConnectionManager.Instance.GetChannel(blockId);

            ClientProto.StatusResponse response = new ClientProto.StatusResponse {
                Type = ClientProto.StatusResponse.Types.StatusType.Success
            };
            // No channel found means last datanode in pipe
            if (channel != null)
            {
                response = await WriteAndForwardBlock(requestStream, context, channel, filePath, blockId, blockSize);
            }
            else // Just write to file
            {
                response = await WriteBlock(requestStream, filePath, blockId, blockSize);
            }

            if (response.Type == ClientProto.StatusResponse.Types.StatusType.Success)
            {
                Console.WriteLine("Done writing block: " + blockId.ToString());
                // Send block report to NameNode
                var client = new DataNodeProto.DataNodeProto.DataNodeProtoClient(ConnectionManager.Instance.NameNodeConnection);
                BlockReport.SendSingleBlockReport(client);
            }

            return(response);
        }
Пример #3
0
        /// <summary>
        /// Writes block to disk and forwards to next DataNode
        /// </summary>
        /// <param name="requestStream">Stream of bytes</param>
        /// <param name="context">Context of call</param>
        /// <param name="channel">Channel for pipe</param>
        /// <param name="filePath">Full path of file</param>
        /// <param name="blockId">Unique identifier of block</param>
        /// <returns>Status of writing the block</returns>
        public async Task <ClientProto.StatusResponse> WriteAndForwardBlock(Grpc.Core.IAsyncStreamReader <ClientProto.BlockData> requestStream, ServerCallContext context, Channel channel, string filePath, Guid blockId, int blockSize)
        {
            bool   success = true;
            string message = "";
            var    client  = new ClientProto.ClientProto.ClientProtoClient(channel);

            using (var call = client.WriteBlock(context.RequestHeaders))
            {
                bool dataNodeFailed = false;

                using (var stream = new FileStream(filePath, FileMode.Append, FileAccess.Write, FileShare.None, 131072, FileOptions.WriteThrough)) // 128KB
                {
                    while (await requestStream.MoveNext())
                    {
                        try
                        {
                            var    blockData = requestStream.Current;
                            byte[] data      = blockData.Data.ToByteArray();

                            // Write to file
                            stream.Write(data, 0, data.Length);

                            // Don't need to forward data in pipe if datanode failed
                            if (!dataNodeFailed)
                            {
                                try
                                {
                                    // Send data through pipe
                                    await call.RequestStream.WriteAsync(blockData);
                                }
                                catch (RpcException e)
                                {
                                    dataNodeFailed = true;
                                    Console.WriteLine("Writing block failed: " + e.Message);
                                    message = e.Message;
                                }
                            }
                        }
                        catch (IOException e)
                        {
                            dataNodeFailed = true;
                            message        = e.Message;
                            success        = false;
                        }
                    }
                    stream.Flush();
                }

                ClientProto.StatusResponse resp = new ClientProto.StatusResponse {
                    Type = ClientProto.StatusResponse.Types.StatusType.Fail, Message = message
                };

                // If DataNode did not fail, get response and shut down
                if (!dataNodeFailed)
                {
                    await call.RequestStream.CompleteAsync();

                    resp = await call.ResponseAsync;
                    ConnectionManager.Instance.ShutDownChannel(blockId, channel);
                }

                // If write was successful and block size is correct, return success
                // Otherwise return the response sent down through pipe
                return((success && BlockStorage.Instance.ValidateBlock(blockId, filePath, blockSize)) ? new ClientProto.StatusResponse {
                    Type = ClientProto.StatusResponse.Types.StatusType.Success
                } : resp);
            }
        }