예제 #1
0
        /// <summary>
        /// Server side handler of the WriteDataBlock RPC. DataNode writes block to a file and then returns success status of write.
        /// </summary>
        /// <param name="requestStream">Stream of bytes</param>
        /// <param name="context">Context of call. Contains metadata containing blockid and list of addresses</param>
        /// <returns>Status of task</returns>
        public override async Task <ClientProto.StatusResponse> WriteBlock(Grpc.Core.IAsyncStreamReader <ClientProto.BlockData> requestStream, ServerCallContext context)
        {
            List <Metadata.Entry> metaData = context.RequestHeaders.ToList();

            // Get blockID
            Guid blockId   = Util.GetBlockID(metaData);
            int  blockSize = Util.GetBlockSize(metaData);

            string filePath = BlockStorage.Instance.CreateFile(blockId);

            Channel channel = ConnectionManager.Instance.GetChannel(blockId);

            ClientProto.StatusResponse response = new ClientProto.StatusResponse {
                Type = ClientProto.StatusResponse.Types.StatusType.Success
            };
            // No channel found means last datanode in pipe
            if (channel != null)
            {
                response = await WriteAndForwardBlock(requestStream, context, channel, filePath, blockId, blockSize);
            }
            else // Just write to file
            {
                response = await WriteBlock(requestStream, filePath, blockId, blockSize);
            }

            if (response.Type == ClientProto.StatusResponse.Types.StatusType.Success)
            {
                Console.WriteLine("Done writing block: " + blockId.ToString());
                // Send block report to NameNode
                var client = new DataNodeProto.DataNodeProto.DataNodeProtoClient(ConnectionManager.Instance.NameNodeConnection);
                BlockReport.SendSingleBlockReport(client);
            }

            return(response);
        }
예제 #2
0
        /// <summary>
        /// Server side handler of the WriteDataBlock RPC. DataNode writes block to a file and then returns success status of write.
        /// </summary>
        /// <param name="requestStream">Data of block</param>
        /// <param name="context">Call Context</param>
        /// <returns>Status of task</returns>
        public override async Task <DataNodeProto.StatusResponse> WriteDataBlock(Grpc.Core.IAsyncStreamReader <DataNodeProto.BlockData> requestStream, ServerCallContext context)
        {
            List <Metadata.Entry> metaData = context.RequestHeaders.ToList();
            Guid blockId   = Util.GetBlockID(metaData);
            int  blockSize = Util.GetBlockSize(metaData);

            string filePath = BlockStorage.Instance.CreateFile(blockId);

            bool writeSuccess = true;

            try
            {
                // Write block to file system
                using (var writerStream = new FileStream(filePath, FileMode.Append, FileAccess.Write))
                {
                    while (await requestStream.MoveNext())
                    {
                        try
                        {
                            var bytes = requestStream.Current.Data.ToByteArray();
                            writerStream.Write(bytes, 0, bytes.Length);
                        }
                        catch
                        {
                            writeSuccess = false;
                        }
                    }
                }
            }
            catch (Exception e)
            {
                Console.WriteLine("Writing block that was forwarded failed: " + e.Message);
            }

            if (BlockStorage.Instance.ValidateBlock(blockId, filePath, blockSize) && writeSuccess)
            {
                // Send block report to NameNode
                var client = new DataNodeProto.DataNodeProto.DataNodeProtoClient(ConnectionManager.Instance.NameNodeConnection);
                BlockReport.SendSingleBlockReport(client);
                return(new DataNodeProto.StatusResponse {
                    Type = DataNodeProto.StatusResponse.Types.StatusType.Success
                });
            }
            else
            {
                return(new DataNodeProto.StatusResponse {
                    Type = DataNodeProto.StatusResponse.Types.StatusType.Fail
                });
            }
        }
예제 #3
0
        static void Main(string[] args)
        {
            Console.WriteLine("Initializing DataNode");
            string nameNodeIp = args[0];

            int port = Convert.ToInt32(args[1]);

            // Use ec2 instance manager to get the private ip address of this data node
            EC2InstanceManager.InstanceManager instanceManager = EC2InstanceManager.InstanceManager.Instance;
            ipAddress = instanceManager.GetPrivateIpAddress();

#if DEBUG
            if (ipAddress == null)
            {
                ipAddress = "localhost";
            }
#endif

            Server server = new Server
            {
                Services = { DataNodeProto.DataNodeProto.BindService(new DataNodeHandler()),
                             ClientProto.ClientProto.BindService(new ClientHandler()) },
                //Ports = { new ServerPort(ipAddress, Constants.Port, ServerCredentials.Insecure) }
                Ports = { new ServerPort(ipAddress, port, ServerCredentials.Insecure) }
            };

            Console.WriteLine("Done initializing");

            server.Start();

            Console.WriteLine("Trying to connect to: " + nameNodeIp);
            Channel channel = new Channel(nameNodeIp + ":" + Constants.Port, ChannelCredentials.Insecure);
            ConnectionManager.Instance.NameNodeConnection = channel;
            var client = new DataNodeProto.DataNodeProto.DataNodeProtoClient(channel);

            // Send initial heartbeat to initialize datanode in namenode
            var response = client.SendHeartBeat(HeartBeat.CreateHeartBeatRequest(), new CallOptions().WithWaitForReady(true));
            Console.WriteLine("Successfully connected to namenode");

            // Initialize blockstorage
            BlockStorage mBlockStorage = BlockStorage.Instance;

            Task heartBeatTask   = HeartBeat.SendHeartBeat(client);
            Task blockReportTask = BlockReport.SendBlockReport(client);
            while (true)
            {
            }
        }