/// <summary> /// Creates the pipeline for streaming the block to DataNodes /// </summary> /// <param name="blockInfo">Info of the block. Contains blockid, blocksize, and ipaddresses for pipeline creation</param> /// <param name="client">Client connection with first DataNode in pipe</param> /// <returns>Success of pipeline creation</returns> private bool GetPipeLineReady(ClientProto.BlockInfo blockInfo, out ClientProto.ClientProto.ClientProtoClient client) { ClientProto.StatusResponse readyResponse = new ClientProto.StatusResponse { Type = ClientProto.StatusResponse.Types.StatusType.Fail }; try { Channel channel = new Channel(blockInfo.IpAddress[0] + ":" + "50051", ChannelCredentials.Insecure); blockInfo.IpAddress.RemoveAt(0); client = new ClientProto.ClientProto.ClientProtoClient(channel); readyResponse = client.GetReady(blockInfo); return(true); } catch (Exception e) { #if Debug Console.WriteLine("Get ready failed: " + e.Message); #endif client = null; return(false); } }
/// <summary> /// Writes the block to the first DataNode /// </summary> /// <param name="client">Client Connection to the first DataNode</param> /// <param name="blockInfo">Info of the block. Contains blockid, blocksize, and ipaddresses for pipeline creation</param> /// <param name="block">Data of block</param> /// <returns>Task of the write or Exception on fail</returns> private async Task WriteBlock(ClientProto.ClientProto.ClientProtoClient client, ClientProto.BlockInfo blockInfo, byte[] block, long contentLength) { Metadata metaData = new Metadata { new Metadata.Entry("blockid", blockInfo.BlockId.Value), new Metadata.Entry("blocksize", blockInfo.BlockSize.ToString()) }; int totalBytesRead = 0; using (var call = client.WriteBlock(metaData)) { bool dataNodeFailed = false; while (totalBytesRead < blockInfo.BlockSize) { int length = Math.Min(Constants.ChunkSize, blockInfo.BlockSize - totalBytesRead); // Make sure correct length of data is sent byte[] chunk = new byte[length]; Buffer.BlockCopy(block, totalBytesRead, chunk, 0, length); totalBytesRead += length; TotalBytesWrittenFromFile += length; try { await call.RequestStream.WriteAsync(new ClientProto.BlockData { Data = Google.Protobuf.ByteString.CopyFrom(chunk) }); // Casts are very necessary here! Console.Write("\rWriting File {0}", (((double)TotalBytesWrittenFromFile / (double)contentLength)).ToString("0.00%")); } catch (Exception e) { dataNodeFailed = true; totalBytesRead = blockInfo.BlockSize; // Stop reading #if Debug throw new Exception("Writing block failed", e); #endif } } ClientProto.StatusResponse resp = new ClientProto.StatusResponse { Type = ClientProto.StatusResponse.Types.StatusType.Fail }; if (!dataNodeFailed) { await call.RequestStream.CompleteAsync(); resp = await call.ResponseAsync; } if (resp.Type == ClientProto.StatusResponse.Types.StatusType.Fail) { throw new Exception("Writing block failed"); } } }