public override Task <ClientProto.StatusResponse> GetReady(ClientProto.BlockInfo blockInfo, ServerCallContext context) { ClientProto.StatusResponse response = new ClientProto.StatusResponse { Type = ClientProto.StatusResponse.Types.StatusType.Success }; if (blockInfo.IpAddress.Count() == 0) { response.Type = ClientProto.StatusResponse.Types.StatusType.Ready; return(Task.FromResult(response)); } else { Guid blockId = Guid.Parse(blockInfo.BlockId.Value); Channel channel = null; string ipAddress = ""; try { ipAddress = blockInfo.IpAddress[0]; blockInfo.IpAddress.RemoveAt(0); channel = ConnectionManager.Instance.CreateChannel(blockId, ipAddress, Constants.Port.ToString()); var client = new ClientProto.ClientProto.ClientProtoClient(channel); client.GetReady(blockInfo); } catch (RpcException e) { ConnectionManager.Instance.ShutDownChannel(blockId, channel); response.Message = "Failed to get ready for " + ipAddress + ": " + e.Message; } // We will always return ready as if it reaches this far there is at least 1 node in the pipeline, which is good enough! return(Task.FromResult(response)); } }
/// <summary> /// Creates the pipeline for streaming the block to DataNodes /// </summary> /// <param name="blockInfo">Info of the block. Contains blockid, blocksize, and ipaddresses for pipeline creation</param> /// <param name="client">Client connection with first DataNode in pipe</param> /// <returns>Success of pipeline creation</returns> private bool GetPipeLineReady(ClientProto.BlockInfo blockInfo, out ClientProto.ClientProto.ClientProtoClient client) { ClientProto.StatusResponse readyResponse = new ClientProto.StatusResponse { Type = ClientProto.StatusResponse.Types.StatusType.Fail }; try { Channel channel = new Channel(blockInfo.IpAddress[0] + ":" + "50051", ChannelCredentials.Insecure); blockInfo.IpAddress.RemoveAt(0); client = new ClientProto.ClientProto.ClientProtoClient(channel); readyResponse = client.GetReady(blockInfo); return(true); } catch (Exception e) { #if Debug Console.WriteLine("Get ready failed: " + e.Message); #endif client = null; return(false); } }
/// <summary> /// Read a file from SUFS /// </summary> /// <param name="nameNodeClient">nameNodeClient</param> /// <param name="remotePath">Path of the file to be read in SUFS</param> /// <param name="localPath">Path of the file to be stored on client node</param> public static void ReadFile(ClientProto.ClientProto.ClientProtoClient nameNodeClient, string remotePath, string localPath) { var blockMessage = nameNodeClient.ReadFile(new ClientProto.Path { FullPath = remotePath }); FileSize = blockMessage.FileSize; BytesReadFromFile = 0; if (FileSize == 0) { Console.WriteLine("Reading Failed: file size 0. File may not exist."); return; } localPath += "/"; localPath += remotePath; Directory.CreateDirectory(System.IO.Path.GetDirectoryName(localPath)); var writerStream = new FileStream(localPath, FileMode.Append, FileAccess.Write); foreach (var blockInfo in blockMessage.BlockInfo) { WriteToFile(blockInfo, writerStream); } writerStream.Flush(); }
/// <summary> /// Writes the block to the first DataNode /// </summary> /// <param name="client">Client Connection to the first DataNode</param> /// <param name="blockInfo">Info of the block. Contains blockid, blocksize, and ipaddresses for pipeline creation</param> /// <param name="block">Data of block</param> /// <returns>Task of the write or Exception on fail</returns> private async Task WriteBlock(ClientProto.ClientProto.ClientProtoClient client, ClientProto.BlockInfo blockInfo, byte[] block, long contentLength) { Metadata metaData = new Metadata { new Metadata.Entry("blockid", blockInfo.BlockId.Value), new Metadata.Entry("blocksize", blockInfo.BlockSize.ToString()) }; int totalBytesRead = 0; using (var call = client.WriteBlock(metaData)) { bool dataNodeFailed = false; while (totalBytesRead < blockInfo.BlockSize) { int length = Math.Min(Constants.ChunkSize, blockInfo.BlockSize - totalBytesRead); // Make sure correct length of data is sent byte[] chunk = new byte[length]; Buffer.BlockCopy(block, totalBytesRead, chunk, 0, length); totalBytesRead += length; TotalBytesWrittenFromFile += length; try { await call.RequestStream.WriteAsync(new ClientProto.BlockData { Data = Google.Protobuf.ByteString.CopyFrom(chunk) }); // Casts are very necessary here! Console.Write("\rWriting File {0}", (((double)TotalBytesWrittenFromFile / (double)contentLength)).ToString("0.00%")); } catch (Exception e) { dataNodeFailed = true; totalBytesRead = blockInfo.BlockSize; // Stop reading #if Debug throw new Exception("Writing block failed", e); #endif } } ClientProto.StatusResponse resp = new ClientProto.StatusResponse { Type = ClientProto.StatusResponse.Types.StatusType.Fail }; if (!dataNodeFailed) { await call.RequestStream.CompleteAsync(); resp = await call.ResponseAsync; } if (resp.Type == ClientProto.StatusResponse.Types.StatusType.Fail) { throw new Exception("Writing block failed"); } } }
/// <summary> /// Stream version write to file /// </summary> private static void WriteToFile(BlockInfo blockInfo, FileStream writerStream) { bool success = false; foreach (var dataNodeIp in blockInfo.IpAddress) { try { var channel = new Channel(dataNodeIp + ":50051", ChannelCredentials.Insecure); var dataNodeClient = new ClientProto.ClientProto.ClientProtoClient(channel); using (var call = dataNodeClient.ReadBlock(blockInfo.BlockId)) { int copiedLength = 0; while (call.ResponseStream.MoveNext().Result) { var bytes = call.ResponseStream.Current.Data.ToByteArray(); WriteToFile(bytes, writerStream); BytesReadFromFile += bytes.Length; // Casts are very necessary here! Console.Write("\rDownloading {0}", (((double)BytesReadFromFile / (double)FileSize)).ToString("0.00%")); copiedLength += bytes.Length; } if (copiedLength > 0) { success = true; break; } } } catch (Exception exception) { Console.WriteLine($"Error when connecting to data node: {dataNodeIp}. Exception: {exception.Message}. Trying next."); } } if (!success) { throw new Exception($"Cannot get block {blockInfo.BlockId.Value}"); } }
/// <summary> /// Writes block to disk and forwards to next DataNode /// </summary> /// <param name="requestStream">Stream of bytes</param> /// <param name="context">Context of call</param> /// <param name="channel">Channel for pipe</param> /// <param name="filePath">Full path of file</param> /// <param name="blockId">Unique identifier of block</param> /// <returns>Status of writing the block</returns> public async Task <ClientProto.StatusResponse> WriteAndForwardBlock(Grpc.Core.IAsyncStreamReader <ClientProto.BlockData> requestStream, ServerCallContext context, Channel channel, string filePath, Guid blockId, int blockSize) { bool success = true; string message = ""; var client = new ClientProto.ClientProto.ClientProtoClient(channel); using (var call = client.WriteBlock(context.RequestHeaders)) { bool dataNodeFailed = false; using (var stream = new FileStream(filePath, FileMode.Append, FileAccess.Write, FileShare.None, 131072, FileOptions.WriteThrough)) // 128KB { while (await requestStream.MoveNext()) { try { var blockData = requestStream.Current; byte[] data = blockData.Data.ToByteArray(); // Write to file stream.Write(data, 0, data.Length); // Don't need to forward data in pipe if datanode failed if (!dataNodeFailed) { try { // Send data through pipe await call.RequestStream.WriteAsync(blockData); } catch (RpcException e) { dataNodeFailed = true; Console.WriteLine("Writing block failed: " + e.Message); message = e.Message; } } } catch (IOException e) { dataNodeFailed = true; message = e.Message; success = false; } } stream.Flush(); } ClientProto.StatusResponse resp = new ClientProto.StatusResponse { Type = ClientProto.StatusResponse.Types.StatusType.Fail, Message = message }; // If DataNode did not fail, get response and shut down if (!dataNodeFailed) { await call.RequestStream.CompleteAsync(); resp = await call.ResponseAsync; ConnectionManager.Instance.ShutDownChannel(blockId, channel); } // If write was successful and block size is correct, return success // Otherwise return the response sent down through pipe return((success && BlockStorage.Instance.ValidateBlock(blockId, filePath, blockSize)) ? new ClientProto.StatusResponse { Type = ClientProto.StatusResponse.Types.StatusType.Success } : resp); } }
public FileCreater(ClientProto.ClientProto.ClientProtoClient nameNodeClient) { client = nameNodeClient; TotalBytesWrittenFromFile = 0; }
public static void Main(string[] args) { if (args.Length < 1) { System.Console.WriteLine("Usage: Client <namenode ip>"); return; } string ip = args[0]; // get connected to server Channel channel = new Channel(ip + ":50051", ChannelCredentials.Insecure); var client = new ClientProto.ClientProto.ClientProtoClient(channel); while (true) { // Ask user to enter action to perform Console.WriteLine("Enter an action to perform (help for more information, quit to end)?"); var line = Console.ReadLine(); var lineData = line.Split(" "); if (lineData.Length == 0) { Console.WriteLine("Enter an action to perform (help for more information, quit to end)?"); } string action = lineData[0]; // Quit if (line == "quit") { channel.ShutdownAsync().Wait(); return; } else if (line == "help") { Console.WriteLine("Possible Actions: "); Console.WriteLine("Create a file: createfile <full path of file to be created> <location of file> <local/s3>"); Console.WriteLine("Read a file: readfile <file path on SUFS>"); Console.WriteLine("Delete a file: deletefile <full path of file on SUFS>"); Console.WriteLine("Create a directory: createdirectory <full path of directory on SUFS>"); Console.WriteLine("Delete a directory: deletedirectory <full path of directory on SUFS>"); Console.WriteLine("Move a file: movefile <full path of file on SUFS> <new full path>"); Console.WriteLine("List a files blocks and their locations: listnodes <full path of file on SUFS>"); Console.WriteLine("List contents of a directory: listcontents <full path of directory on SUFS>"); Console.WriteLine("Quit program: Quit"); Console.WriteLine(); } else if (action.ToLower() == "deletedirectory") { if (lineData.Length != 2) { Console.WriteLine("Invalid arguments, input must be 'DeleteDirectory <full path of directory on SUFS>'"); Console.WriteLine(); continue; } string path = lineData[1]; var reply = client.DeleteDirectory(new ClientProto.Path { FullPath = path }); Console.WriteLine("Delete directory: " + reply.Type.ToString() + " " + reply.Message); } else if (action.ToLower() == "createdirectory") { if (lineData.Length != 2) { Console.WriteLine("Invalid arguments, input must be 'CreateDirectory <full path of directory on SUFS>'"); Console.WriteLine(); continue; } string path = lineData[1]; var reply = client.CreateDirectory(new ClientProto.Path { FullPath = path }); Console.WriteLine("Create directory: " + reply.Type.ToString()); } else if (action.ToLower() == "createfile") { if (lineData.Length != 4) { Console.WriteLine("Invalid arguments, input must be 'CreateFile <full path of file to be created> <location of file> <local/s3>'"); Console.WriteLine(); continue; } try { string path = lineData[1]; FileCreater fileCreater = new FileCreater(client); fileCreater.CreateFile(path, lineData[2], lineData[3]); } catch (Exception e) { Console.WriteLine("Exception while creating file: ", e); } } else if (action.ToLower() == "deletefile") { if (lineData.Length != 2) { Console.WriteLine("Invalid arguments, input must be 'DeleteFile <full path of file on SUFS>'"); Console.WriteLine(); continue; } string path = lineData[1]; var reply = client.DeleteFile(new ClientProto.Path { FullPath = path }); Console.WriteLine("Delete file action: " + reply.Type.ToString() + " " + reply.Message); } else if (action.ToLower() == "readfile") { if (lineData.Length != 2) { Console.WriteLine("Invalid arguments, input must be 'ReadFile <file path on SUFS>'"); Console.WriteLine(); continue; } string path = lineData[1]; var localPath = Directory.GetCurrentDirectory(); FileReader.ReadFile(client, path, localPath); Console.WriteLine(); Console.WriteLine("Done reading file"); } else if (action.ToLower() == "movefile") { if (lineData.Length != 3) { Console.WriteLine("Invalid arguments, input must be 'MoveFile <full path of file on SUFS> <new full path>'"); Console.WriteLine(); continue; } string path = lineData[1]; string newpath = lineData[2]; var reply = client.MoveFile(new ClientProto.DoublePath { Fullpath = path, Newpath = newpath }); Console.WriteLine("Move file action: " + reply.Type.ToString()); } else if (action.ToLower() == "listnodes") { if (lineData.Length != 2) { Console.WriteLine("Invalid arguments, input must be 'ListNodes <full path of file on SUFS>'"); Console.WriteLine(); continue; } string path = lineData[1]; var reply = client.ListNodes(new ClientProto.Path { FullPath = path }); if (reply.Type == ClientProto.ListOfNodesList.Types.StatusType.FileDoesNotExist) { Console.WriteLine("File does not exist."); } else { PrettyPrintNodeList(reply); } } else if (action.ToLower() == "listcontents") { if (lineData.Length != 2) { Console.WriteLine("Invalid arguments, input must be 'ListContents <full path of directory on SUFS>'"); Console.WriteLine(); continue; } string path = lineData[1]; var reply = client.ListContents(new ClientProto.Path { FullPath = path }); if (reply.Type == ClientProto.ListOfContents.Types.StatusType.DirectoryDoesNotExist) { Console.WriteLine("Directory does not exist."); } else { Console.WriteLine("List of directory contents:"); foreach (var s in reply.FileName) { Console.WriteLine(s); } Console.WriteLine(); } } else { Console.WriteLine("Not a valid action."); } } }