/// <exception cref="System.Exception"/> public virtual void TestWrite() { HdfsFileStatus status = nn.GetRpcServer().GetFileInfo("/tmp/bar"); long dirId = status.GetFileId(); FileHandle handle = new FileHandle(dirId); byte[] buffer = new byte[10]; for (int i = 0; i < 10; i++) { buffer[i] = unchecked ((byte)i); } WRITE3Request writeReq = new WRITE3Request(handle, 0, 10, Nfs3Constant.WriteStableHow .DataSync, ByteBuffer.Wrap(buffer)); XDR xdr_req = new XDR(); writeReq.Serialize(xdr_req); // Attempt by an unpriviledged user should fail. WRITE3Response response1 = nfsd.Write(xdr_req.AsReadOnlyWrap(), null, 1, securityHandlerUnpriviledged , new IPEndPoint("localhost", 1234)); NUnit.Framework.Assert.AreEqual("Incorrect return code:", Nfs3Status.Nfs3errAcces , response1.GetStatus()); // Attempt by a priviledged user should pass. WRITE3Response response2 = nfsd.Write(xdr_req.AsReadOnlyWrap(), null, 1, securityHandler , new IPEndPoint("localhost", 1234)); NUnit.Framework.Assert.AreEqual("Incorrect response:", null, response2); }
internal static XDR Write(FileHandle handle, int xid, long offset, int count, byte [] data) { XDR request = new XDR(); RpcCall.GetInstance(xid, Nfs3Constant.Program, Nfs3Constant.Version, Nfs3Constant.NFSPROC3 .Create.GetValue(), new CredentialsNone(), new VerifierNone()).Write(request); WRITE3Request write1 = new WRITE3Request(handle, offset, count, Nfs3Constant.WriteStableHow .Unstable, ByteBuffer.Wrap(data)); write1.Serialize(request); return(request); }
/// <exception cref="System.Exception"/> private void CreateFileUsingNfs(string fileName, byte[] buffer) { DFSTestUtil.CreateFile(hdfs, new Path(fileName), 0, (short)1, 0); HdfsFileStatus status = nn.GetRpcServer().GetFileInfo(fileName); long dirId = status.GetFileId(); FileHandle handle = new FileHandle(dirId); WRITE3Request writeReq = new WRITE3Request(handle, 0, buffer.Length, Nfs3Constant.WriteStableHow .DataSync, ByteBuffer.Wrap(buffer)); XDR xdr_req = new XDR(); writeReq.Serialize(xdr_req); WRITE3Response response = nfsd.Write(xdr_req.AsReadOnlyWrap(), null, 1, securityHandler , new IPEndPoint("localhost", 1234)); NUnit.Framework.Assert.AreEqual("Incorrect response: ", null, response); }
/// <exception cref="System.IO.IOException"/> internal virtual void HandleWrite(DFSClient dfsClient, WRITE3Request request, Org.Jboss.Netty.Channel.Channel channel, int xid, Nfs3FileAttributes preOpAttr) { int count = request.GetCount(); byte[] data = ((byte[])request.GetData().Array()); if (data.Length < count) { WRITE3Response response = new WRITE3Response(Nfs3Status.Nfs3errInval); Nfs3Utils.WriteChannel(channel, response.Serialize(new XDR(), xid, new VerifierNone ()), xid); return; } FileHandle handle = request.GetHandle(); if (Log.IsDebugEnabled()) { Log.Debug("handleWrite " + request); } // Check if there is a stream to write FileHandle fileHandle = request.GetHandle(); OpenFileCtx openFileCtx = fileContextCache.Get(fileHandle); if (openFileCtx == null) { Log.Info("No opened stream for fileId: " + fileHandle.GetFileId()); string fileIdPath = Nfs3Utils.GetFileIdPath(fileHandle.GetFileId()); HdfsDataOutputStream fos = null; Nfs3FileAttributes latestAttr = null; try { int bufferSize = config.GetInt(CommonConfigurationKeysPublic.IoFileBufferSizeKey, CommonConfigurationKeysPublic.IoFileBufferSizeDefault); fos = dfsClient.Append(fileIdPath, bufferSize, EnumSet.Of(CreateFlag.Append), null , null); latestAttr = Nfs3Utils.GetFileAttr(dfsClient, fileIdPath, iug); } catch (RemoteException e) { IOException io = e.UnwrapRemoteException(); if (io is AlreadyBeingCreatedException) { Log.Warn("Can't append file: " + fileIdPath + ". Possibly the file is being closed. Drop the request: " + request + ", wait for the client to retry..."); return; } throw; } catch (IOException e) { Log.Error("Can't append to file: " + fileIdPath, e); if (fos != null) { fos.Close(); } WccData fileWcc = new WccData(Nfs3Utils.GetWccAttr(preOpAttr), preOpAttr); WRITE3Response response = new WRITE3Response(Nfs3Status.Nfs3errIo, fileWcc, count , request.GetStableHow(), Nfs3Constant.WriteCommitVerf); Nfs3Utils.WriteChannel(channel, response.Serialize(new XDR(), xid, new VerifierNone ()), xid); return; } // Add open stream string writeDumpDir = config.Get(NfsConfigKeys.DfsNfsFileDumpDirKey, NfsConfigKeys .DfsNfsFileDumpDirDefault); openFileCtx = new OpenFileCtx(fos, latestAttr, writeDumpDir + "/" + fileHandle.GetFileId (), dfsClient, iug, aixCompatMode, config); if (!AddOpenFileStream(fileHandle, openFileCtx)) { Log.Info("Can't add new stream. Close it. Tell client to retry."); try { fos.Close(); } catch (IOException e) { Log.Error("Can't close stream for fileId: " + handle.GetFileId(), e); } // Notify client to retry WccData fileWcc = new WccData(latestAttr.GetWccAttr(), latestAttr); WRITE3Response response = new WRITE3Response(Nfs3Status.Nfs3errJukebox, fileWcc, 0, request.GetStableHow(), Nfs3Constant.WriteCommitVerf); Nfs3Utils.WriteChannel(channel, response.Serialize(new XDR(), xid, new VerifierNone ()), xid); return; } if (Log.IsDebugEnabled()) { Log.Debug("Opened stream for appending file: " + fileHandle.GetFileId()); } } // Add write into the async job queue openFileCtx.ReceivedNewWrite(dfsClient, request, channel, xid, asyncDataService, iug); return; }
public virtual void TestOOOWrites() { NfsConfiguration config = new NfsConfiguration(); MiniDFSCluster cluster = null; RpcProgramNfs3 nfsd; int bufSize = 32; int numOOO = 3; SecurityHandler securityHandler = Org.Mockito.Mockito.Mock <SecurityHandler>(); Org.Mockito.Mockito.When(securityHandler.GetUser()).ThenReturn(Runtime.GetProperty ("user.name")); string currentUser = Runtime.GetProperty("user.name"); config.Set(DefaultImpersonationProvider.GetTestProvider().GetProxySuperuserGroupConfKey (currentUser), "*"); config.Set(DefaultImpersonationProvider.GetTestProvider().GetProxySuperuserIpConfKey (currentUser), "*"); ProxyUsers.RefreshSuperUserGroupsConfiguration(config); // Use emphral port in case tests are running in parallel config.SetInt("nfs3.mountd.port", 0); config.SetInt("nfs3.server.port", 0); try { cluster = new MiniDFSCluster.Builder(config).NumDataNodes(1).Build(); cluster.WaitActive(); Org.Apache.Hadoop.Hdfs.Nfs.Nfs3.Nfs3 nfs3 = new Org.Apache.Hadoop.Hdfs.Nfs.Nfs3.Nfs3 (config); nfs3.StartServiceInternal(false); nfsd = (RpcProgramNfs3)nfs3.GetRpcProgram(); DFSClient dfsClient = new DFSClient(NameNode.GetAddress(config), config); HdfsFileStatus status = dfsClient.GetFileInfo("/"); FileHandle rootHandle = new FileHandle(status.GetFileId()); CREATE3Request createReq = new CREATE3Request(rootHandle, "out-of-order-write" + Runtime.CurrentTimeMillis(), Nfs3Constant.CreateUnchecked, new SetAttr3(), 0); XDR createXdr = new XDR(); createReq.Serialize(createXdr); CREATE3Response createRsp = nfsd.Create(createXdr.AsReadOnlyWrap(), securityHandler , new IPEndPoint("localhost", 1234)); FileHandle handle = createRsp.GetObjHandle(); byte[][] oooBuf = new byte[][] { new byte[bufSize], new byte[bufSize], new byte[bufSize ] }; for (int i = 0; i < numOOO; i++) { Arrays.Fill(oooBuf[i], unchecked ((byte)i)); } for (int i_1 = 0; i_1 < numOOO; i_1++) { long offset = (numOOO - 1 - i_1) * bufSize; WRITE3Request writeReq = new WRITE3Request(handle, offset, bufSize, Nfs3Constant.WriteStableHow .Unstable, ByteBuffer.Wrap(oooBuf[i_1])); XDR writeXdr = new XDR(); writeReq.Serialize(writeXdr); nfsd.Write(writeXdr.AsReadOnlyWrap(), null, 1, securityHandler, new IPEndPoint("localhost" , 1234)); } WaitWrite(nfsd, handle, 60000); READ3Request readReq = new READ3Request(handle, bufSize, bufSize); XDR readXdr = new XDR(); readReq.Serialize(readXdr); READ3Response readRsp = nfsd.Read(readXdr.AsReadOnlyWrap(), securityHandler, new IPEndPoint("localhost", config.GetInt(NfsConfigKeys.DfsNfsServerPortKey, NfsConfigKeys .DfsNfsServerPortDefault))); NUnit.Framework.Assert.IsTrue(Arrays.Equals(oooBuf[1], ((byte[])readRsp.GetData() .Array()))); } finally { if (cluster != null) { cluster.Shutdown(); } } }
public virtual void TestWriteStableHow() { NfsConfiguration config = new NfsConfiguration(); DFSClient client = null; MiniDFSCluster cluster = null; RpcProgramNfs3 nfsd; SecurityHandler securityHandler = Org.Mockito.Mockito.Mock <SecurityHandler>(); Org.Mockito.Mockito.When(securityHandler.GetUser()).ThenReturn(Runtime.GetProperty ("user.name")); string currentUser = Runtime.GetProperty("user.name"); config.Set(DefaultImpersonationProvider.GetTestProvider().GetProxySuperuserGroupConfKey (currentUser), "*"); config.Set(DefaultImpersonationProvider.GetTestProvider().GetProxySuperuserIpConfKey (currentUser), "*"); ProxyUsers.RefreshSuperUserGroupsConfiguration(config); try { cluster = new MiniDFSCluster.Builder(config).NumDataNodes(1).Build(); cluster.WaitActive(); client = new DFSClient(NameNode.GetAddress(config), config); // Use emphral port in case tests are running in parallel config.SetInt("nfs3.mountd.port", 0); config.SetInt("nfs3.server.port", 0); // Start nfs Org.Apache.Hadoop.Hdfs.Nfs.Nfs3.Nfs3 nfs3 = new Org.Apache.Hadoop.Hdfs.Nfs.Nfs3.Nfs3 (config); nfs3.StartServiceInternal(false); nfsd = (RpcProgramNfs3)nfs3.GetRpcProgram(); HdfsFileStatus status = client.GetFileInfo("/"); FileHandle rootHandle = new FileHandle(status.GetFileId()); // Create file1 CREATE3Request createReq = new CREATE3Request(rootHandle, "file1", Nfs3Constant.CreateUnchecked , new SetAttr3(), 0); XDR createXdr = new XDR(); createReq.Serialize(createXdr); CREATE3Response createRsp = nfsd.Create(createXdr.AsReadOnlyWrap(), securityHandler , new IPEndPoint("localhost", 1234)); FileHandle handle = createRsp.GetObjHandle(); // Test DATA_SYNC byte[] buffer = new byte[10]; for (int i = 0; i < 10; i++) { buffer[i] = unchecked ((byte)i); } WRITE3Request writeReq = new WRITE3Request(handle, 0, 10, Nfs3Constant.WriteStableHow .DataSync, ByteBuffer.Wrap(buffer)); XDR writeXdr = new XDR(); writeReq.Serialize(writeXdr); nfsd.Write(writeXdr.AsReadOnlyWrap(), null, 1, securityHandler, new IPEndPoint("localhost" , 1234)); WaitWrite(nfsd, handle, 60000); // Readback READ3Request readReq = new READ3Request(handle, 0, 10); XDR readXdr = new XDR(); readReq.Serialize(readXdr); READ3Response readRsp = nfsd.Read(readXdr.AsReadOnlyWrap(), securityHandler, new IPEndPoint("localhost", 1234)); NUnit.Framework.Assert.IsTrue(Arrays.Equals(buffer, ((byte[])readRsp.GetData().Array ()))); // Test FILE_SYNC // Create file2 CREATE3Request createReq2 = new CREATE3Request(rootHandle, "file2", Nfs3Constant. CreateUnchecked, new SetAttr3(), 0); XDR createXdr2 = new XDR(); createReq2.Serialize(createXdr2); CREATE3Response createRsp2 = nfsd.Create(createXdr2.AsReadOnlyWrap(), securityHandler , new IPEndPoint("localhost", 1234)); FileHandle handle2 = createRsp2.GetObjHandle(); WRITE3Request writeReq2 = new WRITE3Request(handle2, 0, 10, Nfs3Constant.WriteStableHow .FileSync, ByteBuffer.Wrap(buffer)); XDR writeXdr2 = new XDR(); writeReq2.Serialize(writeXdr2); nfsd.Write(writeXdr2.AsReadOnlyWrap(), null, 1, securityHandler, new IPEndPoint("localhost" , 1234)); WaitWrite(nfsd, handle2, 60000); // Readback READ3Request readReq2 = new READ3Request(handle2, 0, 10); XDR readXdr2 = new XDR(); readReq2.Serialize(readXdr2); READ3Response readRsp2 = nfsd.Read(readXdr2.AsReadOnlyWrap(), securityHandler, new IPEndPoint("localhost", 1234)); NUnit.Framework.Assert.IsTrue(Arrays.Equals(buffer, ((byte[])readRsp2.GetData().Array ()))); // FILE_SYNC should sync the file size status = client.GetFileInfo("/file2"); NUnit.Framework.Assert.IsTrue(status.GetLen() == 10); } finally { if (cluster != null) { cluster.Shutdown(); } } }
public virtual void TestAlterWriteRequest() { int len = 20; byte[] data = new byte[len]; ByteBuffer buffer = ByteBuffer.Wrap(data); for (int i = 0; i < len; i++) { buffer.Put(unchecked ((byte)i)); } buffer.Flip(); int originalCount = ((byte[])buffer.Array()).Length; WRITE3Request request = new WRITE3Request(new FileHandle(), 0, data.Length, Nfs3Constant.WriteStableHow .Unstable, buffer); WriteCtx writeCtx1 = new WriteCtx(request.GetHandle(), request.GetOffset(), request .GetCount(), WriteCtx.InvalidOriginalCount, request.GetStableHow(), request.GetData (), null, 1, false, WriteCtx.DataState.NoDump); NUnit.Framework.Assert.IsTrue(((byte[])writeCtx1.GetData().Array()).Length == originalCount ); // Now change the write request OpenFileCtx.AlterWriteRequest(request, 12); WriteCtx writeCtx2 = new WriteCtx(request.GetHandle(), request.GetOffset(), request .GetCount(), originalCount, request.GetStableHow(), request.GetData(), null, 2, false, WriteCtx.DataState.NoDump); ByteBuffer appendedData = writeCtx2.GetData(); int position = appendedData.Position(); int limit = appendedData.Limit(); NUnit.Framework.Assert.IsTrue(position == 12); NUnit.Framework.Assert.IsTrue(limit - position == 8); NUnit.Framework.Assert.IsTrue(appendedData.Get(position) == unchecked ((byte)12)); NUnit.Framework.Assert.IsTrue(appendedData.Get(position + 1) == unchecked ((byte)13 )); NUnit.Framework.Assert.IsTrue(appendedData.Get(position + 2) == unchecked ((byte)14 )); NUnit.Framework.Assert.IsTrue(appendedData.Get(position + 7) == unchecked ((byte)19 )); // Test current file write offset is at boundaries buffer.Position(0); request = new WRITE3Request(new FileHandle(), 0, data.Length, Nfs3Constant.WriteStableHow .Unstable, buffer); OpenFileCtx.AlterWriteRequest(request, 1); WriteCtx writeCtx3 = new WriteCtx(request.GetHandle(), request.GetOffset(), request .GetCount(), originalCount, request.GetStableHow(), request.GetData(), null, 2, false, WriteCtx.DataState.NoDump); appendedData = writeCtx3.GetData(); position = appendedData.Position(); limit = appendedData.Limit(); NUnit.Framework.Assert.IsTrue(position == 1); NUnit.Framework.Assert.IsTrue(limit - position == 19); NUnit.Framework.Assert.IsTrue(appendedData.Get(position) == unchecked ((byte)1)); NUnit.Framework.Assert.IsTrue(appendedData.Get(position + 18) == unchecked ((byte) 19)); // Reset buffer position before test another boundary buffer.Position(0); request = new WRITE3Request(new FileHandle(), 0, data.Length, Nfs3Constant.WriteStableHow .Unstable, buffer); OpenFileCtx.AlterWriteRequest(request, 19); WriteCtx writeCtx4 = new WriteCtx(request.GetHandle(), request.GetOffset(), request .GetCount(), originalCount, request.GetStableHow(), request.GetData(), null, 2, false, WriteCtx.DataState.NoDump); appendedData = writeCtx4.GetData(); position = appendedData.Position(); limit = appendedData.Limit(); NUnit.Framework.Assert.IsTrue(position == 19); NUnit.Framework.Assert.IsTrue(limit - position == 1); NUnit.Framework.Assert.IsTrue(appendedData.Get(position) == unchecked ((byte)19)); }