/// <exception cref="System.IO.IOException"/> internal SlowWriter(DistributedFileSystem fs, Path filepath, long sleepms) : base(typeof(TestReplaceDatanodeOnFailure.SlowWriter).Name + ":" + filepath) { this.filepath = filepath; this.@out = (HdfsDataOutputStream)fs.Create(filepath, Replication); this.sleepms = sleepms; }
/// <exception cref="System.IO.IOException"/> public virtual void WriteData(HdfsDataOutputStream fos) { Preconditions.CheckState(fos != null); ByteBuffer dataBuffer; try { dataBuffer = GetData(); } catch (Exception e1) { Log.Error("Failed to get request data offset:" + offset + " count:" + count + " error:" + e1); throw new IOException("Can't get WriteCtx.data"); } byte[] data = ((byte[])dataBuffer.Array()); int position = dataBuffer.Position(); int limit = dataBuffer.Limit(); Preconditions.CheckState(limit - position == count); // Modified write has a valid original count if (position != 0) { if (limit != GetOriginalCount()) { throw new IOException("Modified write has differnt original size." + "buff position:" + position + " buff limit:" + limit + ". " + ToString()); } } // Now write data fos.Write(data, position, count); }
public virtual void TestCheckSequential() { DFSClient dfsClient = Org.Mockito.Mockito.Mock <DFSClient>(); Nfs3FileAttributes attr = new Nfs3FileAttributes(); HdfsDataOutputStream fos = Org.Mockito.Mockito.Mock <HdfsDataOutputStream>(); Org.Mockito.Mockito.When(fos.GetPos()).ThenReturn((long)0); NfsConfiguration config = new NfsConfiguration(); config.SetBoolean(NfsConfigKeys.LargeFileUpload, false); OpenFileCtx ctx = new OpenFileCtx(fos, attr, "/dumpFilePath", dfsClient, new ShellBasedIdMapping (config), false, config); ctx.GetPendingWritesForTest()[new OffsetRange(5, 10)] = new WriteCtx(null, 0, 0, 0, null, null, null, 0, false, null); ctx.GetPendingWritesForTest()[new OffsetRange(10, 15)] = new WriteCtx(null, 0, 0, 0, null, null, null, 0, false, null); ctx.GetPendingWritesForTest()[new OffsetRange(20, 25)] = new WriteCtx(null, 0, 0, 0, null, null, null, 0, false, null); NUnit.Framework.Assert.IsTrue(!ctx.CheckSequential(5, 4)); NUnit.Framework.Assert.IsTrue(ctx.CheckSequential(9, 5)); NUnit.Framework.Assert.IsTrue(ctx.CheckSequential(10, 5)); NUnit.Framework.Assert.IsTrue(ctx.CheckSequential(14, 5)); NUnit.Framework.Assert.IsTrue(!ctx.CheckSequential(15, 5)); NUnit.Framework.Assert.IsTrue(!ctx.CheckSequential(20, 5)); NUnit.Framework.Assert.IsTrue(!ctx.CheckSequential(25, 5)); NUnit.Framework.Assert.IsTrue(!ctx.CheckSequential(999, 5)); }
public virtual void TestCheckCommitAixCompatMode() { DFSClient dfsClient = Org.Mockito.Mockito.Mock <DFSClient>(); Nfs3FileAttributes attr = new Nfs3FileAttributes(); HdfsDataOutputStream fos = Org.Mockito.Mockito.Mock <HdfsDataOutputStream>(); NfsConfiguration conf = new NfsConfiguration(); conf.SetBoolean(NfsConfigKeys.LargeFileUpload, false); // Enable AIX compatibility mode. OpenFileCtx ctx = new OpenFileCtx(fos, attr, "/dumpFilePath", dfsClient, new ShellBasedIdMapping (new NfsConfiguration()), true, conf); // Test fall-through to pendingWrites check in the event that commitOffset // is greater than the number of bytes we've so far flushed. Org.Mockito.Mockito.When(fos.GetPos()).ThenReturn((long)2); OpenFileCtx.COMMIT_STATUS status = ctx.CheckCommitInternal(5, null, 1, attr, false ); NUnit.Framework.Assert.IsTrue(status == OpenFileCtx.COMMIT_STATUS.CommitFinished); // Test the case when we actually have received more bytes than we're trying // to commit. ctx.GetPendingWritesForTest()[new OffsetRange(0, 10)] = new WriteCtx(null, 0, 0, 0, null, null, null, 0, false, null); Org.Mockito.Mockito.When(fos.GetPos()).ThenReturn((long)10); ctx.SetNextOffsetForTest((long)10); status = ctx.CheckCommitInternal(5, null, 1, attr, false); NUnit.Framework.Assert.IsTrue(status == OpenFileCtx.COMMIT_STATUS.CommitDoSync); }
/// <summary>Append a file without closing the output stream</summary> /// <exception cref="System.IO.IOException"/> private HdfsDataOutputStream AppendFileWithoutClosing(Path file, int length) { byte[] toAppend = new byte[length]; Random random = new Random(); random.NextBytes(toAppend); HdfsDataOutputStream @out = (HdfsDataOutputStream)hdfs.Append(file); @out.Write(toAppend); return(@out); }
public virtual void TestEviction() { NfsConfiguration conf = new NfsConfiguration(); // Only two entries will be in the cache conf.SetInt(NfsConfigKeys.DfsNfsMaxOpenFilesKey, 2); DFSClient dfsClient = Org.Mockito.Mockito.Mock <DFSClient>(); Nfs3FileAttributes attr = new Nfs3FileAttributes(); HdfsDataOutputStream fos = Org.Mockito.Mockito.Mock <HdfsDataOutputStream>(); Org.Mockito.Mockito.When(fos.GetPos()).ThenReturn((long)0); OpenFileCtx context1 = new OpenFileCtx(fos, attr, "/dumpFilePath", dfsClient, new ShellBasedIdMapping(new NfsConfiguration())); OpenFileCtx context2 = new OpenFileCtx(fos, attr, "/dumpFilePath", dfsClient, new ShellBasedIdMapping(new NfsConfiguration())); OpenFileCtx context3 = new OpenFileCtx(fos, attr, "/dumpFilePath", dfsClient, new ShellBasedIdMapping(new NfsConfiguration())); OpenFileCtx context4 = new OpenFileCtx(fos, attr, "/dumpFilePath", dfsClient, new ShellBasedIdMapping(new NfsConfiguration())); OpenFileCtx context5 = new OpenFileCtx(fos, attr, "/dumpFilePath", dfsClient, new ShellBasedIdMapping(new NfsConfiguration())); OpenFileCtxCache cache = new OpenFileCtxCache(conf, 10 * 60 * 100); bool ret = cache.Put(new FileHandle(1), context1); NUnit.Framework.Assert.IsTrue(ret); Sharpen.Thread.Sleep(1000); ret = cache.Put(new FileHandle(2), context2); NUnit.Framework.Assert.IsTrue(ret); ret = cache.Put(new FileHandle(3), context3); NUnit.Framework.Assert.IsFalse(ret); NUnit.Framework.Assert.IsTrue(cache.Size() == 2); // Wait for the oldest stream to be evict-able, insert again Sharpen.Thread.Sleep(NfsConfigKeys.DfsNfsStreamTimeoutMinDefault); NUnit.Framework.Assert.IsTrue(cache.Size() == 2); ret = cache.Put(new FileHandle(3), context3); NUnit.Framework.Assert.IsTrue(ret); NUnit.Framework.Assert.IsTrue(cache.Size() == 2); NUnit.Framework.Assert.IsTrue(cache.Get(new FileHandle(1)) == null); // Test inactive entry is evicted immediately context3.SetActiveStatusForTest(false); ret = cache.Put(new FileHandle(4), context4); NUnit.Framework.Assert.IsTrue(ret); // Now the cache has context2 and context4 // Test eviction failure if all entries have pending work. context2.GetPendingWritesForTest()[new OffsetRange(0, 100)] = new WriteCtx(null, 0, 0, 0, null, null, null, 0, false, null); context4.GetPendingCommitsForTest()[System.Convert.ToInt64(100)] = new OpenFileCtx.CommitCtx (0, null, 0, attr); Sharpen.Thread.Sleep(NfsConfigKeys.DfsNfsStreamTimeoutMinDefault); ret = cache.Put(new FileHandle(5), context5); NUnit.Framework.Assert.IsFalse(ret); }
/// <summary>Test the fsimage saving/loading while file appending.</summary> /// <exception cref="System.Exception"/> public virtual void TestSaveLoadImageWithAppending() { Path sub1 = new Path(dir, "sub1"); Path sub1file1 = new Path(sub1, "sub1file1"); Path sub1file2 = new Path(sub1, "sub1file2"); DFSTestUtil.CreateFile(hdfs, sub1file1, Blocksize, Replication, seed); DFSTestUtil.CreateFile(hdfs, sub1file2, Blocksize, Replication, seed); // 1. create snapshot s0 hdfs.AllowSnapshot(dir); hdfs.CreateSnapshot(dir, "s0"); // 2. create snapshot s1 before appending sub1file1 finishes HdfsDataOutputStream @out = AppendFileWithoutClosing(sub1file1, Blocksize); @out.Hsync(EnumSet.Of(HdfsDataOutputStream.SyncFlag.UpdateLength)); // also append sub1file2 DFSTestUtil.AppendFile(hdfs, sub1file2, Blocksize); hdfs.CreateSnapshot(dir, "s1"); @out.Close(); // 3. create snapshot s2 before appending finishes @out = AppendFileWithoutClosing(sub1file1, Blocksize); @out.Hsync(EnumSet.Of(HdfsDataOutputStream.SyncFlag.UpdateLength)); hdfs.CreateSnapshot(dir, "s2"); @out.Close(); // 4. save fsimage before appending finishes @out = AppendFileWithoutClosing(sub1file1, Blocksize); @out.Hsync(EnumSet.Of(HdfsDataOutputStream.SyncFlag.UpdateLength)); // dump fsdir FilePath fsnBefore = DumpTree2File("before"); // save the namesystem to a temp file FilePath imageFile = SaveFSImageToTempFile(); // 5. load fsimage and compare // first restart the cluster, and format the cluster @out.Close(); cluster.Shutdown(); cluster = new MiniDFSCluster.Builder(conf).Format(true).NumDataNodes(Replication) .Build(); cluster.WaitActive(); fsn = cluster.GetNamesystem(); hdfs = cluster.GetFileSystem(); // then load the fsimage LoadFSImageFromTempFile(imageFile); // dump the fsdir tree again FilePath fsnAfter = DumpTree2File("after"); // compare two dumped tree SnapshotTestHelper.CompareDumpedTreeInFile(fsnBefore, fsnAfter, true); }
/// <summary> /// Test snapshot during file appending, before the corresponding /// <see cref="Org.Apache.Hadoop.FS.FSDataOutputStream"/> /// instance closes. /// </summary> /// <exception cref="System.Exception"/> public virtual void TestSnapshotWhileAppending() { Path file = new Path(dir, "file"); DFSTestUtil.CreateFile(hdfs, file, Blocksize, Replication, seed); // 1. append without closing stream --> create snapshot HdfsDataOutputStream @out = AppendFileWithoutClosing(file, Blocksize); @out.Hsync(EnumSet.Of(HdfsDataOutputStream.SyncFlag.UpdateLength)); SnapshotTestHelper.CreateSnapshot(hdfs, dir, "s0"); @out.Close(); // check: an INodeFileUnderConstructionWithSnapshot should be stored into s0's // deleted list, with size BLOCKSIZE*2 INodeFile fileNode = (INodeFile)fsdir.GetINode(file.ToString()); NUnit.Framework.Assert.AreEqual(Blocksize * 2, fileNode.ComputeFileSize()); INodeDirectory dirNode = fsdir.GetINode(dir.ToString()).AsDirectory(); DirectoryWithSnapshotFeature.DirectoryDiff last = dirNode.GetDiffs().GetLast(); // 2. append without closing stream @out = AppendFileWithoutClosing(file, Blocksize); @out.Hsync(EnumSet.Of(HdfsDataOutputStream.SyncFlag.UpdateLength)); // re-check nodeInDeleted_S0 dirNode = fsdir.GetINode(dir.ToString()).AsDirectory(); NUnit.Framework.Assert.AreEqual(Blocksize * 2, fileNode.ComputeFileSize(last.GetSnapshotId ())); // 3. take snapshot --> close stream hdfs.CreateSnapshot(dir, "s1"); @out.Close(); // check: an INodeFileUnderConstructionWithSnapshot with size BLOCKSIZE*3 should // have been stored in s1's deleted list fileNode = (INodeFile)fsdir.GetINode(file.ToString()); dirNode = fsdir.GetINode(dir.ToString()).AsDirectory(); last = dirNode.GetDiffs().GetLast(); NUnit.Framework.Assert.IsTrue(fileNode.IsWithSnapshot()); NUnit.Framework.Assert.AreEqual(Blocksize * 3, fileNode.ComputeFileSize(last.GetSnapshotId ())); // 4. modify file --> append without closing stream --> take snapshot --> // close stream hdfs.SetReplication(file, (short)(Replication - 1)); @out = AppendFileWithoutClosing(file, Blocksize); hdfs.CreateSnapshot(dir, "s2"); @out.Close(); // re-check the size of nodeInDeleted_S1 NUnit.Framework.Assert.AreEqual(Blocksize * 3, fileNode.ComputeFileSize(last.GetSnapshotId ())); }
public virtual void TestScan() { NfsConfiguration conf = new NfsConfiguration(); // Only two entries will be in the cache conf.SetInt(NfsConfigKeys.DfsNfsMaxOpenFilesKey, 2); DFSClient dfsClient = Org.Mockito.Mockito.Mock <DFSClient>(); Nfs3FileAttributes attr = new Nfs3FileAttributes(); HdfsDataOutputStream fos = Org.Mockito.Mockito.Mock <HdfsDataOutputStream>(); Org.Mockito.Mockito.When(fos.GetPos()).ThenReturn((long)0); OpenFileCtx context1 = new OpenFileCtx(fos, attr, "/dumpFilePath", dfsClient, new ShellBasedIdMapping(new NfsConfiguration())); OpenFileCtx context2 = new OpenFileCtx(fos, attr, "/dumpFilePath", dfsClient, new ShellBasedIdMapping(new NfsConfiguration())); OpenFileCtx context3 = new OpenFileCtx(fos, attr, "/dumpFilePath", dfsClient, new ShellBasedIdMapping(new NfsConfiguration())); OpenFileCtx context4 = new OpenFileCtx(fos, attr, "/dumpFilePath", dfsClient, new ShellBasedIdMapping(new NfsConfiguration())); OpenFileCtxCache cache = new OpenFileCtxCache(conf, 10 * 60 * 100); // Test cleaning expired entry bool ret = cache.Put(new FileHandle(1), context1); NUnit.Framework.Assert.IsTrue(ret); ret = cache.Put(new FileHandle(2), context2); NUnit.Framework.Assert.IsTrue(ret); Sharpen.Thread.Sleep(NfsConfigKeys.DfsNfsStreamTimeoutMinDefault + 1); cache.Scan(NfsConfigKeys.DfsNfsStreamTimeoutMinDefault); NUnit.Framework.Assert.IsTrue(cache.Size() == 0); // Test cleaning inactive entry ret = cache.Put(new FileHandle(3), context3); NUnit.Framework.Assert.IsTrue(ret); ret = cache.Put(new FileHandle(4), context4); NUnit.Framework.Assert.IsTrue(ret); context3.SetActiveStatusForTest(false); cache.Scan(NfsConfigKeys.DfsNfsStreamTimeoutDefault); NUnit.Framework.Assert.IsTrue(cache.Size() == 1); NUnit.Framework.Assert.IsTrue(cache.Get(new FileHandle(3)) == null); NUnit.Framework.Assert.IsTrue(cache.Get(new FileHandle(4)) != null); }
/// <exception cref="System.Exception"/> public virtual void TestMaxBlocksPerFileLimit() { Configuration conf = new HdfsConfiguration(); // Make a small block size and a low limit long blockSize = 4096; long numBlocks = 2; conf.SetLong(DFSConfigKeys.DfsBlockSizeKey, blockSize); conf.SetLong(DFSConfigKeys.DfsNamenodeMaxBlocksPerFileKey, numBlocks); MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).Build(); FileSystem fs = cluster.GetFileSystem(); HdfsDataOutputStream fout = (HdfsDataOutputStream)fs.Create(new Path("/testmaxfilelimit" )); try { // Write maximum number of blocks fout.Write(new byte[(int)blockSize * (int)numBlocks]); fout.Hflush(); // Try to write one more block try { fout.Write(new byte[1]); fout.Hflush(); System.Diagnostics.Debug.Assert(false, "Expected IOException after writing too many blocks" ); } catch (IOException e) { GenericTestUtils.AssertExceptionContains("File has reached the limit" + " on maximum number of" , e); } } finally { cluster.Shutdown(); } }
/// <summary>Test the fsimage loading while there is file under construction.</summary> /// <exception cref="System.Exception"/> public virtual void TestLoadImageWithAppending() { Path sub1 = new Path(dir, "sub1"); Path sub1file1 = new Path(sub1, "sub1file1"); Path sub1file2 = new Path(sub1, "sub1file2"); DFSTestUtil.CreateFile(hdfs, sub1file1, Blocksize, Replication, seed); DFSTestUtil.CreateFile(hdfs, sub1file2, Blocksize, Replication, seed); hdfs.AllowSnapshot(dir); hdfs.CreateSnapshot(dir, "s0"); HdfsDataOutputStream @out = AppendFileWithoutClosing(sub1file1, Blocksize); @out.Hsync(EnumSet.Of(HdfsDataOutputStream.SyncFlag.UpdateLength)); // save namespace and restart cluster hdfs.SetSafeMode(HdfsConstants.SafeModeAction.SafemodeEnter); hdfs.SaveNamespace(); hdfs.SetSafeMode(HdfsConstants.SafeModeAction.SafemodeLeave); cluster.Shutdown(); cluster = new MiniDFSCluster.Builder(conf).Format(false).NumDataNodes(Replication ).Build(); cluster.WaitActive(); fsn = cluster.GetNamesystem(); hdfs = cluster.GetFileSystem(); }
public virtual void TestGetBlockLocations() { Path root = new Path("/"); Path file = new Path("/file"); DFSTestUtil.CreateFile(hdfs, file, Blocksize, Replication, seed); // take a snapshot on root SnapshotTestHelper.CreateSnapshot(hdfs, root, "s1"); Path fileInSnapshot = SnapshotTestHelper.GetSnapshotPath(root, "s1", file.GetName ()); FileStatus status = hdfs.GetFileStatus(fileInSnapshot); // make sure we record the size for the file NUnit.Framework.Assert.AreEqual(Blocksize, status.GetLen()); // append data to file DFSTestUtil.AppendFile(hdfs, file, Blocksize - 1); status = hdfs.GetFileStatus(fileInSnapshot); // the size of snapshot file should still be BLOCKSIZE NUnit.Framework.Assert.AreEqual(Blocksize, status.GetLen()); // the size of the file should be (2 * BLOCKSIZE - 1) status = hdfs.GetFileStatus(file); NUnit.Framework.Assert.AreEqual(Blocksize * 2 - 1, status.GetLen()); // call DFSClient#callGetBlockLocations for the file in snapshot LocatedBlocks blocks = DFSClientAdapter.CallGetBlockLocations(cluster.GetNameNodeRpc (), fileInSnapshot.ToString(), 0, long.MaxValue); IList <LocatedBlock> blockList = blocks.GetLocatedBlocks(); // should be only one block NUnit.Framework.Assert.AreEqual(Blocksize, blocks.GetFileLength()); NUnit.Framework.Assert.AreEqual(1, blockList.Count); // check the last block LocatedBlock lastBlock = blocks.GetLastLocatedBlock(); NUnit.Framework.Assert.AreEqual(0, lastBlock.GetStartOffset()); NUnit.Framework.Assert.AreEqual(Blocksize, lastBlock.GetBlockSize()); // take another snapshot SnapshotTestHelper.CreateSnapshot(hdfs, root, "s2"); Path fileInSnapshot2 = SnapshotTestHelper.GetSnapshotPath(root, "s2", file.GetName ()); // append data to file without closing HdfsDataOutputStream @out = AppendFileWithoutClosing(file, Blocksize); @out.Hsync(EnumSet.Of(HdfsDataOutputStream.SyncFlag.UpdateLength)); status = hdfs.GetFileStatus(fileInSnapshot2); // the size of snapshot file should be BLOCKSIZE*2-1 NUnit.Framework.Assert.AreEqual(Blocksize * 2 - 1, status.GetLen()); // the size of the file should be (3 * BLOCKSIZE - 1) status = hdfs.GetFileStatus(file); NUnit.Framework.Assert.AreEqual(Blocksize * 3 - 1, status.GetLen()); blocks = DFSClientAdapter.CallGetBlockLocations(cluster.GetNameNodeRpc(), fileInSnapshot2 .ToString(), 0, long.MaxValue); NUnit.Framework.Assert.IsFalse(blocks.IsUnderConstruction()); NUnit.Framework.Assert.IsTrue(blocks.IsLastBlockComplete()); blockList = blocks.GetLocatedBlocks(); // should be 2 blocks NUnit.Framework.Assert.AreEqual(Blocksize * 2 - 1, blocks.GetFileLength()); NUnit.Framework.Assert.AreEqual(2, blockList.Count); // check the last block lastBlock = blocks.GetLastLocatedBlock(); NUnit.Framework.Assert.AreEqual(Blocksize, lastBlock.GetStartOffset()); NUnit.Framework.Assert.AreEqual(Blocksize, lastBlock.GetBlockSize()); blocks = DFSClientAdapter.CallGetBlockLocations(cluster.GetNameNodeRpc(), fileInSnapshot2 .ToString(), Blocksize, 0); blockList = blocks.GetLocatedBlocks(); NUnit.Framework.Assert.AreEqual(1, blockList.Count); // check blocks for file being written blocks = DFSClientAdapter.CallGetBlockLocations(cluster.GetNameNodeRpc(), file.ToString (), 0, long.MaxValue); blockList = blocks.GetLocatedBlocks(); NUnit.Framework.Assert.AreEqual(3, blockList.Count); NUnit.Framework.Assert.IsTrue(blocks.IsUnderConstruction()); NUnit.Framework.Assert.IsFalse(blocks.IsLastBlockComplete()); lastBlock = blocks.GetLastLocatedBlock(); NUnit.Framework.Assert.AreEqual(Blocksize * 2, lastBlock.GetStartOffset()); NUnit.Framework.Assert.AreEqual(Blocksize - 1, lastBlock.GetBlockSize()); @out.Close(); }
/// <exception cref="System.IO.IOException"/> internal virtual void HandleWrite(DFSClient dfsClient, WRITE3Request request, Org.Jboss.Netty.Channel.Channel channel, int xid, Nfs3FileAttributes preOpAttr) { int count = request.GetCount(); byte[] data = ((byte[])request.GetData().Array()); if (data.Length < count) { WRITE3Response response = new WRITE3Response(Nfs3Status.Nfs3errInval); Nfs3Utils.WriteChannel(channel, response.Serialize(new XDR(), xid, new VerifierNone ()), xid); return; } FileHandle handle = request.GetHandle(); if (Log.IsDebugEnabled()) { Log.Debug("handleWrite " + request); } // Check if there is a stream to write FileHandle fileHandle = request.GetHandle(); OpenFileCtx openFileCtx = fileContextCache.Get(fileHandle); if (openFileCtx == null) { Log.Info("No opened stream for fileId: " + fileHandle.GetFileId()); string fileIdPath = Nfs3Utils.GetFileIdPath(fileHandle.GetFileId()); HdfsDataOutputStream fos = null; Nfs3FileAttributes latestAttr = null; try { int bufferSize = config.GetInt(CommonConfigurationKeysPublic.IoFileBufferSizeKey, CommonConfigurationKeysPublic.IoFileBufferSizeDefault); fos = dfsClient.Append(fileIdPath, bufferSize, EnumSet.Of(CreateFlag.Append), null , null); latestAttr = Nfs3Utils.GetFileAttr(dfsClient, fileIdPath, iug); } catch (RemoteException e) { IOException io = e.UnwrapRemoteException(); if (io is AlreadyBeingCreatedException) { Log.Warn("Can't append file: " + fileIdPath + ". Possibly the file is being closed. Drop the request: " + request + ", wait for the client to retry..."); return; } throw; } catch (IOException e) { Log.Error("Can't append to file: " + fileIdPath, e); if (fos != null) { fos.Close(); } WccData fileWcc = new WccData(Nfs3Utils.GetWccAttr(preOpAttr), preOpAttr); WRITE3Response response = new WRITE3Response(Nfs3Status.Nfs3errIo, fileWcc, count , request.GetStableHow(), Nfs3Constant.WriteCommitVerf); Nfs3Utils.WriteChannel(channel, response.Serialize(new XDR(), xid, new VerifierNone ()), xid); return; } // Add open stream string writeDumpDir = config.Get(NfsConfigKeys.DfsNfsFileDumpDirKey, NfsConfigKeys .DfsNfsFileDumpDirDefault); openFileCtx = new OpenFileCtx(fos, latestAttr, writeDumpDir + "/" + fileHandle.GetFileId (), dfsClient, iug, aixCompatMode, config); if (!AddOpenFileStream(fileHandle, openFileCtx)) { Log.Info("Can't add new stream. Close it. Tell client to retry."); try { fos.Close(); } catch (IOException e) { Log.Error("Can't close stream for fileId: " + handle.GetFileId(), e); } // Notify client to retry WccData fileWcc = new WccData(latestAttr.GetWccAttr(), latestAttr); WRITE3Response response = new WRITE3Response(Nfs3Status.Nfs3errJukebox, fileWcc, 0, request.GetStableHow(), Nfs3Constant.WriteCommitVerf); Nfs3Utils.WriteChannel(channel, response.Serialize(new XDR(), xid, new VerifierNone ()), xid); return; } if (Log.IsDebugEnabled()) { Log.Debug("Opened stream for appending file: " + fileHandle.GetFileId()); } } // Add write into the async job queue openFileCtx.ReceivedNewWrite(dfsClient, request, channel, xid, asyncDataService, iug); return; }
public virtual void TestCheckCommit() { // Validate all the commit check return codes OpenFileCtx.COMMIT_STATUS, which // includes COMMIT_FINISHED, COMMIT_WAIT, COMMIT_INACTIVE_CTX, // COMMIT_INACTIVE_WITH_PENDING_WRITE, COMMIT_ERROR, and COMMIT_DO_SYNC. DFSClient dfsClient = Org.Mockito.Mockito.Mock <DFSClient>(); Nfs3FileAttributes attr = new Nfs3FileAttributes(); HdfsDataOutputStream fos = Org.Mockito.Mockito.Mock <HdfsDataOutputStream>(); Org.Mockito.Mockito.When(fos.GetPos()).ThenReturn((long)0); NfsConfiguration conf = new NfsConfiguration(); conf.SetBoolean(NfsConfigKeys.LargeFileUpload, false); OpenFileCtx ctx = new OpenFileCtx(fos, attr, "/dumpFilePath", dfsClient, new ShellBasedIdMapping (conf), false, conf); OpenFileCtx.COMMIT_STATUS ret; // Test inactive open file context ctx.SetActiveStatusForTest(false); Org.Jboss.Netty.Channel.Channel ch = Org.Mockito.Mockito.Mock <Org.Jboss.Netty.Channel.Channel >(); ret = ctx.CheckCommit(dfsClient, 0, ch, 1, attr, false); NUnit.Framework.Assert.IsTrue(ret == OpenFileCtx.COMMIT_STATUS.CommitInactiveCtx); ctx.GetPendingWritesForTest()[new OffsetRange(5, 10)] = new WriteCtx(null, 0, 0, 0, null, null, null, 0, false, null); ret = ctx.CheckCommit(dfsClient, 0, ch, 1, attr, false); NUnit.Framework.Assert.IsTrue(ret == OpenFileCtx.COMMIT_STATUS.CommitInactiveWithPendingWrite ); // Test request with non zero commit offset ctx.SetActiveStatusForTest(true); Org.Mockito.Mockito.When(fos.GetPos()).ThenReturn((long)10); ctx.SetNextOffsetForTest(10); OpenFileCtx.COMMIT_STATUS status = ctx.CheckCommitInternal(5, null, 1, attr, false ); NUnit.Framework.Assert.IsTrue(status == OpenFileCtx.COMMIT_STATUS.CommitDoSync); // Do_SYNC state will be updated to FINISHED after data sync ret = ctx.CheckCommit(dfsClient, 5, ch, 1, attr, false); NUnit.Framework.Assert.IsTrue(ret == OpenFileCtx.COMMIT_STATUS.CommitFinished); status = ctx.CheckCommitInternal(10, ch, 1, attr, false); NUnit.Framework.Assert.IsTrue(status == OpenFileCtx.COMMIT_STATUS.CommitDoSync); ret = ctx.CheckCommit(dfsClient, 10, ch, 1, attr, false); NUnit.Framework.Assert.IsTrue(ret == OpenFileCtx.COMMIT_STATUS.CommitFinished); ConcurrentNavigableMap <long, OpenFileCtx.CommitCtx> commits = ctx.GetPendingCommitsForTest (); NUnit.Framework.Assert.IsTrue(commits.Count == 0); ret = ctx.CheckCommit(dfsClient, 11, ch, 1, attr, false); NUnit.Framework.Assert.IsTrue(ret == OpenFileCtx.COMMIT_STATUS.CommitWait); NUnit.Framework.Assert.IsTrue(commits.Count == 1); long key = commits.FirstKey(); NUnit.Framework.Assert.IsTrue(key == 11); // Test request with zero commit offset Sharpen.Collections.Remove(commits, System.Convert.ToInt64(11)); // There is one pending write [5,10] ret = ctx.CheckCommit(dfsClient, 0, ch, 1, attr, false); NUnit.Framework.Assert.IsTrue(ret == OpenFileCtx.COMMIT_STATUS.CommitWait); NUnit.Framework.Assert.IsTrue(commits.Count == 1); key = commits.FirstKey(); NUnit.Framework.Assert.IsTrue(key == 9); // Empty pending writes Sharpen.Collections.Remove(ctx.GetPendingWritesForTest(), new OffsetRange(5, 10)); ret = ctx.CheckCommit(dfsClient, 0, ch, 1, attr, false); NUnit.Framework.Assert.IsTrue(ret == OpenFileCtx.COMMIT_STATUS.CommitFinished); }
public virtual void TestCheckCommitFromReadLargeFileUpload() { // Validate all the commit check return codes OpenFileCtx.COMMIT_STATUS with large file upload option DFSClient dfsClient = Org.Mockito.Mockito.Mock <DFSClient>(); Nfs3FileAttributes attr = new Nfs3FileAttributes(); HdfsDataOutputStream fos = Org.Mockito.Mockito.Mock <HdfsDataOutputStream>(); Org.Mockito.Mockito.When(fos.GetPos()).ThenReturn((long)0); NfsConfiguration config = new NfsConfiguration(); config.SetBoolean(NfsConfigKeys.LargeFileUpload, true); OpenFileCtx ctx = new OpenFileCtx(fos, attr, "/dumpFilePath", dfsClient, new ShellBasedIdMapping (config), false, config); FileHandle h = new FileHandle(1); // fake handle for "/dumpFilePath" OpenFileCtx.COMMIT_STATUS ret; WriteManager wm = new WriteManager(new ShellBasedIdMapping(config), config, false ); NUnit.Framework.Assert.IsTrue(wm.AddOpenFileStream(h, ctx)); // Test inactive open file context ctx.SetActiveStatusForTest(false); Org.Jboss.Netty.Channel.Channel ch = Org.Mockito.Mockito.Mock <Org.Jboss.Netty.Channel.Channel >(); ret = ctx.CheckCommit(dfsClient, 0, ch, 1, attr, true); NUnit.Framework.Assert.AreEqual(OpenFileCtx.COMMIT_STATUS.CommitInactiveCtx, ret); NUnit.Framework.Assert.AreEqual(Nfs3Status.Nfs3Ok, wm.CommitBeforeRead(dfsClient, h, 0)); ctx.GetPendingWritesForTest()[new OffsetRange(10, 15)] = new WriteCtx(null, 0, 0, 0, null, null, null, 0, false, null); ret = ctx.CheckCommit(dfsClient, 0, ch, 1, attr, true); NUnit.Framework.Assert.AreEqual(OpenFileCtx.COMMIT_STATUS.CommitInactiveWithPendingWrite , ret); NUnit.Framework.Assert.AreEqual(Nfs3Status.Nfs3errIo, wm.CommitBeforeRead(dfsClient , h, 0)); // Test request with non zero commit offset ctx.SetActiveStatusForTest(true); Org.Mockito.Mockito.When(fos.GetPos()).ThenReturn((long)6); ctx.SetNextOffsetForTest((long)10); OpenFileCtx.COMMIT_STATUS status = ctx.CheckCommitInternal(5, ch, 1, attr, false); NUnit.Framework.Assert.AreEqual(OpenFileCtx.COMMIT_STATUS.CommitDoSync, status); // Do_SYNC state will be updated to FINISHED after data sync ret = ctx.CheckCommit(dfsClient, 5, ch, 1, attr, true); NUnit.Framework.Assert.AreEqual(OpenFileCtx.COMMIT_STATUS.CommitFinished, ret); NUnit.Framework.Assert.AreEqual(Nfs3Status.Nfs3Ok, wm.CommitBeforeRead(dfsClient, h, 5)); // Test request with sequential writes status = ctx.CheckCommitInternal(9, ch, 1, attr, true); NUnit.Framework.Assert.IsTrue(status == OpenFileCtx.COMMIT_STATUS.CommitSpecialWait ); ret = ctx.CheckCommit(dfsClient, 9, ch, 1, attr, true); NUnit.Framework.Assert.AreEqual(OpenFileCtx.COMMIT_STATUS.CommitSpecialWait, ret); NUnit.Framework.Assert.AreEqual(Nfs3Status.Nfs3errJukebox, wm.CommitBeforeRead(dfsClient , h, 9)); // Test request with non-sequential writes ConcurrentNavigableMap <long, OpenFileCtx.CommitCtx> commits = ctx.GetPendingCommitsForTest (); NUnit.Framework.Assert.IsTrue(commits.Count == 0); ret = ctx.CheckCommit(dfsClient, 16, ch, 1, attr, true); NUnit.Framework.Assert.AreEqual(OpenFileCtx.COMMIT_STATUS.CommitSpecialSuccess, ret ); NUnit.Framework.Assert.AreEqual(0, commits.Count); // commit triggered by read doesn't wait NUnit.Framework.Assert.AreEqual(Nfs3Status.Nfs3Ok, wm.CommitBeforeRead(dfsClient, h, 16)); // Test request with zero commit offset // There is one pending write [10,15] ret = ctx.CheckCommit(dfsClient, 0, ch, 1, attr, true); NUnit.Framework.Assert.AreEqual(OpenFileCtx.COMMIT_STATUS.CommitSpecialWait, ret); NUnit.Framework.Assert.AreEqual(0, commits.Count); NUnit.Framework.Assert.AreEqual(Nfs3Status.Nfs3errJukebox, wm.CommitBeforeRead(dfsClient , h, 0)); // Empty pending writes Sharpen.Collections.Remove(ctx.GetPendingWritesForTest(), new OffsetRange(10, 15) ); ret = ctx.CheckCommit(dfsClient, 0, ch, 1, attr, true); NUnit.Framework.Assert.AreEqual(OpenFileCtx.COMMIT_STATUS.CommitSpecialWait, ret); NUnit.Framework.Assert.AreEqual(Nfs3Status.Nfs3errJukebox, wm.CommitBeforeRead(dfsClient , h, 0)); }
public virtual void TestCheckCommitLargeFileUpload() { // Validate all the commit check return codes OpenFileCtx.COMMIT_STATUS with // large file upload option. DFSClient dfsClient = Org.Mockito.Mockito.Mock <DFSClient>(); Nfs3FileAttributes attr = new Nfs3FileAttributes(); HdfsDataOutputStream fos = Org.Mockito.Mockito.Mock <HdfsDataOutputStream>(); Org.Mockito.Mockito.When(fos.GetPos()).ThenReturn((long)0); NfsConfiguration conf = new NfsConfiguration(); conf.SetBoolean(NfsConfigKeys.LargeFileUpload, true); OpenFileCtx ctx = new OpenFileCtx(fos, attr, "/dumpFilePath", dfsClient, new ShellBasedIdMapping (conf), false, conf); OpenFileCtx.COMMIT_STATUS ret; // Test inactive open file context ctx.SetActiveStatusForTest(false); Org.Jboss.Netty.Channel.Channel ch = Org.Mockito.Mockito.Mock <Org.Jboss.Netty.Channel.Channel >(); ret = ctx.CheckCommit(dfsClient, 0, ch, 1, attr, false); NUnit.Framework.Assert.IsTrue(ret == OpenFileCtx.COMMIT_STATUS.CommitInactiveCtx); ctx.GetPendingWritesForTest()[new OffsetRange(10, 15)] = new WriteCtx(null, 0, 0, 0, null, null, null, 0, false, null); ret = ctx.CheckCommit(dfsClient, 0, ch, 1, attr, false); NUnit.Framework.Assert.IsTrue(ret == OpenFileCtx.COMMIT_STATUS.CommitInactiveWithPendingWrite ); // Test request with non zero commit offset ctx.SetActiveStatusForTest(true); Org.Mockito.Mockito.When(fos.GetPos()).ThenReturn((long)8); ctx.SetNextOffsetForTest(10); OpenFileCtx.COMMIT_STATUS status = ctx.CheckCommitInternal(5, null, 1, attr, false ); NUnit.Framework.Assert.IsTrue(status == OpenFileCtx.COMMIT_STATUS.CommitDoSync); // Do_SYNC state will be updated to FINISHED after data sync ret = ctx.CheckCommit(dfsClient, 5, ch, 1, attr, false); NUnit.Framework.Assert.IsTrue(ret == OpenFileCtx.COMMIT_STATUS.CommitFinished); // Test commit sequential writes status = ctx.CheckCommitInternal(10, ch, 1, attr, false); NUnit.Framework.Assert.IsTrue(status == OpenFileCtx.COMMIT_STATUS.CommitSpecialWait ); ret = ctx.CheckCommit(dfsClient, 10, ch, 1, attr, false); NUnit.Framework.Assert.IsTrue(ret == OpenFileCtx.COMMIT_STATUS.CommitSpecialWait); // Test commit non-sequential writes ConcurrentNavigableMap <long, OpenFileCtx.CommitCtx> commits = ctx.GetPendingCommitsForTest (); NUnit.Framework.Assert.IsTrue(commits.Count == 1); ret = ctx.CheckCommit(dfsClient, 16, ch, 1, attr, false); NUnit.Framework.Assert.IsTrue(ret == OpenFileCtx.COMMIT_STATUS.CommitSpecialSuccess ); NUnit.Framework.Assert.IsTrue(commits.Count == 1); // Test request with zero commit offset Sharpen.Collections.Remove(commits, System.Convert.ToInt64(10)); // There is one pending write [10,15] ret = ctx.CheckCommitInternal(0, ch, 1, attr, false); NUnit.Framework.Assert.IsTrue(ret == OpenFileCtx.COMMIT_STATUS.CommitSpecialWait); ret = ctx.CheckCommitInternal(9, ch, 1, attr, false); NUnit.Framework.Assert.IsTrue(ret == OpenFileCtx.COMMIT_STATUS.CommitSpecialWait); NUnit.Framework.Assert.IsTrue(commits.Count == 2); // Empty pending writes. nextOffset=10, flushed pos=8 Sharpen.Collections.Remove(ctx.GetPendingWritesForTest(), new OffsetRange(10, 15) ); ret = ctx.CheckCommit(dfsClient, 0, ch, 1, attr, false); NUnit.Framework.Assert.IsTrue(ret == OpenFileCtx.COMMIT_STATUS.CommitSpecialWait); // Empty pending writes ctx.SetNextOffsetForTest((long)8); // flushed pos = 8 ret = ctx.CheckCommit(dfsClient, 0, ch, 1, attr, false); NUnit.Framework.Assert.IsTrue(ret == OpenFileCtx.COMMIT_STATUS.CommitFinished); }