public virtual void Test2GBMmapLimit() { Assume.AssumeTrue(BlockReaderTestUtil.ShouldTestLargeFiles()); HdfsConfiguration conf = InitZeroCopyTest(); long TestFileLength = 2469605888L; conf.Set(DFSConfigKeys.DfsChecksumTypeKey, "NULL"); conf.SetLong(DFSConfigKeys.DfsBlockSizeKey, TestFileLength); MiniDFSCluster cluster = null; Path TestPath = new Path("/a"); string Context = "test2GBMmapLimit"; conf.Set(DFSConfigKeys.DfsClientContext, Context); FSDataInputStream fsIn = null; FSDataInputStream fsIn2 = null; ByteBuffer buf1 = null; ByteBuffer buf2 = null; try { cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(1).Build(); cluster.WaitActive(); DistributedFileSystem fs = cluster.GetFileSystem(); DFSTestUtil.CreateFile(fs, TestPath, TestFileLength, (short)1, unchecked ((int)(0xB ))); DFSTestUtil.WaitReplication(fs, TestPath, (short)1); fsIn = fs.Open(TestPath); buf1 = fsIn.Read(null, 1, EnumSet.Of(ReadOption.SkipChecksums)); NUnit.Framework.Assert.AreEqual(1, buf1.Remaining()); fsIn.ReleaseBuffer(buf1); buf1 = null; fsIn.Seek(2147483640L); buf1 = fsIn.Read(null, 1024, EnumSet.Of(ReadOption.SkipChecksums)); NUnit.Framework.Assert.AreEqual(7, buf1.Remaining()); NUnit.Framework.Assert.AreEqual(int.MaxValue, buf1.Limit()); fsIn.ReleaseBuffer(buf1); buf1 = null; NUnit.Framework.Assert.AreEqual(2147483647L, fsIn.GetPos()); try { buf1 = fsIn.Read(null, 1024, EnumSet.Of(ReadOption.SkipChecksums)); NUnit.Framework.Assert.Fail("expected UnsupportedOperationException"); } catch (NotSupportedException) { } // expected; can't read past 2GB boundary. fsIn.Close(); fsIn = null; // Now create another file with normal-sized blocks, and verify we // can read past 2GB Path TestPath2 = new Path("/b"); conf.SetLong(DFSConfigKeys.DfsBlockSizeKey, 268435456L); DFSTestUtil.CreateFile(fs, TestPath2, 1024 * 1024, TestFileLength, 268435456L, (short )1, unchecked ((int)(0xA))); fsIn2 = fs.Open(TestPath2); fsIn2.Seek(2147483640L); buf2 = fsIn2.Read(null, 1024, EnumSet.Of(ReadOption.SkipChecksums)); NUnit.Framework.Assert.AreEqual(8, buf2.Remaining()); NUnit.Framework.Assert.AreEqual(2147483648L, fsIn2.GetPos()); fsIn2.ReleaseBuffer(buf2); buf2 = null; buf2 = fsIn2.Read(null, 1024, EnumSet.Of(ReadOption.SkipChecksums)); NUnit.Framework.Assert.AreEqual(1024, buf2.Remaining()); NUnit.Framework.Assert.AreEqual(2147484672L, fsIn2.GetPos()); fsIn2.ReleaseBuffer(buf2); buf2 = null; } finally { if (buf1 != null) { fsIn.ReleaseBuffer(buf1); } if (buf2 != null) { fsIn2.ReleaseBuffer(buf2); } IOUtils.Cleanup(null, fsIn, fsIn2); if (cluster != null) { cluster.Shutdown(); } } }
public virtual void TestZeroCopyReads() { HdfsConfiguration conf = InitZeroCopyTest(); MiniDFSCluster cluster = null; Path TestPath = new Path("/a"); FSDataInputStream fsIn = null; int TestFileLength = 3 * BlockSize; FileSystem fs = null; try { cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(1).Build(); cluster.WaitActive(); fs = cluster.GetFileSystem(); DFSTestUtil.CreateFile(fs, TestPath, TestFileLength, (short)1, 7567L); try { DFSTestUtil.WaitReplication(fs, TestPath, (short)1); } catch (Exception e) { NUnit.Framework.Assert.Fail("unexpected InterruptedException during " + "waitReplication: " + e); } catch (TimeoutException e) { NUnit.Framework.Assert.Fail("unexpected TimeoutException during " + "waitReplication: " + e); } fsIn = fs.Open(TestPath); byte[] original = new byte[TestFileLength]; IOUtils.ReadFully(fsIn, original, 0, TestFileLength); fsIn.Close(); fsIn = fs.Open(TestPath); ByteBuffer result = fsIn.Read(null, BlockSize, EnumSet.Of(ReadOption.SkipChecksums )); NUnit.Framework.Assert.AreEqual(BlockSize, result.Remaining()); HdfsDataInputStream dfsIn = (HdfsDataInputStream)fsIn; NUnit.Framework.Assert.AreEqual(BlockSize, dfsIn.GetReadStatistics().GetTotalBytesRead ()); NUnit.Framework.Assert.AreEqual(BlockSize, dfsIn.GetReadStatistics().GetTotalZeroCopyBytesRead ()); Assert.AssertArrayEquals(Arrays.CopyOfRange(original, 0, BlockSize), ByteBufferToArray (result)); fsIn.ReleaseBuffer(result); } finally { if (fsIn != null) { fsIn.Close(); } if (fs != null) { fs.Close(); } if (cluster != null) { cluster.Shutdown(); } } }
/// <summary> /// Test that we can zero-copy read cached data even without disabling /// checksums. /// </summary> /// <exception cref="System.Exception"/> public virtual void TestZeroCopyReadOfCachedData() { BlockReaderTestUtil.EnableShortCircuitShmTracing(); BlockReaderTestUtil.EnableBlockReaderFactoryTracing(); BlockReaderTestUtil.EnableHdfsCachingTracing(); int TestFileLength = BlockSize; Path TestPath = new Path("/a"); int RandomSeed = 23453; HdfsConfiguration conf = InitZeroCopyTest(); conf.SetBoolean(DFSConfigKeys.DfsClientReadShortcircuitSkipChecksumKey, false); string Context = "testZeroCopyReadOfCachedData"; conf.Set(DFSConfigKeys.DfsClientContext, Context); conf.SetLong(DFSConfigKeys.DfsDatanodeMaxLockedMemoryKey, DFSTestUtil.RoundUpToMultiple (TestFileLength, (int)NativeIO.POSIX.GetCacheManipulator().GetOperatingSystemPageSize ())); MiniDFSCluster cluster = null; ByteBuffer result = null; ByteBuffer result2 = null; cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(1).Build(); cluster.WaitActive(); FsDatasetSpi <object> fsd = cluster.GetDataNodes()[0].GetFSDataset(); DistributedFileSystem fs = cluster.GetFileSystem(); DFSTestUtil.CreateFile(fs, TestPath, TestFileLength, (short)1, RandomSeed); DFSTestUtil.WaitReplication(fs, TestPath, (short)1); byte[] original = DFSTestUtil.CalculateFileContentsFromSeed(RandomSeed, TestFileLength ); // Prior to caching, the file can't be read via zero-copy FSDataInputStream fsIn = fs.Open(TestPath); try { result = fsIn.Read(null, TestFileLength / 2, EnumSet.NoneOf <ReadOption>()); NUnit.Framework.Assert.Fail("expected UnsupportedOperationException"); } catch (NotSupportedException) { } // expected // Cache the file fs.AddCachePool(new CachePoolInfo("pool1")); long directiveId = fs.AddCacheDirective(new CacheDirectiveInfo.Builder().SetPath( TestPath).SetReplication((short)1).SetPool("pool1").Build()); int numBlocks = (int)Math.Ceil((double)TestFileLength / BlockSize); DFSTestUtil.VerifyExpectedCacheUsage(DFSTestUtil.RoundUpToMultiple(TestFileLength , BlockSize), numBlocks, cluster.GetDataNodes()[0].GetFSDataset()); try { result = fsIn.Read(null, TestFileLength, EnumSet.NoneOf <ReadOption>()); } catch (NotSupportedException) { NUnit.Framework.Assert.Fail("expected to be able to read cached file via zero-copy" ); } Assert.AssertArrayEquals(Arrays.CopyOfRange(original, 0, BlockSize), ByteBufferToArray (result)); // Test that files opened after the cache operation has finished // still get the benefits of zero-copy (regression test for HDFS-6086) FSDataInputStream fsIn2 = fs.Open(TestPath); try { result2 = fsIn2.Read(null, TestFileLength, EnumSet.NoneOf <ReadOption>()); } catch (NotSupportedException) { NUnit.Framework.Assert.Fail("expected to be able to read cached file via zero-copy" ); } Assert.AssertArrayEquals(Arrays.CopyOfRange(original, 0, BlockSize), ByteBufferToArray (result2)); fsIn2.ReleaseBuffer(result2); fsIn2.Close(); // check that the replica is anchored ExtendedBlock firstBlock = DFSTestUtil.GetFirstBlock(fs, TestPath); ShortCircuitCache cache = ClientContext.Get(Context, new DFSClient.Conf(conf)).GetShortCircuitCache (); WaitForReplicaAnchorStatus(cache, firstBlock, true, true, 1); // Uncache the replica fs.RemoveCacheDirective(directiveId); WaitForReplicaAnchorStatus(cache, firstBlock, false, true, 1); fsIn.ReleaseBuffer(result); WaitForReplicaAnchorStatus(cache, firstBlock, false, false, 1); DFSTestUtil.VerifyExpectedCacheUsage(0, 0, fsd); fsIn.Close(); fs.Close(); cluster.Shutdown(); }
public virtual void TestClientMmapDisable() { HdfsConfiguration conf = InitZeroCopyTest(); conf.SetBoolean(DFSConfigKeys.DfsClientMmapEnabled, false); MiniDFSCluster cluster = null; Path TestPath = new Path("/a"); int TestFileLength = 16385; int RandomSeed = 23453; string Context = "testClientMmapDisable"; FSDataInputStream fsIn = null; DistributedFileSystem fs = null; conf.Set(DFSConfigKeys.DfsClientContext, Context); try { // With DFS_CLIENT_MMAP_ENABLED set to false, we should not do memory // mapped reads. cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(1).Build(); cluster.WaitActive(); fs = cluster.GetFileSystem(); DFSTestUtil.CreateFile(fs, TestPath, TestFileLength, (short)1, RandomSeed); DFSTestUtil.WaitReplication(fs, TestPath, (short)1); fsIn = fs.Open(TestPath); try { fsIn.Read(null, 1, EnumSet.Of(ReadOption.SkipChecksums)); NUnit.Framework.Assert.Fail("expected zero-copy read to fail when client mmaps " + "were disabled."); } catch (NotSupportedException) { } } finally { if (fsIn != null) { fsIn.Close(); } if (fs != null) { fs.Close(); } if (cluster != null) { cluster.Shutdown(); } } fsIn = null; fs = null; cluster = null; try { // Now try again with DFS_CLIENT_MMAP_CACHE_SIZE == 0. It should work. conf.SetBoolean(DFSConfigKeys.DfsClientMmapEnabled, true); conf.SetInt(DFSConfigKeys.DfsClientMmapCacheSize, 0); conf.Set(DFSConfigKeys.DfsClientContext, Context + ".1"); cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(1).Build(); cluster.WaitActive(); fs = cluster.GetFileSystem(); DFSTestUtil.CreateFile(fs, TestPath, TestFileLength, (short)1, RandomSeed); DFSTestUtil.WaitReplication(fs, TestPath, (short)1); fsIn = fs.Open(TestPath); ByteBuffer buf = fsIn.Read(null, 1, EnumSet.Of(ReadOption.SkipChecksums)); fsIn.ReleaseBuffer(buf); // Test EOF behavior IOUtils.SkipFully(fsIn, TestFileLength - 1); buf = fsIn.Read(null, 1, EnumSet.Of(ReadOption.SkipChecksums)); NUnit.Framework.Assert.AreEqual(null, buf); } finally { if (fsIn != null) { fsIn.Close(); } if (fs != null) { fs.Close(); } if (cluster != null) { cluster.Shutdown(); } } }
public virtual void TestZeroCopyMmapCache() { HdfsConfiguration conf = InitZeroCopyTest(); MiniDFSCluster cluster = null; Path TestPath = new Path("/a"); int TestFileLength = 5 * BlockSize; int RandomSeed = 23453; string Context = "testZeroCopyMmapCacheContext"; FSDataInputStream fsIn = null; ByteBuffer[] results = new ByteBuffer[] { null, null, null, null }; DistributedFileSystem fs = null; conf.Set(DFSConfigKeys.DfsClientContext, Context); cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(1).Build(); cluster.WaitActive(); fs = cluster.GetFileSystem(); DFSTestUtil.CreateFile(fs, TestPath, TestFileLength, (short)1, RandomSeed); try { DFSTestUtil.WaitReplication(fs, TestPath, (short)1); } catch (Exception e) { NUnit.Framework.Assert.Fail("unexpected InterruptedException during " + "waitReplication: " + e); } catch (TimeoutException e) { NUnit.Framework.Assert.Fail("unexpected TimeoutException during " + "waitReplication: " + e); } fsIn = fs.Open(TestPath); byte[] original = new byte[TestFileLength]; IOUtils.ReadFully(fsIn, original, 0, TestFileLength); fsIn.Close(); fsIn = fs.Open(TestPath); ShortCircuitCache cache = ClientContext.Get(Context, new DFSClient.Conf(conf)).GetShortCircuitCache (); cache.Accept(new TestEnhancedByteBufferAccess.CountingVisitor(0, 5, 5, 0)); results[0] = fsIn.Read(null, BlockSize, EnumSet.Of(ReadOption.SkipChecksums)); fsIn.Seek(0); results[1] = fsIn.Read(null, BlockSize, EnumSet.Of(ReadOption.SkipChecksums)); // The mmap should be of the first block of the file. ExtendedBlock firstBlock = DFSTestUtil.GetFirstBlock(fs, TestPath); cache.Accept(new _CacheVisitor_373(firstBlock)); // The replica should not yet be evictable, since we have it open. // Read more blocks. results[2] = fsIn.Read(null, BlockSize, EnumSet.Of(ReadOption.SkipChecksums)); results[3] = fsIn.Read(null, BlockSize, EnumSet.Of(ReadOption.SkipChecksums)); // we should have 3 mmaps, 1 evictable cache.Accept(new TestEnhancedByteBufferAccess.CountingVisitor(3, 5, 2, 0)); // After we close the cursors, the mmaps should be evictable for // a brief period of time. Then, they should be closed (we're // using a very quick timeout) foreach (ByteBuffer buffer in results) { if (buffer != null) { fsIn.ReleaseBuffer(buffer); } } fsIn.Close(); GenericTestUtils.WaitFor(new _Supplier_407(cache), 10, 60000); cache.Accept(new TestEnhancedByteBufferAccess.CountingVisitor(0, -1, -1, -1)); fs.Close(); cluster.Shutdown(); }