/// <exception cref="System.Exception"/> public virtual void TestUncacheQuiesces() { // Create a file Path fileName = new Path("/testUncacheQuiesces"); int fileLen = 4096; DFSTestUtil.CreateFile(fs, fileName, fileLen, (short)1, unchecked ((int)(0xFDFD))); // Cache it DistributedFileSystem dfs = cluster.GetFileSystem(); dfs.AddCachePool(new CachePoolInfo("pool")); dfs.AddCacheDirective(new CacheDirectiveInfo.Builder().SetPool("pool").SetPath(fileName ).SetReplication((short)3).Build()); GenericTestUtils.WaitFor(new _Supplier_484(), 1000, 30000); // Uncache it dfs.RemoveCacheDirective(1); GenericTestUtils.WaitFor(new _Supplier_495(), 1000, 30000); // Make sure that no additional messages were sent Sharpen.Thread.Sleep(10000); MetricsRecordBuilder dnMetrics = MetricsAsserts.GetMetrics(dn.GetMetrics().Name() ); MetricsAsserts.AssertCounter("BlocksCached", 1l, dnMetrics); MetricsAsserts.AssertCounter("BlocksUncached", 1l, dnMetrics); }
/// <summary> /// Test that when we have an uncache request, and the client refuses to release /// the replica for a long time, we will un-mlock it. /// </summary> /// <exception cref="System.Exception"/> public virtual void TestRevocation() { Assume.AssumeTrue(NativeCodeLoader.IsNativeCodeLoaded() && !Path.Windows); BlockReaderTestUtil.EnableHdfsCachingTracing(); BlockReaderTestUtil.EnableShortCircuitShmTracing(); Configuration conf = GetDefaultConf(); // Set a really short revocation timeout. conf.SetLong(DFSConfigKeys.DfsDatanodeCacheRevocationTimeoutMs, 250L); // Poll very often conf.SetLong(DFSConfigKeys.DfsDatanodeCacheRevocationPollingMs, 2L); MiniDFSCluster cluster = null; cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(1).Build(); cluster.WaitActive(); DistributedFileSystem dfs = cluster.GetFileSystem(); // Create and cache a file. string TestFile = "/test_file2"; DFSTestUtil.CreateFile(dfs, new Path(TestFile), BlockSize, (short)1, unchecked ((int )(0xcafe))); dfs.AddCachePool(new CachePoolInfo("pool")); long cacheDirectiveId = dfs.AddCacheDirective(new CacheDirectiveInfo.Builder().SetPool ("pool").SetPath(new Path(TestFile)).SetReplication((short)1).Build()); FsDatasetSpi <object> fsd = cluster.GetDataNodes()[0].GetFSDataset(); DFSTestUtil.VerifyExpectedCacheUsage(BlockSize, 1, fsd); // Mmap the file. FSDataInputStream @in = dfs.Open(new Path(TestFile)); ByteBuffer buf = @in.Read(null, BlockSize, EnumSet.NoneOf <ReadOption>()); // Attempt to uncache file. The file should get uncached. Log.Info("removing cache directive {}", cacheDirectiveId); dfs.RemoveCacheDirective(cacheDirectiveId); Log.Info("finished removing cache directive {}", cacheDirectiveId); Sharpen.Thread.Sleep(1000); DFSTestUtil.VerifyExpectedCacheUsage(0, 0, fsd); // Cleanup @in.ReleaseBuffer(buf); @in.Close(); cluster.Shutdown(); }
/// <exception cref="System.Exception"/> public virtual void TestReCacheAfterUncache() { int TotalBlocksPerCache = Ints.CheckedCast(CacheCapacity / BlockSize); BlockReaderTestUtil.EnableHdfsCachingTracing(); NUnit.Framework.Assert.AreEqual(0, CacheCapacity % BlockSize); // Create a small file Path SmallFile = new Path("/smallFile"); DFSTestUtil.CreateFile(fs, SmallFile, BlockSize, (short)1, unchecked ((int)(0xcafe ))); // Create a file that will take up the whole cache Path BigFile = new Path("/bigFile"); DFSTestUtil.CreateFile(fs, BigFile, TotalBlocksPerCache * BlockSize, (short)1, unchecked ( (int)(0xbeef))); DistributedFileSystem dfs = cluster.GetFileSystem(); dfs.AddCachePool(new CachePoolInfo("pool")); long bigCacheDirectiveId = dfs.AddCacheDirective(new CacheDirectiveInfo.Builder() .SetPool("pool").SetPath(BigFile).SetReplication((short)1).Build()); GenericTestUtils.WaitFor(new _Supplier_532(TotalBlocksPerCache), 1000, 30000); // Try to cache a smaller file. It should fail. long shortCacheDirectiveId = dfs.AddCacheDirective(new CacheDirectiveInfo.Builder ().SetPool("pool").SetPath(SmallFile).SetReplication((short)1).Build()); Sharpen.Thread.Sleep(10000); MetricsRecordBuilder dnMetrics = MetricsAsserts.GetMetrics(dn.GetMetrics().Name() ); NUnit.Framework.Assert.AreEqual(TotalBlocksPerCache, MetricsAsserts.GetLongCounter ("BlocksCached", dnMetrics)); // Uncache the big file and verify that the small file can now be // cached (regression test for HDFS-6107) dfs.RemoveCacheDirective(bigCacheDirectiveId); GenericTestUtils.WaitFor(new _Supplier_560(dfs, shortCacheDirectiveId), 1000, 30000 ); dfs.RemoveCacheDirective(shortCacheDirectiveId); }
/// <summary>Add a cache pool.</summary> /// <param name="info">The request to add a cache pool.</param> /// <exception cref="System.IO.IOException"> /// /// If the request could not be completed. /// </exception> public virtual void AddCachePool(CachePoolInfo info) { dfs.AddCachePool(info); }
/// <exception cref="System.IO.IOException"/> public virtual int Run(Configuration conf, IList <string> args) { string name = StringUtils.PopFirstNonOption(args); if (name == null) { System.Console.Error.WriteLine("You must specify a name when creating a " + "cache pool." ); return(1); } CachePoolInfo info = new CachePoolInfo(name); string owner = StringUtils.PopOptionWithArgument("-owner", args); if (owner != null) { info.SetOwnerName(owner); } string group = StringUtils.PopOptionWithArgument("-group", args); if (group != null) { info.SetGroupName(group); } string modeString = StringUtils.PopOptionWithArgument("-mode", args); if (modeString != null) { short mode = short.ParseShort(modeString, 8); info.SetMode(new FsPermission(mode)); } string limitString = StringUtils.PopOptionWithArgument("-limit", args); long limit = AdminHelper.ParseLimitString(limitString); if (limit != null) { info.SetLimit(limit); } string maxTtlString = StringUtils.PopOptionWithArgument("-maxTtl", args); try { long maxTtl = AdminHelper.ParseTtlString(maxTtlString); if (maxTtl != null) { info.SetMaxRelativeExpiryMs(maxTtl); } } catch (IOException e) { System.Console.Error.WriteLine("Error while parsing maxTtl value: " + e.Message); return(1); } if (!args.IsEmpty()) { System.Console.Error.Write("Can't understand arguments: " + Joiner.On(" ").Join(args ) + "\n"); System.Console.Error.WriteLine("Usage is " + GetShortUsage()); return(1); } DistributedFileSystem dfs = AdminHelper.GetDFS(conf); try { dfs.AddCachePool(info); } catch (IOException e) { System.Console.Error.WriteLine(AdminHelper.PrettifyException(e)); return(2); } System.Console.Out.WriteLine("Successfully added cache pool " + name + "."); return(0); }
/// <summary> /// Test that we can zero-copy read cached data even without disabling /// checksums. /// </summary> /// <exception cref="System.Exception"/> public virtual void TestZeroCopyReadOfCachedData() { BlockReaderTestUtil.EnableShortCircuitShmTracing(); BlockReaderTestUtil.EnableBlockReaderFactoryTracing(); BlockReaderTestUtil.EnableHdfsCachingTracing(); int TestFileLength = BlockSize; Path TestPath = new Path("/a"); int RandomSeed = 23453; HdfsConfiguration conf = InitZeroCopyTest(); conf.SetBoolean(DFSConfigKeys.DfsClientReadShortcircuitSkipChecksumKey, false); string Context = "testZeroCopyReadOfCachedData"; conf.Set(DFSConfigKeys.DfsClientContext, Context); conf.SetLong(DFSConfigKeys.DfsDatanodeMaxLockedMemoryKey, DFSTestUtil.RoundUpToMultiple (TestFileLength, (int)NativeIO.POSIX.GetCacheManipulator().GetOperatingSystemPageSize ())); MiniDFSCluster cluster = null; ByteBuffer result = null; ByteBuffer result2 = null; cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(1).Build(); cluster.WaitActive(); FsDatasetSpi <object> fsd = cluster.GetDataNodes()[0].GetFSDataset(); DistributedFileSystem fs = cluster.GetFileSystem(); DFSTestUtil.CreateFile(fs, TestPath, TestFileLength, (short)1, RandomSeed); DFSTestUtil.WaitReplication(fs, TestPath, (short)1); byte[] original = DFSTestUtil.CalculateFileContentsFromSeed(RandomSeed, TestFileLength ); // Prior to caching, the file can't be read via zero-copy FSDataInputStream fsIn = fs.Open(TestPath); try { result = fsIn.Read(null, TestFileLength / 2, EnumSet.NoneOf <ReadOption>()); NUnit.Framework.Assert.Fail("expected UnsupportedOperationException"); } catch (NotSupportedException) { } // expected // Cache the file fs.AddCachePool(new CachePoolInfo("pool1")); long directiveId = fs.AddCacheDirective(new CacheDirectiveInfo.Builder().SetPath( TestPath).SetReplication((short)1).SetPool("pool1").Build()); int numBlocks = (int)Math.Ceil((double)TestFileLength / BlockSize); DFSTestUtil.VerifyExpectedCacheUsage(DFSTestUtil.RoundUpToMultiple(TestFileLength , BlockSize), numBlocks, cluster.GetDataNodes()[0].GetFSDataset()); try { result = fsIn.Read(null, TestFileLength, EnumSet.NoneOf <ReadOption>()); } catch (NotSupportedException) { NUnit.Framework.Assert.Fail("expected to be able to read cached file via zero-copy" ); } Assert.AssertArrayEquals(Arrays.CopyOfRange(original, 0, BlockSize), ByteBufferToArray (result)); // Test that files opened after the cache operation has finished // still get the benefits of zero-copy (regression test for HDFS-6086) FSDataInputStream fsIn2 = fs.Open(TestPath); try { result2 = fsIn2.Read(null, TestFileLength, EnumSet.NoneOf <ReadOption>()); } catch (NotSupportedException) { NUnit.Framework.Assert.Fail("expected to be able to read cached file via zero-copy" ); } Assert.AssertArrayEquals(Arrays.CopyOfRange(original, 0, BlockSize), ByteBufferToArray (result2)); fsIn2.ReleaseBuffer(result2); fsIn2.Close(); // check that the replica is anchored ExtendedBlock firstBlock = DFSTestUtil.GetFirstBlock(fs, TestPath); ShortCircuitCache cache = ClientContext.Get(Context, new DFSClient.Conf(conf)).GetShortCircuitCache (); WaitForReplicaAnchorStatus(cache, firstBlock, true, true, 1); // Uncache the replica fs.RemoveCacheDirective(directiveId); WaitForReplicaAnchorStatus(cache, firstBlock, false, true, 1); fsIn.ReleaseBuffer(result); WaitForReplicaAnchorStatus(cache, firstBlock, false, false, 1); DFSTestUtil.VerifyExpectedCacheUsage(0, 0, fsd); fsIn.Close(); fs.Close(); cluster.Shutdown(); }