public virtual void TestDisableCache() { HdfsConfiguration confWithoutCache = new HdfsConfiguration(); // Configure a new instance with no peer caching, ensure that it doesn't // cache anything confWithoutCache.SetInt(DFSConfigKeys.DfsClientSocketCacheCapacityKey, 0); BlockReaderTestUtil util = new BlockReaderTestUtil(1, confWithoutCache); Path testFile = new Path("/testConnCache.dat"); util.WriteFile(testFile, FileSize / 1024); FileSystem fsWithoutCache = FileSystem.NewInstance(util.GetConf()); try { DFSTestUtil.ReadFile(fsWithoutCache, testFile); NUnit.Framework.Assert.AreEqual(0, ((DistributedFileSystem)fsWithoutCache).dfs.GetClientContext ().GetPeerCache().Size()); } finally { fsWithoutCache.Close(); util.Shutdown(); } }
/// <exception cref="System.Exception"/> public static void SetupCluster(int replicationFactor, HdfsConfiguration conf) { util = new BlockReaderTestUtil(replicationFactor, conf); dfsClient = util.GetDFSClient(); long seed = Time.Now(); Log.Info("Random seed: " + seed); rand = new Random(seed); }
public static void SetupCluster() { int ReplicationFactor = 1; util = new BlockReaderTestUtil(ReplicationFactor); util.WriteFile(TestFile, FileSizeK); IList <LocatedBlock> blkList = util.GetFileBlocks(TestFile, FileSizeK); testBlock = blkList[0]; }
public void Run() { try { IList <LocatedBlock> locatedBlocks = cluster.GetNameNode().GetRpcServer().GetBlockLocations (TestFile, 0, TestFileLen).GetLocatedBlocks(); LocatedBlock lblock = locatedBlocks[0]; BlockReader blockReader = null; try { blockReader = BlockReaderTestUtil.GetBlockReader(cluster, lblock, 0, TestFileLen); NUnit.Framework.Assert.Fail("expected getBlockReader to fail the first time."); } catch (Exception t) { NUnit.Framework.Assert.IsTrue("expected to see 'TCP reads were disabled " + "for testing' in exception " + t, t.Message.Contains("TCP reads were disabled for testing")); } finally { if (blockReader != null) { blockReader.Close(); } } gotFailureLatch.CountDown(); shouldRetryLatch.Await(); try { blockReader = BlockReaderTestUtil.GetBlockReader(cluster, lblock, 0, TestFileLen); } catch (Exception t) { TestBlockReaderFactory.Log.Error("error trying to retrieve a block reader " + "the second time." , t); throw; } finally { if (blockReader != null) { blockReader.Close(); } } } catch (Exception t) { TestBlockReaderFactory.Log.Error("getBlockReader failure", t); testFailed.Set(true); } }
/// <summary> /// Test the case where we have a failure to complete a short circuit read /// that occurs, and then later on, we have a success. /// </summary> /// <remarks> /// Test the case where we have a failure to complete a short circuit read /// that occurs, and then later on, we have a success. /// Any thread waiting on a cache load should receive the failure (if it /// occurs); however, the failure result should not be cached. We want /// to be able to retry later and succeed. /// </remarks> /// <exception cref="System.Exception"/> public virtual void TestShortCircuitCacheTemporaryFailure() { BlockReaderTestUtil.EnableBlockReaderFactoryTracing(); AtomicBoolean replicaCreationShouldFail = new AtomicBoolean(true); AtomicBoolean testFailed = new AtomicBoolean(false); DFSInputStream.tcpReadsDisabledForTesting = true; BlockReaderFactory.createShortCircuitReplicaInfoCallback = new _ShortCircuitReplicaCreator_215 (replicaCreationShouldFail); // Insert a short delay to increase the chance that one client // thread waits for the other client thread's failure via // a condition variable. TemporarySocketDirectory sockDir = new TemporarySocketDirectory(); Configuration conf = CreateShortCircuitConf("testShortCircuitCacheTemporaryFailure" , sockDir); MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(1).Build(); cluster.WaitActive(); DistributedFileSystem dfs = cluster.GetFileSystem(); string TestFile = "/test_file"; int TestFileLen = 4000; int NumThreads = 2; int Seed = unchecked ((int)(0xFADED)); CountDownLatch gotFailureLatch = new CountDownLatch(NumThreads); CountDownLatch shouldRetryLatch = new CountDownLatch(1); DFSTestUtil.CreateFile(dfs, new Path(TestFile), TestFileLen, (short)1, Seed); Runnable readerRunnable = new _Runnable_243(cluster, TestFile, TestFileLen, gotFailureLatch , shouldRetryLatch, testFailed); // First time should fail. // first block // keep findbugs happy // Second time should succeed. Sharpen.Thread[] threads = new Sharpen.Thread[NumThreads]; for (int i = 0; i < NumThreads; i++) { threads[i] = new Sharpen.Thread(readerRunnable); threads[i].Start(); } gotFailureLatch.Await(); replicaCreationShouldFail.Set(false); shouldRetryLatch.CountDown(); for (int i_1 = 0; i_1 < NumThreads; i_1++) { Uninterruptibles.JoinUninterruptibly(threads[i_1]); } cluster.Shutdown(); sockDir.Close(); NUnit.Framework.Assert.IsFalse(testFailed.Get()); }
public void Run() { try { while (true) { BlockReader blockReader = null; try { blockReader = BlockReaderTestUtil.GetBlockReader(cluster, lblock, 0, TestFileLen); sem.Release(); try { blockReader.ReadAll(buf, 0, TestFileLen); } finally { sem.AcquireUninterruptibly(); } } catch (ClosedByInterruptException e) { TestBlockReaderFactory.Log.Info("got the expected ClosedByInterruptException", e); sem.Release(); break; } finally { if (blockReader != null) { blockReader.Close(); } } TestBlockReaderFactory.Log.Info("read another " + TestFileLen + " bytes."); } } catch (Exception t) { TestBlockReaderFactory.Log.Error("getBlockReader failure", t); testFailed.Set(true); sem.Release(); } }
public virtual void TestReadFromOneDN() { HdfsConfiguration configuration = new HdfsConfiguration(); // One of the goals of this test is to verify that we don't open more // than one socket. So use a different client context, so that we // get our own socket cache, rather than sharing with the other test // instances. Also use a really long socket timeout so that nothing // gets closed before we get around to checking the cache size at the end. string contextName = "testReadFromOneDNContext"; configuration.Set(DFSConfigKeys.DfsClientContext, contextName); configuration.SetLong(DFSConfigKeys.DfsClientSocketTimeoutKey, 100000000L); BlockReaderTestUtil util = new BlockReaderTestUtil(1, configuration); Path testFile = new Path("/testConnCache.dat"); byte[] authenticData = util.WriteFile(testFile, FileSize / 1024); DFSClient client = new DFSClient(new IPEndPoint("localhost", util.GetCluster().GetNameNodePort ()), util.GetConf()); ClientContext cacheContext = ClientContext.Get(contextName, client.GetConf()); DFSInputStream @in = client.Open(testFile.ToString()); Log.Info("opened " + testFile.ToString()); byte[] dataBuf = new byte[BlockSize]; // Initial read Pread(@in, 0, dataBuf, 0, dataBuf.Length, authenticData); // Read again and verify that the socket is the same Pread(@in, FileSize - dataBuf.Length, dataBuf, 0, dataBuf.Length, authenticData); Pread(@in, 1024, dataBuf, 0, dataBuf.Length, authenticData); // No seek; just read Pread(@in, -1, dataBuf, 0, dataBuf.Length, authenticData); Pread(@in, 64, dataBuf, 0, dataBuf.Length / 2, authenticData); @in.Close(); client.Close(); NUnit.Framework.Assert.AreEqual(1, ClientContext.GetFromConf(configuration).GetPeerCache ().Size()); }
/// <summary> /// When an InterruptedException is sent to a thread calling /// FileChannel#read, the FileChannel is immediately closed and the /// thread gets an exception. /// </summary> /// <remarks> /// When an InterruptedException is sent to a thread calling /// FileChannel#read, the FileChannel is immediately closed and the /// thread gets an exception. This effectively means that we might have /// someone asynchronously calling close() on the file descriptors we use /// in BlockReaderLocal. So when unreferencing a ShortCircuitReplica in /// ShortCircuitCache#unref, we should check if the FileChannel objects /// are still open. If not, we should purge the replica to avoid giving /// it out to any future readers. /// This is a regression test for HDFS-6227: Short circuit read failed /// due to ClosedChannelException. /// Note that you may still get ClosedChannelException errors if two threads /// are reading from the same replica and an InterruptedException is delivered /// to one of them. /// </remarks> /// <exception cref="System.Exception"/> public virtual void TestPurgingClosedReplicas() { BlockReaderTestUtil.EnableBlockReaderFactoryTracing(); AtomicInteger replicasCreated = new AtomicInteger(0); AtomicBoolean testFailed = new AtomicBoolean(false); DFSInputStream.tcpReadsDisabledForTesting = true; BlockReaderFactory.createShortCircuitReplicaInfoCallback = new _ShortCircuitReplicaCreator_443 (replicasCreated); TemporarySocketDirectory sockDir = new TemporarySocketDirectory(); Configuration conf = CreateShortCircuitConf("testPurgingClosedReplicas", sockDir); MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(1).Build(); cluster.WaitActive(); DistributedFileSystem dfs = cluster.GetFileSystem(); string TestFile = "/test_file"; int TestFileLen = 4095; int Seed = unchecked ((int)(0xFADE0)); DistributedFileSystem fs = (DistributedFileSystem)FileSystem.Get(cluster.GetURI(0 ), conf); DFSTestUtil.CreateFile(fs, new Path(TestFile), TestFileLen, (short)1, Seed); Semaphore sem = Sharpen.Extensions.CreateSemaphore(0); IList <LocatedBlock> locatedBlocks = cluster.GetNameNode().GetRpcServer().GetBlockLocations (TestFile, 0, TestFileLen).GetLocatedBlocks(); LocatedBlock lblock = locatedBlocks[0]; // first block byte[] buf = new byte[TestFileLen]; Runnable readerRunnable = new _Runnable_471(cluster, lblock, TestFileLen, sem, buf , testFailed); Sharpen.Thread thread = new Sharpen.Thread(readerRunnable); thread.Start(); // While the thread is reading, send it interrupts. // These should trigger a ClosedChannelException. while (thread.IsAlive()) { sem.AcquireUninterruptibly(); thread.Interrupt(); sem.Release(); } NUnit.Framework.Assert.IsFalse(testFailed.Get()); // We should be able to read from the file without // getting a ClosedChannelException. BlockReader blockReader = null; try { blockReader = BlockReaderTestUtil.GetBlockReader(cluster, lblock, 0, TestFileLen); blockReader.ReadFully(buf, 0, TestFileLen); } finally { if (blockReader != null) { blockReader.Close(); } } byte[] expected = DFSTestUtil.CalculateFileContentsFromSeed(Seed, TestFileLen); NUnit.Framework.Assert.IsTrue(Arrays.Equals(buf, expected)); // Another ShortCircuitReplica object should have been created. NUnit.Framework.Assert.AreEqual(2, replicasCreated.Get()); dfs.Close(); cluster.Shutdown(); sockDir.Close(); }