Пример #1
0
        // Regression test for HDFS-8070
        /// <exception cref="System.Exception"/>
        public virtual void TestPreReceiptVerificationDfsClientCanDoScr()
        {
            BlockReaderTestUtil.EnableShortCircuitShmTracing();
            TemporarySocketDirectory sockDir = new TemporarySocketDirectory();
            Configuration            conf    = CreateShortCircuitConf("testPreReceiptVerificationDfsClientCanDoScr"
                                                                      , sockDir);

            conf.SetLong(DFSConfigKeys.DfsClientReadShortcircuitStreamsCacheExpiryMsKey, 1000000000L
                         );
            MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(1).Build();

            cluster.WaitActive();
            DistributedFileSystem fs = cluster.GetFileSystem();

            fs.GetClient().GetConf().brfFailureInjector = new TestShortCircuitCache.TestPreReceiptVerificationFailureInjector
                                                              ();
            Path TestPath1 = new Path("/test_file1");

            DFSTestUtil.CreateFile(fs, TestPath1, 4096, (short)1, unchecked ((int)(0xFADE2)));
            Path TestPath2 = new Path("/test_file2");

            DFSTestUtil.CreateFile(fs, TestPath2, 4096, (short)1, unchecked ((int)(0xFADE2)));
            DFSTestUtil.ReadFileBuffer(fs, TestPath1);
            DFSTestUtil.ReadFileBuffer(fs, TestPath2);
            ShortCircuitRegistry registry = cluster.GetDataNodes()[0].GetShortCircuitRegistry
                                                ();

            registry.Visit(new _Visitor_780());
            cluster.Shutdown();
            sockDir.Close();
        }
Пример #2
0
        public virtual void TestBothOldAndNewShortCircuitConfigured()
        {
            short ReplFactor = 1;
            int   FileLength = 512;

            Assume.AssumeTrue(null == DomainSocket.GetLoadingFailureReason());
            TemporarySocketDirectory socketDir = new TemporarySocketDirectory();
            HdfsConfiguration        conf      = GetConfiguration(socketDir);
            MiniDFSCluster           cluster   = new MiniDFSCluster.Builder(conf).NumDataNodes(1).Build();

            cluster.WaitActive();
            socketDir.Close();
            FileSystem fs   = cluster.GetFileSystem();
            Path       path = new Path("/foo");

            byte[] orig = new byte[FileLength];
            for (int i = 0; i < orig.Length; i++)
            {
                orig[i] = unchecked ((byte)(i % 10));
            }
            FSDataOutputStream fos = fs.Create(path, (short)1);

            fos.Write(orig);
            fos.Close();
            DFSTestUtil.WaitReplication(fs, path, ReplFactor);
            FSDataInputStream fis = cluster.GetFileSystem().Open(path);

            byte[] buf = new byte[FileLength];
            IOUtils.ReadFully(fis, buf, 0, FileLength);
            fis.Close();
            Assert.AssertArrayEquals(orig, buf);
            Arrays.Equals(orig, buf);
            cluster.Shutdown();
        }
Пример #3
0
        /// <summary>
        /// If we have a UNIX domain socket configured,
        /// and we have dfs.client.domain.socket.data.traffic set to true,
        /// and short-circuit access fails, we should still be able to pass
        /// data traffic over the UNIX domain socket.
        /// </summary>
        /// <remarks>
        /// If we have a UNIX domain socket configured,
        /// and we have dfs.client.domain.socket.data.traffic set to true,
        /// and short-circuit access fails, we should still be able to pass
        /// data traffic over the UNIX domain socket.  Test this.
        /// </remarks>
        /// <exception cref="System.Exception"/>
        public virtual void TestFallbackFromShortCircuitToUnixDomainTraffic()
        {
            DFSInputStream.tcpReadsDisabledForTesting = true;
            TemporarySocketDirectory sockDir = new TemporarySocketDirectory();
            // The server is NOT configured with short-circuit local reads;
            // the client is.  Both support UNIX domain reads.
            Configuration clientConf = CreateShortCircuitConf("testFallbackFromShortCircuitToUnixDomainTraffic"
                                                              , sockDir);

            clientConf.Set(DFSConfigKeys.DfsClientContext, "testFallbackFromShortCircuitToUnixDomainTraffic_clientContext"
                           );
            clientConf.SetBoolean(DFSConfigKeys.DfsClientDomainSocketDataTraffic, true);
            Configuration serverConf = new Configuration(clientConf);

            serverConf.SetBoolean(DFSConfigKeys.DfsClientReadShortcircuitKey, false);
            MiniDFSCluster cluster = new MiniDFSCluster.Builder(serverConf).NumDataNodes(1).Build
                                         ();

            cluster.WaitActive();
            FileSystem dfs         = FileSystem.Get(cluster.GetURI(0), clientConf);
            string     TestFile    = "/test_file";
            int        TestFileLen = 8193;
            int        Seed        = unchecked ((int)(0xFADED));

            DFSTestUtil.CreateFile(dfs, new Path(TestFile), TestFileLen, (short)1, Seed);
            byte[] contents = DFSTestUtil.ReadFileBuffer(dfs, new Path(TestFile));
            byte[] expected = DFSTestUtil.CalculateFileContentsFromSeed(Seed, TestFileLen);
            NUnit.Framework.Assert.IsTrue(Arrays.Equals(contents, expected));
            cluster.Shutdown();
            sockDir.Close();
        }
Пример #4
0
        public virtual void TestShortCircuitCacheShutdown()
        {
            TemporarySocketDirectory sockDir = new TemporarySocketDirectory();
            Configuration            conf    = CreateShortCircuitConf("testShortCircuitCacheShutdown", sockDir
                                                                      );

            conf.Set(DFSConfigKeys.DfsClientContext, "testShortCircuitCacheShutdown");
            Configuration serverConf = new Configuration(conf);

            DFSInputStream.tcpReadsDisabledForTesting = true;
            MiniDFSCluster cluster = new MiniDFSCluster.Builder(serverConf).NumDataNodes(1).Build
                                         ();

            cluster.WaitActive();
            DistributedFileSystem fs = (DistributedFileSystem)FileSystem.Get(cluster.GetURI(0
                                                                                            ), conf);
            string TestFile    = "/test_file";
            int    TestFileLen = 4000;
            int    Seed        = unchecked ((int)(0xFADEC));

            DFSTestUtil.CreateFile(fs, new Path(TestFile), TestFileLen, (short)1, Seed);
            byte[] contents = DFSTestUtil.ReadFileBuffer(fs, new Path(TestFile));
            byte[] expected = DFSTestUtil.CalculateFileContentsFromSeed(Seed, TestFileLen);
            NUnit.Framework.Assert.IsTrue(Arrays.Equals(contents, expected));
            ShortCircuitCache cache = fs.dfs.GetClientContext().GetShortCircuitCache();

            cache.Close();
            NUnit.Framework.Assert.IsTrue(cache.GetDfsClientShmManager().GetDomainSocketWatcher
                                              ().IsClosed());
            cluster.Shutdown();
            sockDir.Close();
        }
Пример #5
0
        /// <exception cref="System.Exception"/>
        public virtual void TestShmBasedStaleness()
        {
            BlockReaderTestUtil.EnableShortCircuitShmTracing();
            TemporarySocketDirectory sockDir = new TemporarySocketDirectory();
            Configuration            conf    = CreateShortCircuitConf("testShmBasedStaleness", sockDir);
            MiniDFSCluster           cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(1).Build();

            cluster.WaitActive();
            DistributedFileSystem fs    = cluster.GetFileSystem();
            ShortCircuitCache     cache = fs.GetClient().GetClientContext().GetShortCircuitCache(
                );
            string TestFile    = "/test_file";
            int    TestFileLen = 8193;
            int    Seed        = unchecked ((int)(0xFADED));

            DFSTestUtil.CreateFile(fs, new Path(TestFile), TestFileLen, (short)1, Seed);
            FSDataInputStream fis = fs.Open(new Path(TestFile));
            int           first   = fis.Read();
            ExtendedBlock block   = DFSTestUtil.GetFirstBlock(fs, new Path(TestFile));

            NUnit.Framework.Assert.IsTrue(first != -1);
            cache.Accept(new _CacheVisitor_502(block));
            // Stop the Namenode.  This will close the socket keeping the client's
            // shared memory segment alive, and make it stale.
            cluster.GetDataNodes()[0].Shutdown();
            cache.Accept(new _CacheVisitor_518(block));
            cluster.Shutdown();
            sockDir.Close();
        }
Пример #6
0
 public static void TeardownCluster()
 {
     if (DomainSocket.GetLoadingFailureReason() != null)
     {
         return;
     }
     sockDir.Close();
     TestParallelReadUtil.TeardownCluster();
 }
Пример #7
0
        // Regression test for HADOOP-11802
        /// <exception cref="System.Exception"/>
        public virtual void TestDataXceiverHandlesRequestShortCircuitShmFailure()
        {
            BlockReaderTestUtil.EnableShortCircuitShmTracing();
            TemporarySocketDirectory sockDir = new TemporarySocketDirectory();
            Configuration            conf    = CreateShortCircuitConf("testDataXceiverHandlesRequestShortCircuitShmFailure"
                                                                      , sockDir);

            conf.SetLong(DFSConfigKeys.DfsClientReadShortcircuitStreamsCacheExpiryMsKey, 1000000000L
                         );
            MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(1).Build();

            cluster.WaitActive();
            DistributedFileSystem fs = cluster.GetFileSystem();
            Path TestPath1           = new Path("/test_file1");

            DFSTestUtil.CreateFile(fs, TestPath1, 4096, (short)1, unchecked ((int)(0xFADE1)));
            Log.Info("Setting failure injector and performing a read which " + "should fail..."
                     );
            DataNodeFaultInjector failureInjector = Org.Mockito.Mockito.Mock <DataNodeFaultInjector
                                                                              >();

            Org.Mockito.Mockito.DoAnswer(new _Answer_710()).When(failureInjector).SendShortCircuitShmResponse
                ();
            DataNodeFaultInjector prevInjector = DataNodeFaultInjector.instance;

            DataNodeFaultInjector.instance = failureInjector;
            try
            {
                // The first read will try to allocate a shared memory segment and slot.
                // The shared memory segment allocation will fail because of the failure
                // injector.
                DFSTestUtil.ReadFileBuffer(fs, TestPath1);
                NUnit.Framework.Assert.Fail("expected readFileBuffer to fail, but it succeeded.");
            }
            catch (Exception t)
            {
                GenericTestUtils.AssertExceptionContains("TCP reads were disabled for " + "testing, but we failed to do a non-TCP read."
                                                         , t);
            }
            CheckNumberOfSegmentsAndSlots(0, 0, cluster.GetDataNodes()[0].GetShortCircuitRegistry
                                              ());
            Log.Info("Clearing failure injector and performing another read...");
            DataNodeFaultInjector.instance = prevInjector;
            fs.GetClient().GetClientContext().GetDomainSocketFactory().ClearPathMap();
            // The second read should succeed.
            DFSTestUtil.ReadFileBuffer(fs, TestPath1);
            // We should have added a new short-circuit shared memory segment and slot.
            CheckNumberOfSegmentsAndSlots(1, 1, cluster.GetDataNodes()[0].GetShortCircuitRegistry
                                              ());
            cluster.Shutdown();
            sockDir.Close();
        }
Пример #8
0
        /// <summary>
        /// Test the case where we have a failure to complete a short circuit read
        /// that occurs, and then later on, we have a success.
        /// </summary>
        /// <remarks>
        /// Test the case where we have a failure to complete a short circuit read
        /// that occurs, and then later on, we have a success.
        /// Any thread waiting on a cache load should receive the failure (if it
        /// occurs);  however, the failure result should not be cached.  We want
        /// to be able to retry later and succeed.
        /// </remarks>
        /// <exception cref="System.Exception"/>
        public virtual void TestShortCircuitCacheTemporaryFailure()
        {
            BlockReaderTestUtil.EnableBlockReaderFactoryTracing();
            AtomicBoolean replicaCreationShouldFail = new AtomicBoolean(true);
            AtomicBoolean testFailed = new AtomicBoolean(false);

            DFSInputStream.tcpReadsDisabledForTesting = true;
            BlockReaderFactory.createShortCircuitReplicaInfoCallback = new _ShortCircuitReplicaCreator_215
                                                                           (replicaCreationShouldFail);
            // Insert a short delay to increase the chance that one client
            // thread waits for the other client thread's failure via
            // a condition variable.
            TemporarySocketDirectory sockDir = new TemporarySocketDirectory();
            Configuration            conf    = CreateShortCircuitConf("testShortCircuitCacheTemporaryFailure"
                                                                      , sockDir);
            MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(1).Build();

            cluster.WaitActive();
            DistributedFileSystem dfs       = cluster.GetFileSystem();
            string         TestFile         = "/test_file";
            int            TestFileLen      = 4000;
            int            NumThreads       = 2;
            int            Seed             = unchecked ((int)(0xFADED));
            CountDownLatch gotFailureLatch  = new CountDownLatch(NumThreads);
            CountDownLatch shouldRetryLatch = new CountDownLatch(1);

            DFSTestUtil.CreateFile(dfs, new Path(TestFile), TestFileLen, (short)1, Seed);
            Runnable readerRunnable = new _Runnable_243(cluster, TestFile, TestFileLen, gotFailureLatch
                                                        , shouldRetryLatch, testFailed);

            // First time should fail.
            // first block
            // keep findbugs happy
            // Second time should succeed.
            Sharpen.Thread[] threads = new Sharpen.Thread[NumThreads];
            for (int i = 0; i < NumThreads; i++)
            {
                threads[i] = new Sharpen.Thread(readerRunnable);
                threads[i].Start();
            }
            gotFailureLatch.Await();
            replicaCreationShouldFail.Set(false);
            shouldRetryLatch.CountDown();
            for (int i_1 = 0; i_1 < NumThreads; i_1++)
            {
                Uninterruptibles.JoinUninterruptibly(threads[i_1]);
            }
            cluster.Shutdown();
            sockDir.Close();
            NUnit.Framework.Assert.IsFalse(testFailed.Get());
        }
Пример #9
0
        public virtual void TestCreateAndDestroySpanReceiver()
        {
            Configuration conf = new Configuration();

            conf = new Configuration();
            conf.Set(DFSConfigKeys.DfsServerHtracePrefix + SpanReceiverHost.SpanReceiversConfSuffix
                     , string.Empty);
            MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(3).Build();

            cluster.WaitActive();
            TemporarySocketDirectory tempDir = new TemporarySocketDirectory();
            string tracePath = new FilePath(tempDir.GetDir(), "tracefile").GetAbsolutePath();

            try
            {
                TraceAdmin trace = new TraceAdmin();
                trace.SetConf(conf);
                NUnit.Framework.Assert.AreEqual("ret:0, [no span receivers found]" + Newline, RunTraceCommand
                                                    (trace, "-list", "-host", GetHostPortForNN(cluster)));
                NUnit.Framework.Assert.AreEqual("ret:0, Added trace span receiver 1 with " + "configuration local-file-span-receiver.path = "
                                                + tracePath + Newline, RunTraceCommand(trace, "-add", "-host", GetHostPortForNN
                                                                                           (cluster), "-class", "org.apache.htrace.impl.LocalFileSpanReceiver", "-Clocal-file-span-receiver.path="
                                                                                       + tracePath));
                string list = RunTraceCommand(trace, "-list", "-host", GetHostPortForNN(cluster));
                NUnit.Framework.Assert.IsTrue(list.StartsWith("ret:0"));
                NUnit.Framework.Assert.IsTrue(list.Contains("1   org.apache.htrace.impl.LocalFileSpanReceiver"
                                                            ));
                NUnit.Framework.Assert.AreEqual("ret:0, Removed trace span receiver 1" + Newline,
                                                RunTraceCommand(trace, "-remove", "1", "-host", GetHostPortForNN(cluster)));
                NUnit.Framework.Assert.AreEqual("ret:0, [no span receivers found]" + Newline, RunTraceCommand
                                                    (trace, "-list", "-host", GetHostPortForNN(cluster)));
                NUnit.Framework.Assert.AreEqual("ret:0, Added trace span receiver 2 with " + "configuration local-file-span-receiver.path = "
                                                + tracePath + Newline, RunTraceCommand(trace, "-add", "-host", GetHostPortForNN
                                                                                           (cluster), "-class", "LocalFileSpanReceiver", "-Clocal-file-span-receiver.path="
                                                                                       + tracePath));
                NUnit.Framework.Assert.AreEqual("ret:0, Removed trace span receiver 2" + Newline,
                                                RunTraceCommand(trace, "-remove", "2", "-host", GetHostPortForNN(cluster)));
            }
            finally
            {
                cluster.Shutdown();
                tempDir.Close();
            }
        }
Пример #10
0
        /// <summary>Test unlinking a file whose blocks we are caching in the DFSClient.</summary>
        /// <remarks>
        /// Test unlinking a file whose blocks we are caching in the DFSClient.
        /// The DataNode will notify the DFSClient that the replica is stale via the
        /// ShortCircuitShm.
        /// </remarks>
        /// <exception cref="System.Exception"/>
        public virtual void TestUnlinkingReplicasInFileDescriptorCache()
        {
            BlockReaderTestUtil.EnableShortCircuitShmTracing();
            TemporarySocketDirectory sockDir = new TemporarySocketDirectory();
            Configuration            conf    = CreateShortCircuitConf("testUnlinkingReplicasInFileDescriptorCache"
                                                                      , sockDir);

            // We don't want the CacheCleaner to time out short-circuit shared memory
            // segments during the test, so set the timeout really high.
            conf.SetLong(DFSConfigKeys.DfsClientReadShortcircuitStreamsCacheExpiryMsKey, 1000000000L
                         );
            MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(1).Build();

            cluster.WaitActive();
            DistributedFileSystem fs    = cluster.GetFileSystem();
            ShortCircuitCache     cache = fs.GetClient().GetClientContext().GetShortCircuitCache(
                );

            cache.GetDfsClientShmManager().Visit(new _Visitor_556());
            // The ClientShmManager starts off empty.
            Path TestPath    = new Path("/test_file");
            int  TestFileLen = 8193;
            int  Seed        = unchecked ((int)(0xFADE0));

            DFSTestUtil.CreateFile(fs, TestPath, TestFileLen, (short)1, Seed);
            byte[] contents = DFSTestUtil.ReadFileBuffer(fs, TestPath);
            byte[] expected = DFSTestUtil.CalculateFileContentsFromSeed(Seed, TestFileLen);
            NUnit.Framework.Assert.IsTrue(Arrays.Equals(contents, expected));
            // Loading this file brought the ShortCircuitReplica into our local
            // replica cache.
            DatanodeInfo datanode = new DatanodeInfo(cluster.GetDataNodes()[0].GetDatanodeId(
                                                         ));

            cache.GetDfsClientShmManager().Visit(new _Visitor_577(datanode));
            // Remove the file whose blocks we just read.
            fs.Delete(TestPath, false);
            // Wait for the replica to be purged from the DFSClient's cache.
            GenericTestUtils.WaitFor(new _Supplier_593(this, cache, datanode), 10, 60000);
            // Check that all slots have been invalidated.
            cluster.Shutdown();
            sockDir.Close();
        }
Пример #11
0
        /// <summary>
        /// Test the case where we have multiple threads waiting on the
        /// ShortCircuitCache delivering a certain ShortCircuitReplica.
        /// </summary>
        /// <remarks>
        /// Test the case where we have multiple threads waiting on the
        /// ShortCircuitCache delivering a certain ShortCircuitReplica.
        /// In this case, there should only be one call to
        /// createShortCircuitReplicaInfo.  This one replica should be shared
        /// by all threads.
        /// </remarks>
        /// <exception cref="System.Exception"/>
        public virtual void TestMultipleWaitersOnShortCircuitCache()
        {
            CountDownLatch latch             = new CountDownLatch(1);
            AtomicBoolean  creationIsBlocked = new AtomicBoolean(true);
            AtomicBoolean  testFailed        = new AtomicBoolean(false);

            DFSInputStream.tcpReadsDisabledForTesting = true;
            BlockReaderFactory.createShortCircuitReplicaInfoCallback = new _ShortCircuitReplicaCreator_146
                                                                           (latch, creationIsBlocked);
            TemporarySocketDirectory sockDir = new TemporarySocketDirectory();
            Configuration            conf    = CreateShortCircuitConf("testMultipleWaitersOnShortCircuitCache"
                                                                      , sockDir);
            MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(1).Build();

            cluster.WaitActive();
            DistributedFileSystem dfs = cluster.GetFileSystem();
            string TestFile           = "/test_file";
            int    TestFileLen        = 4000;
            int    Seed       = unchecked ((int)(0xFADED));
            int    NumThreads = 10;

            DFSTestUtil.CreateFile(dfs, new Path(TestFile), TestFileLen, (short)1, Seed);
            Runnable readerRunnable = new _Runnable_170(dfs, TestFile, creationIsBlocked, Seed
                                                        , TestFileLen, testFailed);

            Sharpen.Thread[] threads = new Sharpen.Thread[NumThreads];
            for (int i = 0; i < NumThreads; i++)
            {
                threads[i] = new Sharpen.Thread(readerRunnable);
                threads[i].Start();
            }
            Sharpen.Thread.Sleep(500);
            latch.CountDown();
            for (int i_1 = 0; i_1 < NumThreads; i_1++)
            {
                Uninterruptibles.JoinUninterruptibly(threads[i_1]);
            }
            cluster.Shutdown();
            sockDir.Close();
            NUnit.Framework.Assert.IsFalse(testFailed.Get());
        }
Пример #12
0
        // Regression test for HDFS-7915
        /// <exception cref="System.Exception"/>
        public virtual void TestDataXceiverCleansUpSlotsOnFailure()
        {
            BlockReaderTestUtil.EnableShortCircuitShmTracing();
            TemporarySocketDirectory sockDir = new TemporarySocketDirectory();
            Configuration            conf    = CreateShortCircuitConf("testDataXceiverCleansUpSlotsOnFailure"
                                                                      , sockDir);

            conf.SetLong(DFSConfigKeys.DfsClientReadShortcircuitStreamsCacheExpiryMsKey, 1000000000L
                         );
            MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(1).Build();

            cluster.WaitActive();
            DistributedFileSystem fs = cluster.GetFileSystem();
            Path TestPath1           = new Path("/test_file1");
            Path TestPath2           = new Path("/test_file2");
            int  TestFileLen         = 4096;
            int  Seed = unchecked ((int)(0xFADE1));

            DFSTestUtil.CreateFile(fs, TestPath1, TestFileLen, (short)1, Seed);
            DFSTestUtil.CreateFile(fs, TestPath2, TestFileLen, (short)1, Seed);
            // The first read should allocate one shared memory segment and slot.
            DFSTestUtil.ReadFileBuffer(fs, TestPath1);
            // The second read should fail, and we should only have 1 segment and 1 slot
            // left.
            fs.GetClient().GetConf().brfFailureInjector = new TestShortCircuitCache.TestCleanupFailureInjector
                                                              ();
            try
            {
                DFSTestUtil.ReadFileBuffer(fs, TestPath2);
            }
            catch (Exception t)
            {
                GenericTestUtils.AssertExceptionContains("TCP reads were disabled for " + "testing, but we failed to do a non-TCP read."
                                                         , t);
            }
            CheckNumberOfSegmentsAndSlots(1, 1, cluster.GetDataNodes()[0].GetShortCircuitRegistry
                                              ());
            cluster.Shutdown();
            sockDir.Close();
        }
Пример #13
0
        /// <exception cref="System.Exception"/>
        public virtual void TestAllocShm()
        {
            BlockReaderTestUtil.EnableShortCircuitShmTracing();
            TemporarySocketDirectory sockDir = new TemporarySocketDirectory();
            Configuration            conf    = CreateShortCircuitConf("testAllocShm", sockDir);
            MiniDFSCluster           cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(1).Build();

            cluster.WaitActive();
            DistributedFileSystem fs    = cluster.GetFileSystem();
            ShortCircuitCache     cache = fs.GetClient().GetClientContext().GetShortCircuitCache(
                );

            cache.GetDfsClientShmManager().Visit(new _Visitor_423());
            // The ClientShmManager starts off empty
            DomainPeer      peer     = GetDomainPeerToDn(conf);
            MutableBoolean  usedPeer = new MutableBoolean(false);
            ExtendedBlockId blockId  = new ExtendedBlockId(123, "xyz");
            DatanodeInfo    datanode = new DatanodeInfo(cluster.GetDataNodes()[0].GetDatanodeId(
                                                            ));

            // Allocating the first shm slot requires using up a peer.
            ShortCircuitShm.Slot slot = cache.AllocShmSlot(datanode, peer, usedPeer, blockId,
                                                           "testAllocShm_client");
            NUnit.Framework.Assert.IsNotNull(slot);
            NUnit.Framework.Assert.IsTrue(usedPeer.BooleanValue());
            cache.GetDfsClientShmManager().Visit(new _Visitor_441(datanode));
            // The ClientShmManager starts off empty
            cache.ScheduleSlotReleaser(slot);
            // Wait for the slot to be released, and the shared memory area to be
            // closed.  Since we didn't register this shared memory segment on the
            // server, it will also be a test of how well the server deals with
            // bogus client behavior.
            GenericTestUtils.WaitFor(new _Supplier_458(cache, datanode), 10, 60000);
            cluster.Shutdown();
            sockDir.Close();
        }
Пример #14
0
 public static void Shutdown()
 {
     sockDir.Close();
 }
Пример #15
0
        /// <summary>
        /// When an InterruptedException is sent to a thread calling
        /// FileChannel#read, the FileChannel is immediately closed and the
        /// thread gets an exception.
        /// </summary>
        /// <remarks>
        /// When an InterruptedException is sent to a thread calling
        /// FileChannel#read, the FileChannel is immediately closed and the
        /// thread gets an exception.  This effectively means that we might have
        /// someone asynchronously calling close() on the file descriptors we use
        /// in BlockReaderLocal.  So when unreferencing a ShortCircuitReplica in
        /// ShortCircuitCache#unref, we should check if the FileChannel objects
        /// are still open.  If not, we should purge the replica to avoid giving
        /// it out to any future readers.
        /// This is a regression test for HDFS-6227: Short circuit read failed
        /// due to ClosedChannelException.
        /// Note that you may still get ClosedChannelException errors if two threads
        /// are reading from the same replica and an InterruptedException is delivered
        /// to one of them.
        /// </remarks>
        /// <exception cref="System.Exception"/>
        public virtual void TestPurgingClosedReplicas()
        {
            BlockReaderTestUtil.EnableBlockReaderFactoryTracing();
            AtomicInteger replicasCreated = new AtomicInteger(0);
            AtomicBoolean testFailed      = new AtomicBoolean(false);

            DFSInputStream.tcpReadsDisabledForTesting = true;
            BlockReaderFactory.createShortCircuitReplicaInfoCallback = new _ShortCircuitReplicaCreator_443
                                                                           (replicasCreated);
            TemporarySocketDirectory sockDir = new TemporarySocketDirectory();
            Configuration            conf    = CreateShortCircuitConf("testPurgingClosedReplicas", sockDir);
            MiniDFSCluster           cluster = new MiniDFSCluster.Builder(conf).NumDataNodes(1).Build();

            cluster.WaitActive();
            DistributedFileSystem dfs = cluster.GetFileSystem();
            string TestFile           = "/test_file";
            int    TestFileLen        = 4095;
            int    Seed = unchecked ((int)(0xFADE0));
            DistributedFileSystem fs = (DistributedFileSystem)FileSystem.Get(cluster.GetURI(0
                                                                                            ), conf);

            DFSTestUtil.CreateFile(fs, new Path(TestFile), TestFileLen, (short)1, Seed);
            Semaphore            sem           = Sharpen.Extensions.CreateSemaphore(0);
            IList <LocatedBlock> locatedBlocks = cluster.GetNameNode().GetRpcServer().GetBlockLocations
                                                     (TestFile, 0, TestFileLen).GetLocatedBlocks();
            LocatedBlock lblock = locatedBlocks[0];

            // first block
            byte[]   buf            = new byte[TestFileLen];
            Runnable readerRunnable = new _Runnable_471(cluster, lblock, TestFileLen, sem, buf
                                                        , testFailed);

            Sharpen.Thread thread = new Sharpen.Thread(readerRunnable);
            thread.Start();
            // While the thread is reading, send it interrupts.
            // These should trigger a ClosedChannelException.
            while (thread.IsAlive())
            {
                sem.AcquireUninterruptibly();
                thread.Interrupt();
                sem.Release();
            }
            NUnit.Framework.Assert.IsFalse(testFailed.Get());
            // We should be able to read from the file without
            // getting a ClosedChannelException.
            BlockReader blockReader = null;

            try
            {
                blockReader = BlockReaderTestUtil.GetBlockReader(cluster, lblock, 0, TestFileLen);
                blockReader.ReadFully(buf, 0, TestFileLen);
            }
            finally
            {
                if (blockReader != null)
                {
                    blockReader.Close();
                }
            }
            byte[] expected = DFSTestUtil.CalculateFileContentsFromSeed(Seed, TestFileLen);
            NUnit.Framework.Assert.IsTrue(Arrays.Equals(buf, expected));
            // Another ShortCircuitReplica object should have been created.
            NUnit.Framework.Assert.AreEqual(2, replicasCreated.Get());
            dfs.Close();
            cluster.Shutdown();
            sockDir.Close();
        }
 public virtual void TearDown()
 {
     // Restore the original CacheManipulator
     NativeIO.POSIX.SetCacheManipulator(prevCacheManipulator);
     sockDir.Close();
 }
Пример #17
0
 /// <exception cref="System.IO.IOException"/>
 public virtual void Close()
 {
     IOUtils.Cleanup(Log, fis);
     dir.Close();
 }