Пример #1
0
        /// <summary>
        /// Regression test for HDFS-2693: when doing state transitions, we need to
        /// lock the FSNamesystem so that we don't end up doing any writes while it's
        /// "in between" states.
        /// </summary>
        /// <remarks>
        /// Regression test for HDFS-2693: when doing state transitions, we need to
        /// lock the FSNamesystem so that we don't end up doing any writes while it's
        /// "in between" states.
        /// This test case starts up several client threads which do mutation operations
        /// while flipping a NN back and forth from active to standby.
        /// </remarks>
        /// <exception cref="System.Exception"/>
        public virtual void TestTransitionSynchronization()
        {
            Configuration  conf    = new Configuration();
            MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NnTopology(MiniDFSNNTopology
                                                                                 .SimpleHATopology()).NumDataNodes(0).Build();

            try
            {
                cluster.WaitActive();
                ReentrantReadWriteLock spyLock = NameNodeAdapter.SpyOnFsLock(cluster.GetNameNode(
                                                                                 0).GetNamesystem());
                Org.Mockito.Mockito.DoAnswer(new GenericTestUtils.SleepAnswer(50)).When(spyLock).
                WriteLock();
                FileSystem fs = HATestUtil.ConfigureFailoverFs(cluster, conf);
                MultithreadedTestUtil.TestContext ctx = new MultithreadedTestUtil.TestContext();
                for (int i = 0; i < 50; i++)
                {
                    int finalI = i;
                    ctx.AddThread(new _RepeatingTestThread_256(finalI, fs, ctx));
                }
                ctx.AddThread(new _RepeatingTestThread_266(cluster, ctx));
                ctx.StartThreads();
                ctx.WaitFor(20000);
                ctx.Stop();
            }
            finally
            {
                cluster.Shutdown();
            }
        }
        public virtual void TestFencingStress()
        {
            HAStressTestHarness harness = new HAStressTestHarness();

            harness.conf.SetInt(DFSConfigKeys.DfsBlockreportIntervalMsecKey, 1000);
            harness.conf.SetInt(DFSConfigKeys.DfsNamenodeHeartbeatRecheckIntervalKey, 1);
            harness.conf.SetInt(DFSConfigKeys.DfsNamenodeReplicationIntervalKey, 1);
            MiniDFSCluster cluster = harness.StartCluster();

            try
            {
                cluster.WaitActive();
                cluster.TransitionToActive(0);
                FileSystem fs = harness.GetFailoverFs();
                MultithreadedTestUtil.TestContext togglers = new MultithreadedTestUtil.TestContext
                                                                 ();
                for (int i = 0; i < NumThreads; i++)
                {
                    Path p = new Path("/test-" + i);
                    DFSTestUtil.CreateFile(fs, p, BlockSize * 10, (short)3, (long)i);
                    togglers.AddThread(new TestDNFencingWithReplication.ReplicationToggler(togglers,
                                                                                           fs, p));
                }
                // Start a separate thread which will make sure that replication
                // happens quickly by triggering deletion reports and replication
                // work calculation frequently.
                harness.AddReplicationTriggerThread(500);
                harness.AddFailoverThread(5000);
                harness.StartThreads();
                togglers.StartThreads();
                togglers.WaitFor(Runtime);
                togglers.Stop();
                harness.StopThreads();
                // CHeck that the files can be read without throwing
                for (int i_1 = 0; i_1 < NumThreads; i_1++)
                {
                    Path p = new Path("/test-" + i_1);
                    DFSTestUtil.ReadFile(fs, p);
                }
            }
            finally
            {
                System.Console.Error.WriteLine("===========================\n\n\n\n");
                harness.Shutdown();
            }
        }
Пример #3
0
        /// <summary>Stress test for pipeline/lease recovery.</summary>
        /// <remarks>
        /// Stress test for pipeline/lease recovery. Starts a number of
        /// threads, each of which creates a file and has another client
        /// break the lease. While these threads run, failover proceeds
        /// back and forth between two namenodes.
        /// </remarks>
        /// <exception cref="System.Exception"/>
        public virtual void TestPipelineRecoveryStress()
        {
            HAStressTestHarness harness = new HAStressTestHarness();

            // Disable permissions so that another user can recover the lease.
            harness.conf.SetBoolean(DFSConfigKeys.DfsPermissionsEnabledKey, false);
            // This test triggers rapid NN failovers.  The client retry policy uses an
            // exponential backoff.  This can quickly lead to long sleep times and even
            // timeout the whole test.  Cap the sleep time at 1s to prevent this.
            harness.conf.SetInt(DFSConfigKeys.DfsClientFailoverSleeptimeMaxKey, 1000);
            MiniDFSCluster cluster = harness.StartCluster();

            try
            {
                cluster.WaitActive();
                cluster.TransitionToActive(0);
                FileSystem                        fs            = harness.GetFailoverFs();
                DistributedFileSystem             fsAsOtherUser = CreateFsAsOtherUser(cluster, harness.conf);
                MultithreadedTestUtil.TestContext testers       = new MultithreadedTestUtil.TestContext
                                                                      ();
                for (int i = 0; i < StressNumThreads; i++)
                {
                    Path p = new Path("/test-" + i);
                    testers.AddThread(new TestPipelinesFailover.PipelineTestThread(testers, fs, fsAsOtherUser
                                                                                   , p));
                }
                // Start a separate thread which will make sure that replication
                // happens quickly by triggering deletion reports and replication
                // work calculation frequently.
                harness.AddReplicationTriggerThread(500);
                harness.AddFailoverThread(5000);
                harness.StartThreads();
                testers.StartThreads();
                testers.WaitFor(StressRuntime);
                testers.Stop();
                harness.StopThreads();
            }
            finally
            {
                System.Console.Error.WriteLine("===========================\n\n\n\n");
                harness.Shutdown();
            }
        }
Пример #4
0
 /// <exception cref="System.Exception"/>
 public virtual int Run(string[] args)
 {
     RPCCallBenchmark.MyOptions opts = new RPCCallBenchmark.MyOptions(args);
     if (opts.failed)
     {
         return(-1);
     }
     // Set RPC engine to the configured RPC engine
     RPC.SetProtocolEngine(conf, typeof(TestProtoBufRpc.TestRpcService), opts.rpcEngine
                           );
     RPC.Server server = StartServer(opts);
     try
     {
         MultithreadedTestUtil.TestContext ctx = SetupClientTestContext(opts);
         if (ctx != null)
         {
             long totalCalls = 0;
             ctx.StartThreads();
             long veryStart = Runtime.NanoTime();
             // Loop printing results every second until the specified
             // time has elapsed
             for (int i = 0; i < opts.secondsToRun; i++)
             {
                 long st = Runtime.NanoTime();
                 ctx.WaitFor(1000);
                 long et = Runtime.NanoTime();
                 long ct = callCount.GetAndSet(0);
                 totalCalls += ct;
                 double callsPerSec = (ct * 1000000000) / (et - st);
                 System.Console.Out.WriteLine("Calls per second: " + callsPerSec);
             }
             // Print results
             if (totalCalls > 0)
             {
                 long   veryEnd        = Runtime.NanoTime();
                 double callsPerSec    = (totalCalls * 1000000000) / (veryEnd - veryStart);
                 long   cpuNanosClient = GetTotalCpuTime(ctx.GetTestThreads());
                 long   cpuNanosServer = -1;
                 if (server != null)
                 {
                     cpuNanosServer = GetTotalCpuTime(server.GetHandlers());
                 }
                 System.Console.Out.WriteLine("====== Results ======");
                 System.Console.Out.WriteLine("Options:\n" + opts);
                 System.Console.Out.WriteLine("Total calls per second: " + callsPerSec);
                 System.Console.Out.WriteLine("CPU time per call on client: " + (cpuNanosClient /
                                                                                 totalCalls) + " ns");
                 if (server != null)
                 {
                     System.Console.Out.WriteLine("CPU time per call on server: " + (cpuNanosServer /
                                                                                     totalCalls) + " ns");
                 }
             }
             else
             {
                 System.Console.Out.WriteLine("No calls!");
             }
             ctx.Stop();
         }
         else
         {
             while (true)
             {
                 Thread.Sleep(10000);
             }
         }
     }
     finally
     {
         if (server != null)
         {
             server.Stop();
         }
     }
     return(0);
 }