/// <summary> /// Regression test for HDFS-2693: when doing state transitions, we need to /// lock the FSNamesystem so that we don't end up doing any writes while it's /// "in between" states. /// </summary> /// <remarks> /// Regression test for HDFS-2693: when doing state transitions, we need to /// lock the FSNamesystem so that we don't end up doing any writes while it's /// "in between" states. /// This test case starts up several client threads which do mutation operations /// while flipping a NN back and forth from active to standby. /// </remarks> /// <exception cref="System.Exception"/> public virtual void TestTransitionSynchronization() { Configuration conf = new Configuration(); MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NnTopology(MiniDFSNNTopology .SimpleHATopology()).NumDataNodes(0).Build(); try { cluster.WaitActive(); ReentrantReadWriteLock spyLock = NameNodeAdapter.SpyOnFsLock(cluster.GetNameNode( 0).GetNamesystem()); Org.Mockito.Mockito.DoAnswer(new GenericTestUtils.SleepAnswer(50)).When(spyLock). WriteLock(); FileSystem fs = HATestUtil.ConfigureFailoverFs(cluster, conf); MultithreadedTestUtil.TestContext ctx = new MultithreadedTestUtil.TestContext(); for (int i = 0; i < 50; i++) { int finalI = i; ctx.AddThread(new _RepeatingTestThread_256(finalI, fs, ctx)); } ctx.AddThread(new _RepeatingTestThread_266(cluster, ctx)); ctx.StartThreads(); ctx.WaitFor(20000); ctx.Stop(); } finally { cluster.Shutdown(); } }
public virtual void Setup() { conf = new Configuration(); // Specify the quorum per-nameservice, to ensure that these configs // can be nameservice-scoped. conf.Set(ZKFailoverController.ZkQuorumKey + ".ns1", hostPort); conf.Set(DFSConfigKeys.DfsHaFenceMethodsKey, typeof(TestNodeFencer.AlwaysSucceedFencer ).FullName); conf.SetBoolean(DFSConfigKeys.DfsHaAutoFailoverEnabledKey, true); // Turn off IPC client caching, so that the suite can handle // the restart of the daemons between test cases. conf.SetInt(CommonConfigurationKeysPublic.IpcClientConnectionMaxidletimeKey, 0); conf.SetInt(DFSConfigKeys.DfsHaZkfcPortKey + ".ns1.nn1", 10023); conf.SetInt(DFSConfigKeys.DfsHaZkfcPortKey + ".ns1.nn2", 10024); MiniDFSNNTopology topology = new MiniDFSNNTopology().AddNameservice(new MiniDFSNNTopology.NSConf ("ns1").AddNN(new MiniDFSNNTopology.NNConf("nn1").SetIpcPort(10021)).AddNN(new MiniDFSNNTopology.NNConf ("nn2").SetIpcPort(10022))); cluster = new MiniDFSCluster.Builder(conf).NnTopology(topology).NumDataNodes(0).Build (); cluster.WaitActive(); ctx = new MultithreadedTestUtil.TestContext(); ctx.AddThread(thr1 = new TestDFSZKFailoverController.ZKFCThread(this, ctx, 0)); NUnit.Framework.Assert.AreEqual(0, thr1.zkfc.Run(new string[] { "-formatZK" })); thr1.Start(); WaitForHAState(0, HAServiceProtocol.HAServiceState.Active); ctx.AddThread(thr2 = new TestDFSZKFailoverController.ZKFCThread(this, ctx, 1)); thr2.Start(); // Wait for the ZKFCs to fully start up ZKFCTestUtil.WaitForHealthState(thr1.zkfc, HealthMonitor.State.ServiceHealthy, ctx ); ZKFCTestUtil.WaitForHealthState(thr2.zkfc, HealthMonitor.State.ServiceHealthy, ctx ); fs = HATestUtil.ConfigureFailoverFs(cluster, conf); }
public virtual void TestFencingStress() { HAStressTestHarness harness = new HAStressTestHarness(); harness.conf.SetInt(DFSConfigKeys.DfsBlockreportIntervalMsecKey, 1000); harness.conf.SetInt(DFSConfigKeys.DfsNamenodeHeartbeatRecheckIntervalKey, 1); harness.conf.SetInt(DFSConfigKeys.DfsNamenodeReplicationIntervalKey, 1); MiniDFSCluster cluster = harness.StartCluster(); try { cluster.WaitActive(); cluster.TransitionToActive(0); FileSystem fs = harness.GetFailoverFs(); MultithreadedTestUtil.TestContext togglers = new MultithreadedTestUtil.TestContext (); for (int i = 0; i < NumThreads; i++) { Path p = new Path("/test-" + i); DFSTestUtil.CreateFile(fs, p, BlockSize * 10, (short)3, (long)i); togglers.AddThread(new TestDNFencingWithReplication.ReplicationToggler(togglers, fs, p)); } // Start a separate thread which will make sure that replication // happens quickly by triggering deletion reports and replication // work calculation frequently. harness.AddReplicationTriggerThread(500); harness.AddFailoverThread(5000); harness.StartThreads(); togglers.StartThreads(); togglers.WaitFor(Runtime); togglers.Stop(); harness.StopThreads(); // CHeck that the files can be read without throwing for (int i_1 = 0; i_1 < NumThreads; i_1++) { Path p = new Path("/test-" + i_1); DFSTestUtil.ReadFile(fs, p); } } finally { System.Console.Error.WriteLine("===========================\n\n\n\n"); harness.Shutdown(); } }
/// <summary>Stress test for pipeline/lease recovery.</summary> /// <remarks> /// Stress test for pipeline/lease recovery. Starts a number of /// threads, each of which creates a file and has another client /// break the lease. While these threads run, failover proceeds /// back and forth between two namenodes. /// </remarks> /// <exception cref="System.Exception"/> public virtual void TestPipelineRecoveryStress() { HAStressTestHarness harness = new HAStressTestHarness(); // Disable permissions so that another user can recover the lease. harness.conf.SetBoolean(DFSConfigKeys.DfsPermissionsEnabledKey, false); // This test triggers rapid NN failovers. The client retry policy uses an // exponential backoff. This can quickly lead to long sleep times and even // timeout the whole test. Cap the sleep time at 1s to prevent this. harness.conf.SetInt(DFSConfigKeys.DfsClientFailoverSleeptimeMaxKey, 1000); MiniDFSCluster cluster = harness.StartCluster(); try { cluster.WaitActive(); cluster.TransitionToActive(0); FileSystem fs = harness.GetFailoverFs(); DistributedFileSystem fsAsOtherUser = CreateFsAsOtherUser(cluster, harness.conf); MultithreadedTestUtil.TestContext testers = new MultithreadedTestUtil.TestContext (); for (int i = 0; i < StressNumThreads; i++) { Path p = new Path("/test-" + i); testers.AddThread(new TestPipelinesFailover.PipelineTestThread(testers, fs, fsAsOtherUser , p)); } // Start a separate thread which will make sure that replication // happens quickly by triggering deletion reports and replication // work calculation frequently. harness.AddReplicationTriggerThread(500); harness.AddFailoverThread(5000); harness.StartThreads(); testers.StartThreads(); testers.WaitFor(StressRuntime); testers.Stop(); harness.StopThreads(); } finally { System.Console.Error.WriteLine("===========================\n\n\n\n"); harness.Shutdown(); } }
/// <exception cref="System.IO.IOException"/> /// <exception cref="System.Exception"/> private MultithreadedTestUtil.TestContext SetupClientTestContext(RPCCallBenchmark.MyOptions opts) { if (opts.clientThreads <= 0) { return(null); } // Set up a separate proxy for each client thread, // rather than making them share TCP pipes. int numProxies = opts.clientThreads; RPCCallBenchmark.RpcServiceWrapper[] proxies = new RPCCallBenchmark.RpcServiceWrapper [numProxies]; for (int i = 0; i < numProxies; i++) { proxies[i] = UserGroupInformation.CreateUserForTesting("proxy-" + i, new string[] { }).DoAs(new _PrivilegedExceptionAction_347(this, opts)); } // Create an echo message of the desired length StringBuilder msgBuilder = new StringBuilder(opts.msgSize); for (int c = 0; c < opts.msgSize; c++) { msgBuilder.Append('x'); } string echoMessage = msgBuilder.ToString(); // Create the clients in a test context MultithreadedTestUtil.TestContext ctx = new MultithreadedTestUtil.TestContext(); for (int i_1 = 0; i_1 < opts.clientThreads; i_1++) { RPCCallBenchmark.RpcServiceWrapper proxy = proxies[i_1 % numProxies]; ctx.AddThread(new _RepeatingTestThread_367(this, proxy, echoMessage, ctx)); } return(ctx); }
/// <summary> /// Add a thread which periodically triggers deletion reports, /// heartbeats, and NN-side block work. /// </summary> /// <param name="interval">millisecond period on which to run</param> public virtual void AddReplicationTriggerThread(int interval) { testCtx.AddThread(new _RepeatingTestThread_83(this, interval, testCtx)); }