/// <summary> /// Regression test for HDFS-2693: when doing state transitions, we need to /// lock the FSNamesystem so that we don't end up doing any writes while it's /// "in between" states. /// </summary> /// <remarks> /// Regression test for HDFS-2693: when doing state transitions, we need to /// lock the FSNamesystem so that we don't end up doing any writes while it's /// "in between" states. /// This test case starts up several client threads which do mutation operations /// while flipping a NN back and forth from active to standby. /// </remarks> /// <exception cref="System.Exception"/> public virtual void TestTransitionSynchronization() { Configuration conf = new Configuration(); MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NnTopology(MiniDFSNNTopology .SimpleHATopology()).NumDataNodes(0).Build(); try { cluster.WaitActive(); ReentrantReadWriteLock spyLock = NameNodeAdapter.SpyOnFsLock(cluster.GetNameNode( 0).GetNamesystem()); Org.Mockito.Mockito.DoAnswer(new GenericTestUtils.SleepAnswer(50)).When(spyLock). WriteLock(); FileSystem fs = HATestUtil.ConfigureFailoverFs(cluster, conf); MultithreadedTestUtil.TestContext ctx = new MultithreadedTestUtil.TestContext(); for (int i = 0; i < 50; i++) { int finalI = i; ctx.AddThread(new _RepeatingTestThread_256(finalI, fs, ctx)); } ctx.AddThread(new _RepeatingTestThread_266(cluster, ctx)); ctx.StartThreads(); ctx.WaitFor(20000); ctx.Stop(); } finally { cluster.Shutdown(); } }
public _RepeatingTestThread_256(int finalI, FileSystem fs, MultithreadedTestUtil.TestContext baseArg1) : base(baseArg1) { this.finalI = finalI; this.fs = fs; }
public _RepeatingTestThread_106(HAStressTestHarness _enclosing, int msBetweenFailovers , MultithreadedTestUtil.TestContext baseArg1) : base(baseArg1) { this._enclosing = _enclosing; this.msBetweenFailovers = msBetweenFailovers; }
public DummyZKFCThread(MiniZKFCCluster _enclosing, MultithreadedTestUtil.TestContext ctx, DummyHAService svc) : base(ctx) { this._enclosing = _enclosing; this.zkfc = new MiniZKFCCluster.DummyZKFC(this._enclosing.conf, svc); }
public virtual void Setup() { conf = new Configuration(); // Specify the quorum per-nameservice, to ensure that these configs // can be nameservice-scoped. conf.Set(ZKFailoverController.ZkQuorumKey + ".ns1", hostPort); conf.Set(DFSConfigKeys.DfsHaFenceMethodsKey, typeof(TestNodeFencer.AlwaysSucceedFencer ).FullName); conf.SetBoolean(DFSConfigKeys.DfsHaAutoFailoverEnabledKey, true); // Turn off IPC client caching, so that the suite can handle // the restart of the daemons between test cases. conf.SetInt(CommonConfigurationKeysPublic.IpcClientConnectionMaxidletimeKey, 0); conf.SetInt(DFSConfigKeys.DfsHaZkfcPortKey + ".ns1.nn1", 10023); conf.SetInt(DFSConfigKeys.DfsHaZkfcPortKey + ".ns1.nn2", 10024); MiniDFSNNTopology topology = new MiniDFSNNTopology().AddNameservice(new MiniDFSNNTopology.NSConf ("ns1").AddNN(new MiniDFSNNTopology.NNConf("nn1").SetIpcPort(10021)).AddNN(new MiniDFSNNTopology.NNConf ("nn2").SetIpcPort(10022))); cluster = new MiniDFSCluster.Builder(conf).NnTopology(topology).NumDataNodes(0).Build (); cluster.WaitActive(); ctx = new MultithreadedTestUtil.TestContext(); ctx.AddThread(thr1 = new TestDFSZKFailoverController.ZKFCThread(this, ctx, 0)); NUnit.Framework.Assert.AreEqual(0, thr1.zkfc.Run(new string[] { "-formatZK" })); thr1.Start(); WaitForHAState(0, HAServiceProtocol.HAServiceState.Active); ctx.AddThread(thr2 = new TestDFSZKFailoverController.ZKFCThread(this, ctx, 1)); thr2.Start(); // Wait for the ZKFCs to fully start up ZKFCTestUtil.WaitForHealthState(thr1.zkfc, HealthMonitor.State.ServiceHealthy, ctx ); ZKFCTestUtil.WaitForHealthState(thr2.zkfc, HealthMonitor.State.ServiceHealthy, ctx ); fs = HATestUtil.ConfigureFailoverFs(cluster, conf); }
public ZKFCThread(TestDFSZKFailoverController _enclosing, MultithreadedTestUtil.TestContext ctx, int idx) : base(ctx) { this._enclosing = _enclosing; this.zkfc = DFSZKFailoverController.Create(this._enclosing.cluster.GetConfiguration (idx)); }
public _RepeatingTestThread_367(RPCCallBenchmark _enclosing, RPCCallBenchmark.RpcServiceWrapper proxy, string echoMessage, MultithreadedTestUtil.TestContext baseArg1) : base(baseArg1) { this._enclosing = _enclosing; this.proxy = proxy; this.echoMessage = echoMessage; }
public PipelineTestThread(MultithreadedTestUtil.TestContext ctx, FileSystem fs, FileSystem fsOtherUser, Path p) : base(ctx) { this.fs = fs; this.fsOtherUser = fsOtherUser; this.path = p; }
public ReplicationToggler(MultithreadedTestUtil.TestContext ctx, FileSystem fs, Path p) : base(ctx) { // How long should the test try to run for. In practice // it runs for ~20-30s longer than this constant due to startup/ // shutdown time. this.fs = fs; this.path = p; }
/// <exception cref="System.Exception"/> public static void WaitForHealthState(ZKFailoverController zkfc, HealthMonitor.State state, MultithreadedTestUtil.TestContext ctx) { while (zkfc.GetLastHealthState() != state) { if (ctx != null) { ctx.CheckException(); } Thread.Sleep(50); } }
/// <exception cref="System.Exception"/> public static void WaitForElectorState(MultithreadedTestUtil.TestContext ctx, ActiveStandbyElector elector, ActiveStandbyElector.State state) { while (elector.GetStateForTests() != state) { if (ctx != null) { ctx.CheckException(); } Thread.Sleep(50); } }
public MiniZKFCCluster(Configuration conf, ZooKeeperServer zks) { this.conf = conf; // Fast check interval so tests run faster conf.SetInt(CommonConfigurationKeys.HaHmCheckIntervalKey, 50); conf.SetInt(CommonConfigurationKeys.HaHmConnectRetryIntervalKey, 50); conf.SetInt(CommonConfigurationKeys.HaHmSleepAfterDisconnectKey, 50); svcs = new DummyHAService[2]; svcs[0] = new DummyHAService(HAServiceProtocol.HAServiceState.Initializing, new IPEndPoint ("svc1", 1234)); svcs[0].SetSharedResource(sharedResource); svcs[1] = new DummyHAService(HAServiceProtocol.HAServiceState.Initializing, new IPEndPoint ("svc2", 1234)); svcs[1].SetSharedResource(sharedResource); this.ctx = new MultithreadedTestUtil.TestContext(); this.zks = zks; }
public virtual void TestFencingStress() { HAStressTestHarness harness = new HAStressTestHarness(); harness.conf.SetInt(DFSConfigKeys.DfsBlockreportIntervalMsecKey, 1000); harness.conf.SetInt(DFSConfigKeys.DfsNamenodeHeartbeatRecheckIntervalKey, 1); harness.conf.SetInt(DFSConfigKeys.DfsNamenodeReplicationIntervalKey, 1); MiniDFSCluster cluster = harness.StartCluster(); try { cluster.WaitActive(); cluster.TransitionToActive(0); FileSystem fs = harness.GetFailoverFs(); MultithreadedTestUtil.TestContext togglers = new MultithreadedTestUtil.TestContext (); for (int i = 0; i < NumThreads; i++) { Path p = new Path("/test-" + i); DFSTestUtil.CreateFile(fs, p, BlockSize * 10, (short)3, (long)i); togglers.AddThread(new TestDNFencingWithReplication.ReplicationToggler(togglers, fs, p)); } // Start a separate thread which will make sure that replication // happens quickly by triggering deletion reports and replication // work calculation frequently. harness.AddReplicationTriggerThread(500); harness.AddFailoverThread(5000); harness.StartThreads(); togglers.StartThreads(); togglers.WaitFor(Runtime); togglers.Stop(); harness.StopThreads(); // CHeck that the files can be read without throwing for (int i_1 = 0; i_1 < NumThreads; i_1++) { Path p = new Path("/test-" + i_1); DFSTestUtil.ReadFile(fs, p); } } finally { System.Console.Error.WriteLine("===========================\n\n\n\n"); harness.Shutdown(); } }
/// <summary>Stress test for pipeline/lease recovery.</summary> /// <remarks> /// Stress test for pipeline/lease recovery. Starts a number of /// threads, each of which creates a file and has another client /// break the lease. While these threads run, failover proceeds /// back and forth between two namenodes. /// </remarks> /// <exception cref="System.Exception"/> public virtual void TestPipelineRecoveryStress() { HAStressTestHarness harness = new HAStressTestHarness(); // Disable permissions so that another user can recover the lease. harness.conf.SetBoolean(DFSConfigKeys.DfsPermissionsEnabledKey, false); // This test triggers rapid NN failovers. The client retry policy uses an // exponential backoff. This can quickly lead to long sleep times and even // timeout the whole test. Cap the sleep time at 1s to prevent this. harness.conf.SetInt(DFSConfigKeys.DfsClientFailoverSleeptimeMaxKey, 1000); MiniDFSCluster cluster = harness.StartCluster(); try { cluster.WaitActive(); cluster.TransitionToActive(0); FileSystem fs = harness.GetFailoverFs(); DistributedFileSystem fsAsOtherUser = CreateFsAsOtherUser(cluster, harness.conf); MultithreadedTestUtil.TestContext testers = new MultithreadedTestUtil.TestContext (); for (int i = 0; i < StressNumThreads; i++) { Path p = new Path("/test-" + i); testers.AddThread(new TestPipelinesFailover.PipelineTestThread(testers, fs, fsAsOtherUser , p)); } // Start a separate thread which will make sure that replication // happens quickly by triggering deletion reports and replication // work calculation frequently. harness.AddReplicationTriggerThread(500); harness.AddFailoverThread(5000); harness.StartThreads(); testers.StartThreads(); testers.WaitFor(StressRuntime); testers.Stop(); harness.StopThreads(); } finally { System.Console.Error.WriteLine("===========================\n\n\n\n"); harness.Shutdown(); } }
/// <exception cref="System.Exception"/> public static void WaitForActiveLockData(MultithreadedTestUtil.TestContext ctx, ZooKeeperServer zks, string parentDir, byte[] activeData) { long st = Time.Now(); long lastPrint = st; while (true) { if (ctx != null) { ctx.CheckException(); } try { Stat stat = new Stat(); byte[] data = zks.GetZKDatabase().GetData(parentDir + "/" + ActiveStandbyElector. LockFilename, stat, null); if (activeData != null && Arrays.Equals(activeData, data)) { return; } if (Time.Now() > lastPrint + LogIntervalMs) { Log.Info("Cur data: " + StringUtils.ByteToHexString(data)); lastPrint = Time.Now(); } } catch (KeeperException.NoNodeException) { if (activeData == null) { return; } if (Time.Now() > lastPrint + LogIntervalMs) { Log.Info("Cur data: no node"); lastPrint = Time.Now(); } } Thread.Sleep(50); } }
/// <exception cref="System.IO.IOException"/> /// <exception cref="System.Exception"/> private MultithreadedTestUtil.TestContext SetupClientTestContext(RPCCallBenchmark.MyOptions opts) { if (opts.clientThreads <= 0) { return(null); } // Set up a separate proxy for each client thread, // rather than making them share TCP pipes. int numProxies = opts.clientThreads; RPCCallBenchmark.RpcServiceWrapper[] proxies = new RPCCallBenchmark.RpcServiceWrapper [numProxies]; for (int i = 0; i < numProxies; i++) { proxies[i] = UserGroupInformation.CreateUserForTesting("proxy-" + i, new string[] { }).DoAs(new _PrivilegedExceptionAction_347(this, opts)); } // Create an echo message of the desired length StringBuilder msgBuilder = new StringBuilder(opts.msgSize); for (int c = 0; c < opts.msgSize; c++) { msgBuilder.Append('x'); } string echoMessage = msgBuilder.ToString(); // Create the clients in a test context MultithreadedTestUtil.TestContext ctx = new MultithreadedTestUtil.TestContext(); for (int i_1 = 0; i_1 < opts.clientThreads; i_1++) { RPCCallBenchmark.RpcServiceWrapper proxy = proxies[i_1 % numProxies]; ctx.AddThread(new _RepeatingTestThread_367(this, proxy, echoMessage, ctx)); } return(ctx); }
public _RepeatingTestThread_266(MiniDFSCluster cluster, MultithreadedTestUtil.TestContext baseArg1) : base(baseArg1) { this.cluster = cluster; }
public _RepeatingTestThread_83(HAStressTestHarness _enclosing, int interval, MultithreadedTestUtil.TestContext baseArg1) : base(baseArg1) { this._enclosing = _enclosing; this.interval = interval; }
public _TestingThread_121(InputStream @in, MultithreadedTestUtil.TestContext baseArg1 ) : base(baseArg1) { this.@in = @in; }
/// <exception cref="System.Exception"/> public virtual int Run(string[] args) { RPCCallBenchmark.MyOptions opts = new RPCCallBenchmark.MyOptions(args); if (opts.failed) { return(-1); } // Set RPC engine to the configured RPC engine RPC.SetProtocolEngine(conf, typeof(TestProtoBufRpc.TestRpcService), opts.rpcEngine ); RPC.Server server = StartServer(opts); try { MultithreadedTestUtil.TestContext ctx = SetupClientTestContext(opts); if (ctx != null) { long totalCalls = 0; ctx.StartThreads(); long veryStart = Runtime.NanoTime(); // Loop printing results every second until the specified // time has elapsed for (int i = 0; i < opts.secondsToRun; i++) { long st = Runtime.NanoTime(); ctx.WaitFor(1000); long et = Runtime.NanoTime(); long ct = callCount.GetAndSet(0); totalCalls += ct; double callsPerSec = (ct * 1000000000) / (et - st); System.Console.Out.WriteLine("Calls per second: " + callsPerSec); } // Print results if (totalCalls > 0) { long veryEnd = Runtime.NanoTime(); double callsPerSec = (totalCalls * 1000000000) / (veryEnd - veryStart); long cpuNanosClient = GetTotalCpuTime(ctx.GetTestThreads()); long cpuNanosServer = -1; if (server != null) { cpuNanosServer = GetTotalCpuTime(server.GetHandlers()); } System.Console.Out.WriteLine("====== Results ======"); System.Console.Out.WriteLine("Options:\n" + opts); System.Console.Out.WriteLine("Total calls per second: " + callsPerSec); System.Console.Out.WriteLine("CPU time per call on client: " + (cpuNanosClient / totalCalls) + " ns"); if (server != null) { System.Console.Out.WriteLine("CPU time per call on server: " + (cpuNanosServer / totalCalls) + " ns"); } } else { System.Console.Out.WriteLine("No calls!"); } ctx.Stop(); } else { while (true) { Thread.Sleep(10000); } } } finally { if (server != null) { server.Stop(); } } return(0); }