/// <summary>
/// Regression test for HDFS-2693: when doing state transitions, we need to
/// lock the FSNamesystem so that we don't end up doing any writes while it's
/// "in between" states.
/// </summary>
/// <remarks>
/// Regression test for HDFS-2693: when doing state transitions, we need to
/// lock the FSNamesystem so that we don't end up doing any writes while it's
/// "in between" states.
/// This test case starts up several client threads which do mutation operations
/// while flipping a NN back and forth from active to standby.
/// </remarks>
/// <exception cref="System.Exception"/>
public virtual void TestTransitionSynchronization()
{
Configuration conf = new Configuration();
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NnTopology(MiniDFSNNTopology
.SimpleHATopology()).NumDataNodes(0).Build();
try
{
cluster.WaitActive();
ReentrantReadWriteLock spyLock = NameNodeAdapter.SpyOnFsLock(cluster.GetNameNode(
0).GetNamesystem());
Org.Mockito.Mockito.DoAnswer(new GenericTestUtils.SleepAnswer(50)).When(spyLock).
WriteLock();
FileSystem fs = HATestUtil.ConfigureFailoverFs(cluster, conf);
MultithreadedTestUtil.TestContext ctx = new MultithreadedTestUtil.TestContext();
for (int i = 0; i < 50; i++)
{
int finalI = i;
ctx.AddThread(new _RepeatingTestThread_256(finalI, fs, ctx));
}
ctx.AddThread(new _RepeatingTestThread_266(cluster, ctx));
ctx.StartThreads();
ctx.WaitFor(20000);
ctx.Stop();
}
finally
{
cluster.Shutdown();
}
}
public virtual void Setup()
{
conf = new Configuration();
// Specify the quorum per-nameservice, to ensure that these configs
// can be nameservice-scoped.
conf.Set(ZKFailoverController.ZkQuorumKey + ".ns1", hostPort);
conf.Set(DFSConfigKeys.DfsHaFenceMethodsKey, typeof(TestNodeFencer.AlwaysSucceedFencer
).FullName);
conf.SetBoolean(DFSConfigKeys.DfsHaAutoFailoverEnabledKey, true);
// Turn off IPC client caching, so that the suite can handle
// the restart of the daemons between test cases.
conf.SetInt(CommonConfigurationKeysPublic.IpcClientConnectionMaxidletimeKey, 0);
conf.SetInt(DFSConfigKeys.DfsHaZkfcPortKey + ".ns1.nn1", 10023);
conf.SetInt(DFSConfigKeys.DfsHaZkfcPortKey + ".ns1.nn2", 10024);
MiniDFSNNTopology topology = new MiniDFSNNTopology().AddNameservice(new MiniDFSNNTopology.NSConf
("ns1").AddNN(new MiniDFSNNTopology.NNConf("nn1").SetIpcPort(10021)).AddNN(new MiniDFSNNTopology.NNConf
("nn2").SetIpcPort(10022)));
cluster = new MiniDFSCluster.Builder(conf).NnTopology(topology).NumDataNodes(0).Build
();
cluster.WaitActive();
ctx = new MultithreadedTestUtil.TestContext();
ctx.AddThread(thr1 = new TestDFSZKFailoverController.ZKFCThread(this, ctx, 0));
NUnit.Framework.Assert.AreEqual(0, thr1.zkfc.Run(new string[] { "-formatZK" }));
thr1.Start();
WaitForHAState(0, HAServiceProtocol.HAServiceState.Active);
ctx.AddThread(thr2 = new TestDFSZKFailoverController.ZKFCThread(this, ctx, 1));
thr2.Start();
// Wait for the ZKFCs to fully start up
ZKFCTestUtil.WaitForHealthState(thr1.zkfc, HealthMonitor.State.ServiceHealthy, ctx
);
ZKFCTestUtil.WaitForHealthState(thr2.zkfc, HealthMonitor.State.ServiceHealthy, ctx
);
fs = HATestUtil.ConfigureFailoverFs(cluster, conf);
}
public virtual void TestFencingStress()
{
HAStressTestHarness harness = new HAStressTestHarness();
harness.conf.SetInt(DFSConfigKeys.DfsBlockreportIntervalMsecKey, 1000);
harness.conf.SetInt(DFSConfigKeys.DfsNamenodeHeartbeatRecheckIntervalKey, 1);
harness.conf.SetInt(DFSConfigKeys.DfsNamenodeReplicationIntervalKey, 1);
MiniDFSCluster cluster = harness.StartCluster();
try
{
cluster.WaitActive();
cluster.TransitionToActive(0);
FileSystem fs = harness.GetFailoverFs();
MultithreadedTestUtil.TestContext togglers = new MultithreadedTestUtil.TestContext
();
for (int i = 0; i < NumThreads; i++)
{
Path p = new Path("/test-" + i);
DFSTestUtil.CreateFile(fs, p, BlockSize * 10, (short)3, (long)i);
togglers.AddThread(new TestDNFencingWithReplication.ReplicationToggler(togglers,
fs, p));
}
// Start a separate thread which will make sure that replication
// happens quickly by triggering deletion reports and replication
// work calculation frequently.
harness.AddReplicationTriggerThread(500);
harness.AddFailoverThread(5000);
harness.StartThreads();
togglers.StartThreads();
togglers.WaitFor(Runtime);
togglers.Stop();
harness.StopThreads();
// CHeck that the files can be read without throwing
for (int i_1 = 0; i_1 < NumThreads; i_1++)
{
Path p = new Path("/test-" + i_1);
DFSTestUtil.ReadFile(fs, p);
}
}
finally
{
System.Console.Error.WriteLine("===========================\n\n\n\n");
harness.Shutdown();
}
}
/// <summary>Stress test for pipeline/lease recovery.</summary>
/// <remarks>
/// Stress test for pipeline/lease recovery. Starts a number of
/// threads, each of which creates a file and has another client
/// break the lease. While these threads run, failover proceeds
/// back and forth between two namenodes.
/// </remarks>
/// <exception cref="System.Exception"/>
public virtual void TestPipelineRecoveryStress()
{
HAStressTestHarness harness = new HAStressTestHarness();
// Disable permissions so that another user can recover the lease.
harness.conf.SetBoolean(DFSConfigKeys.DfsPermissionsEnabledKey, false);
// This test triggers rapid NN failovers. The client retry policy uses an
// exponential backoff. This can quickly lead to long sleep times and even
// timeout the whole test. Cap the sleep time at 1s to prevent this.
harness.conf.SetInt(DFSConfigKeys.DfsClientFailoverSleeptimeMaxKey, 1000);
MiniDFSCluster cluster = harness.StartCluster();
try
{
cluster.WaitActive();
cluster.TransitionToActive(0);
FileSystem fs = harness.GetFailoverFs();
DistributedFileSystem fsAsOtherUser = CreateFsAsOtherUser(cluster, harness.conf);
MultithreadedTestUtil.TestContext testers = new MultithreadedTestUtil.TestContext
();
for (int i = 0; i < StressNumThreads; i++)
{
Path p = new Path("/test-" + i);
testers.AddThread(new TestPipelinesFailover.PipelineTestThread(testers, fs, fsAsOtherUser
, p));
}
// Start a separate thread which will make sure that replication
// happens quickly by triggering deletion reports and replication
// work calculation frequently.
harness.AddReplicationTriggerThread(500);
harness.AddFailoverThread(5000);
harness.StartThreads();
testers.StartThreads();
testers.WaitFor(StressRuntime);
testers.Stop();
harness.StopThreads();
}
finally
{
System.Console.Error.WriteLine("===========================\n\n\n\n");
harness.Shutdown();
}
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
private MultithreadedTestUtil.TestContext SetupClientTestContext(RPCCallBenchmark.MyOptions
opts)
{
if (opts.clientThreads <= 0)
{
return(null);
}
// Set up a separate proxy for each client thread,
// rather than making them share TCP pipes.
int numProxies = opts.clientThreads;
RPCCallBenchmark.RpcServiceWrapper[] proxies = new RPCCallBenchmark.RpcServiceWrapper
[numProxies];
for (int i = 0; i < numProxies; i++)
{
proxies[i] = UserGroupInformation.CreateUserForTesting("proxy-" + i, new string[]
{ }).DoAs(new _PrivilegedExceptionAction_347(this, opts));
}
// Create an echo message of the desired length
StringBuilder msgBuilder = new StringBuilder(opts.msgSize);
for (int c = 0; c < opts.msgSize; c++)
{
msgBuilder.Append('x');
}
string echoMessage = msgBuilder.ToString();
// Create the clients in a test context
MultithreadedTestUtil.TestContext ctx = new MultithreadedTestUtil.TestContext();
for (int i_1 = 0; i_1 < opts.clientThreads; i_1++)
{
RPCCallBenchmark.RpcServiceWrapper proxy = proxies[i_1 % numProxies];
ctx.AddThread(new _RepeatingTestThread_367(this, proxy, echoMessage, ctx));
}
return(ctx);
}
/// <summary>
/// Add a thread which periodically triggers deletion reports,
/// heartbeats, and NN-side block work.
/// </summary>
/// <param name="interval">millisecond period on which to run</param>
public virtual void AddReplicationTriggerThread(int interval)
{
testCtx.AddThread(new _RepeatingTestThread_83(this, interval, testCtx));
}