/// <summary>
/// Regression test for HDFS-2693: when doing state transitions, we need to
/// lock the FSNamesystem so that we don't end up doing any writes while it's
/// "in between" states.
/// </summary>
/// <remarks>
/// Regression test for HDFS-2693: when doing state transitions, we need to
/// lock the FSNamesystem so that we don't end up doing any writes while it's
/// "in between" states.
/// This test case starts up several client threads which do mutation operations
/// while flipping a NN back and forth from active to standby.
/// </remarks>
/// <exception cref="System.Exception"/>
public virtual void TestTransitionSynchronization()
{
Configuration conf = new Configuration();
MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).NnTopology(MiniDFSNNTopology
.SimpleHATopology()).NumDataNodes(0).Build();
try
{
cluster.WaitActive();
ReentrantReadWriteLock spyLock = NameNodeAdapter.SpyOnFsLock(cluster.GetNameNode(
0).GetNamesystem());
Org.Mockito.Mockito.DoAnswer(new GenericTestUtils.SleepAnswer(50)).When(spyLock).
WriteLock();
FileSystem fs = HATestUtil.ConfigureFailoverFs(cluster, conf);
MultithreadedTestUtil.TestContext ctx = new MultithreadedTestUtil.TestContext();
for (int i = 0; i < 50; i++)
{
int finalI = i;
ctx.AddThread(new _RepeatingTestThread_256(finalI, fs, ctx));
}
ctx.AddThread(new _RepeatingTestThread_266(cluster, ctx));
ctx.StartThreads();
ctx.WaitFor(20000);
ctx.Stop();
}
finally
{
cluster.Shutdown();
}
}
public _RepeatingTestThread_256(int finalI, FileSystem fs, MultithreadedTestUtil.TestContext
baseArg1)
: base(baseArg1)
{
this.finalI = finalI;
this.fs = fs;
}
public _RepeatingTestThread_106(HAStressTestHarness _enclosing, int msBetweenFailovers
, MultithreadedTestUtil.TestContext baseArg1)
: base(baseArg1)
{
this._enclosing = _enclosing;
this.msBetweenFailovers = msBetweenFailovers;
}
public DummyZKFCThread(MiniZKFCCluster _enclosing, MultithreadedTestUtil.TestContext
ctx, DummyHAService svc)
: base(ctx)
{
this._enclosing = _enclosing;
this.zkfc = new MiniZKFCCluster.DummyZKFC(this._enclosing.conf, svc);
}
public virtual void Setup()
{
conf = new Configuration();
// Specify the quorum per-nameservice, to ensure that these configs
// can be nameservice-scoped.
conf.Set(ZKFailoverController.ZkQuorumKey + ".ns1", hostPort);
conf.Set(DFSConfigKeys.DfsHaFenceMethodsKey, typeof(TestNodeFencer.AlwaysSucceedFencer
).FullName);
conf.SetBoolean(DFSConfigKeys.DfsHaAutoFailoverEnabledKey, true);
// Turn off IPC client caching, so that the suite can handle
// the restart of the daemons between test cases.
conf.SetInt(CommonConfigurationKeysPublic.IpcClientConnectionMaxidletimeKey, 0);
conf.SetInt(DFSConfigKeys.DfsHaZkfcPortKey + ".ns1.nn1", 10023);
conf.SetInt(DFSConfigKeys.DfsHaZkfcPortKey + ".ns1.nn2", 10024);
MiniDFSNNTopology topology = new MiniDFSNNTopology().AddNameservice(new MiniDFSNNTopology.NSConf
("ns1").AddNN(new MiniDFSNNTopology.NNConf("nn1").SetIpcPort(10021)).AddNN(new MiniDFSNNTopology.NNConf
("nn2").SetIpcPort(10022)));
cluster = new MiniDFSCluster.Builder(conf).NnTopology(topology).NumDataNodes(0).Build
();
cluster.WaitActive();
ctx = new MultithreadedTestUtil.TestContext();
ctx.AddThread(thr1 = new TestDFSZKFailoverController.ZKFCThread(this, ctx, 0));
NUnit.Framework.Assert.AreEqual(0, thr1.zkfc.Run(new string[] { "-formatZK" }));
thr1.Start();
WaitForHAState(0, HAServiceProtocol.HAServiceState.Active);
ctx.AddThread(thr2 = new TestDFSZKFailoverController.ZKFCThread(this, ctx, 1));
thr2.Start();
// Wait for the ZKFCs to fully start up
ZKFCTestUtil.WaitForHealthState(thr1.zkfc, HealthMonitor.State.ServiceHealthy, ctx
);
ZKFCTestUtil.WaitForHealthState(thr2.zkfc, HealthMonitor.State.ServiceHealthy, ctx
);
fs = HATestUtil.ConfigureFailoverFs(cluster, conf);
}
public ZKFCThread(TestDFSZKFailoverController _enclosing, MultithreadedTestUtil.TestContext
ctx, int idx)
: base(ctx)
{
this._enclosing = _enclosing;
this.zkfc = DFSZKFailoverController.Create(this._enclosing.cluster.GetConfiguration
(idx));
}
public _RepeatingTestThread_367(RPCCallBenchmark _enclosing, RPCCallBenchmark.RpcServiceWrapper
proxy, string echoMessage, MultithreadedTestUtil.TestContext baseArg1)
: base(baseArg1)
{
this._enclosing = _enclosing;
this.proxy = proxy;
this.echoMessage = echoMessage;
}
public PipelineTestThread(MultithreadedTestUtil.TestContext ctx, FileSystem fs, FileSystem
fsOtherUser, Path p)
: base(ctx)
{
this.fs = fs;
this.fsOtherUser = fsOtherUser;
this.path = p;
}
public ReplicationToggler(MultithreadedTestUtil.TestContext ctx, FileSystem fs, Path
p)
: base(ctx)
{
// How long should the test try to run for. In practice
// it runs for ~20-30s longer than this constant due to startup/
// shutdown time.
this.fs = fs;
this.path = p;
}
/// <exception cref="System.Exception"/>
public static void WaitForHealthState(ZKFailoverController zkfc, HealthMonitor.State
state, MultithreadedTestUtil.TestContext ctx)
{
while (zkfc.GetLastHealthState() != state)
{
if (ctx != null)
{
ctx.CheckException();
}
Thread.Sleep(50);
}
}
/// <exception cref="System.Exception"/>
public static void WaitForElectorState(MultithreadedTestUtil.TestContext ctx, ActiveStandbyElector
elector, ActiveStandbyElector.State state)
{
while (elector.GetStateForTests() != state)
{
if (ctx != null)
{
ctx.CheckException();
}
Thread.Sleep(50);
}
}
public MiniZKFCCluster(Configuration conf, ZooKeeperServer zks)
{
this.conf = conf;
// Fast check interval so tests run faster
conf.SetInt(CommonConfigurationKeys.HaHmCheckIntervalKey, 50);
conf.SetInt(CommonConfigurationKeys.HaHmConnectRetryIntervalKey, 50);
conf.SetInt(CommonConfigurationKeys.HaHmSleepAfterDisconnectKey, 50);
svcs = new DummyHAService[2];
svcs[0] = new DummyHAService(HAServiceProtocol.HAServiceState.Initializing, new IPEndPoint
("svc1", 1234));
svcs[0].SetSharedResource(sharedResource);
svcs[1] = new DummyHAService(HAServiceProtocol.HAServiceState.Initializing, new IPEndPoint
("svc2", 1234));
svcs[1].SetSharedResource(sharedResource);
this.ctx = new MultithreadedTestUtil.TestContext();
this.zks = zks;
}
public virtual void TestFencingStress()
{
HAStressTestHarness harness = new HAStressTestHarness();
harness.conf.SetInt(DFSConfigKeys.DfsBlockreportIntervalMsecKey, 1000);
harness.conf.SetInt(DFSConfigKeys.DfsNamenodeHeartbeatRecheckIntervalKey, 1);
harness.conf.SetInt(DFSConfigKeys.DfsNamenodeReplicationIntervalKey, 1);
MiniDFSCluster cluster = harness.StartCluster();
try
{
cluster.WaitActive();
cluster.TransitionToActive(0);
FileSystem fs = harness.GetFailoverFs();
MultithreadedTestUtil.TestContext togglers = new MultithreadedTestUtil.TestContext
();
for (int i = 0; i < NumThreads; i++)
{
Path p = new Path("/test-" + i);
DFSTestUtil.CreateFile(fs, p, BlockSize * 10, (short)3, (long)i);
togglers.AddThread(new TestDNFencingWithReplication.ReplicationToggler(togglers,
fs, p));
}
// Start a separate thread which will make sure that replication
// happens quickly by triggering deletion reports and replication
// work calculation frequently.
harness.AddReplicationTriggerThread(500);
harness.AddFailoverThread(5000);
harness.StartThreads();
togglers.StartThreads();
togglers.WaitFor(Runtime);
togglers.Stop();
harness.StopThreads();
// CHeck that the files can be read without throwing
for (int i_1 = 0; i_1 < NumThreads; i_1++)
{
Path p = new Path("/test-" + i_1);
DFSTestUtil.ReadFile(fs, p);
}
}
finally
{
System.Console.Error.WriteLine("===========================\n\n\n\n");
harness.Shutdown();
}
}
/// <summary>Stress test for pipeline/lease recovery.</summary>
/// <remarks>
/// Stress test for pipeline/lease recovery. Starts a number of
/// threads, each of which creates a file and has another client
/// break the lease. While these threads run, failover proceeds
/// back and forth between two namenodes.
/// </remarks>
/// <exception cref="System.Exception"/>
public virtual void TestPipelineRecoveryStress()
{
HAStressTestHarness harness = new HAStressTestHarness();
// Disable permissions so that another user can recover the lease.
harness.conf.SetBoolean(DFSConfigKeys.DfsPermissionsEnabledKey, false);
// This test triggers rapid NN failovers. The client retry policy uses an
// exponential backoff. This can quickly lead to long sleep times and even
// timeout the whole test. Cap the sleep time at 1s to prevent this.
harness.conf.SetInt(DFSConfigKeys.DfsClientFailoverSleeptimeMaxKey, 1000);
MiniDFSCluster cluster = harness.StartCluster();
try
{
cluster.WaitActive();
cluster.TransitionToActive(0);
FileSystem fs = harness.GetFailoverFs();
DistributedFileSystem fsAsOtherUser = CreateFsAsOtherUser(cluster, harness.conf);
MultithreadedTestUtil.TestContext testers = new MultithreadedTestUtil.TestContext
();
for (int i = 0; i < StressNumThreads; i++)
{
Path p = new Path("/test-" + i);
testers.AddThread(new TestPipelinesFailover.PipelineTestThread(testers, fs, fsAsOtherUser
, p));
}
// Start a separate thread which will make sure that replication
// happens quickly by triggering deletion reports and replication
// work calculation frequently.
harness.AddReplicationTriggerThread(500);
harness.AddFailoverThread(5000);
harness.StartThreads();
testers.StartThreads();
testers.WaitFor(StressRuntime);
testers.Stop();
harness.StopThreads();
}
finally
{
System.Console.Error.WriteLine("===========================\n\n\n\n");
harness.Shutdown();
}
}
/// <exception cref="System.Exception"/>
public static void WaitForActiveLockData(MultithreadedTestUtil.TestContext ctx, ZooKeeperServer
zks, string parentDir, byte[] activeData)
{
long st = Time.Now();
long lastPrint = st;
while (true)
{
if (ctx != null)
{
ctx.CheckException();
}
try
{
Stat stat = new Stat();
byte[] data = zks.GetZKDatabase().GetData(parentDir + "/" + ActiveStandbyElector.
LockFilename, stat, null);
if (activeData != null && Arrays.Equals(activeData, data))
{
return;
}
if (Time.Now() > lastPrint + LogIntervalMs)
{
Log.Info("Cur data: " + StringUtils.ByteToHexString(data));
lastPrint = Time.Now();
}
}
catch (KeeperException.NoNodeException)
{
if (activeData == null)
{
return;
}
if (Time.Now() > lastPrint + LogIntervalMs)
{
Log.Info("Cur data: no node");
lastPrint = Time.Now();
}
}
Thread.Sleep(50);
}
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="System.Exception"/>
private MultithreadedTestUtil.TestContext SetupClientTestContext(RPCCallBenchmark.MyOptions
opts)
{
if (opts.clientThreads <= 0)
{
return(null);
}
// Set up a separate proxy for each client thread,
// rather than making them share TCP pipes.
int numProxies = opts.clientThreads;
RPCCallBenchmark.RpcServiceWrapper[] proxies = new RPCCallBenchmark.RpcServiceWrapper
[numProxies];
for (int i = 0; i < numProxies; i++)
{
proxies[i] = UserGroupInformation.CreateUserForTesting("proxy-" + i, new string[]
{ }).DoAs(new _PrivilegedExceptionAction_347(this, opts));
}
// Create an echo message of the desired length
StringBuilder msgBuilder = new StringBuilder(opts.msgSize);
for (int c = 0; c < opts.msgSize; c++)
{
msgBuilder.Append('x');
}
string echoMessage = msgBuilder.ToString();
// Create the clients in a test context
MultithreadedTestUtil.TestContext ctx = new MultithreadedTestUtil.TestContext();
for (int i_1 = 0; i_1 < opts.clientThreads; i_1++)
{
RPCCallBenchmark.RpcServiceWrapper proxy = proxies[i_1 % numProxies];
ctx.AddThread(new _RepeatingTestThread_367(this, proxy, echoMessage, ctx));
}
return(ctx);
}
public _RepeatingTestThread_266(MiniDFSCluster cluster, MultithreadedTestUtil.TestContext
baseArg1)
: base(baseArg1)
{
this.cluster = cluster;
}
public _RepeatingTestThread_83(HAStressTestHarness _enclosing, int interval, MultithreadedTestUtil.TestContext
baseArg1)
: base(baseArg1)
{
this._enclosing = _enclosing;
this.interval = interval;
}
public _TestingThread_121(InputStream @in, MultithreadedTestUtil.TestContext baseArg1
)
: base(baseArg1)
{
this.@in = @in;
}
/// <exception cref="System.Exception"/>
public virtual int Run(string[] args)
{
RPCCallBenchmark.MyOptions opts = new RPCCallBenchmark.MyOptions(args);
if (opts.failed)
{
return(-1);
}
// Set RPC engine to the configured RPC engine
RPC.SetProtocolEngine(conf, typeof(TestProtoBufRpc.TestRpcService), opts.rpcEngine
);
RPC.Server server = StartServer(opts);
try
{
MultithreadedTestUtil.TestContext ctx = SetupClientTestContext(opts);
if (ctx != null)
{
long totalCalls = 0;
ctx.StartThreads();
long veryStart = Runtime.NanoTime();
// Loop printing results every second until the specified
// time has elapsed
for (int i = 0; i < opts.secondsToRun; i++)
{
long st = Runtime.NanoTime();
ctx.WaitFor(1000);
long et = Runtime.NanoTime();
long ct = callCount.GetAndSet(0);
totalCalls += ct;
double callsPerSec = (ct * 1000000000) / (et - st);
System.Console.Out.WriteLine("Calls per second: " + callsPerSec);
}
// Print results
if (totalCalls > 0)
{
long veryEnd = Runtime.NanoTime();
double callsPerSec = (totalCalls * 1000000000) / (veryEnd - veryStart);
long cpuNanosClient = GetTotalCpuTime(ctx.GetTestThreads());
long cpuNanosServer = -1;
if (server != null)
{
cpuNanosServer = GetTotalCpuTime(server.GetHandlers());
}
System.Console.Out.WriteLine("====== Results ======");
System.Console.Out.WriteLine("Options:\n" + opts);
System.Console.Out.WriteLine("Total calls per second: " + callsPerSec);
System.Console.Out.WriteLine("CPU time per call on client: " + (cpuNanosClient /
totalCalls) + " ns");
if (server != null)
{
System.Console.Out.WriteLine("CPU time per call on server: " + (cpuNanosServer /
totalCalls) + " ns");
}
}
else
{
System.Console.Out.WriteLine("No calls!");
}
ctx.Stop();
}
else
{
while (true)
{
Thread.Sleep(10000);
}
}
}
finally
{
if (server != null)
{
server.Stop();
}
}
return(0);
}
