/// <summary>
/// Test that if ZooKeeper is not running, the correct error
/// code is returned.
/// </summary>
/// <exception cref="System.Exception"/>
public virtual void TestNoZK()
{
StopServer();
DummyHAService svc = cluster.GetService(1);
Assert.Equal(ZKFailoverController.ErrCodeNoZk, RunFC(svc));
}
public virtual void TestFailbackToFaultyServiceFails()
{
DummyHAService svc1 = new DummyHAService(HAServiceProtocol.HAServiceState.Active,
svc1Addr);
Org.Mockito.Mockito.DoThrow(new ServiceFailedException("Failed!")).When(svc1.proxy
).TransitionToActive(AnyReqInfo());
DummyHAService svc2 = new DummyHAService(HAServiceProtocol.HAServiceState.Standby
, svc2Addr);
Org.Mockito.Mockito.DoThrow(new ServiceFailedException("Failed!")).When(svc2.proxy
).TransitionToActive(AnyReqInfo());
svc1.fencer = svc2.fencer = TestNodeFencer.SetupFencer(typeof(TestNodeFencer.AlwaysSucceedFencer
).FullName);
try
{
DoFailover(svc1, svc2, false, false);
NUnit.Framework.Assert.Fail("Failover to already active service");
}
catch (FailoverFailedException)
{
}
// Expected
Assert.Equal(HAServiceProtocol.HAServiceState.Standby, svc1.state
);
Assert.Equal(HAServiceProtocol.HAServiceState.Standby, svc2.state
);
}
public DummyZKFCThread(MiniZKFCCluster _enclosing, MultithreadedTestUtil.TestContext
ctx, DummyHAService svc)
: base(ctx)
{
this._enclosing = _enclosing;
this.zkfc = new MiniZKFCCluster.DummyZKFC(this._enclosing.conf, svc);
}
public virtual void TestWeDontFailbackIfActiveWasFenced()
{
DummyHAService svc1 = new DummyHAService(HAServiceProtocol.HAServiceState.Active,
svc1Addr);
DummyHAService svc2 = new DummyHAService(HAServiceProtocol.HAServiceState.Standby
, svc2Addr);
Org.Mockito.Mockito.DoThrow(new ServiceFailedException("Failed!")).When(svc2.proxy
).TransitionToActive(AnyReqInfo());
svc1.fencer = svc2.fencer = TestNodeFencer.SetupFencer(typeof(TestNodeFencer.AlwaysSucceedFencer
).FullName);
try
{
DoFailover(svc1, svc2, true, false);
NUnit.Framework.Assert.Fail("Failed over to service that won't transition to active"
);
}
catch (FailoverFailedException)
{
}
// Expected
// We failed to failover and did not failback because we fenced
// svc1 (we forced it), therefore svc1 and svc2 should be standby.
Assert.Equal(HAServiceProtocol.HAServiceState.Standby, svc1.state
);
Assert.Equal(HAServiceProtocol.HAServiceState.Standby, svc2.state
);
}
public virtual void TestFencingFailureDuringFailover()
{
DummyHAService svc1 = new DummyHAService(HAServiceProtocol.HAServiceState.Active,
svc1Addr);
DummyHAService svc2 = new DummyHAService(HAServiceProtocol.HAServiceState.Standby
, svc2Addr);
svc1.fencer = svc2.fencer = TestNodeFencer.SetupFencer(typeof(TestNodeFencer.AlwaysFailFencer
).FullName);
TestNodeFencer.AlwaysFailFencer.fenceCalled = 0;
try
{
DoFailover(svc1, svc2, true, false);
NUnit.Framework.Assert.Fail("Failed over even though fencing requested and failed"
);
}
catch (FailoverFailedException)
{
}
// Expected
// If fencing was requested and it failed we don't try to make
// svc2 active anyway, and we don't failback to svc1.
Assert.Equal(1, TestNodeFencer.AlwaysFailFencer.fenceCalled);
NUnit.Framework.Assert.AreSame(svc1, TestNodeFencer.AlwaysFailFencer.fencedSvc);
Assert.Equal(HAServiceProtocol.HAServiceState.Standby, svc1.state
);
Assert.Equal(HAServiceProtocol.HAServiceState.Standby, svc2.state
);
}
public virtual void TestFailoverFromNonExistantServiceWithFencer()
{
DummyHAService svc1 = Org.Mockito.Mockito.Spy(new DummyHAService(null, svc1Addr));
// Getting a proxy to a dead server will throw IOException on call,
// not on creation of the proxy.
HAServiceProtocol errorThrowingProxy = Org.Mockito.Mockito.Mock <HAServiceProtocol
>(Org.Mockito.Mockito.WithSettings().DefaultAnswer(new ThrowsException(new IOException
("Could not connect to host"))).ExtraInterfaces(typeof(IDisposable)));
Org.Mockito.Mockito.DoNothing().When((IDisposable)errorThrowingProxy).Close();
Org.Mockito.Mockito.DoReturn(errorThrowingProxy).When(svc1).GetProxy(Org.Mockito.Mockito
.Any <Configuration>(), Org.Mockito.Mockito.AnyInt());
DummyHAService svc2 = new DummyHAService(HAServiceProtocol.HAServiceState.Standby
, svc2Addr);
svc1.fencer = svc2.fencer = TestNodeFencer.SetupFencer(typeof(TestNodeFencer.AlwaysSucceedFencer
).FullName);
try
{
DoFailover(svc1, svc2, false, false);
}
catch (FailoverFailedException)
{
NUnit.Framework.Assert.Fail("Non-existant active prevented failover");
}
// Verify that the proxy created to try to make it go to standby
// gracefully used the right rpc timeout
Org.Mockito.Mockito.Verify(svc1).GetProxy(Org.Mockito.Mockito.Any <Configuration>(
), Org.Mockito.Mockito.Eq(CommonConfigurationKeys.HaFcGracefulFenceTimeoutDefault
));
// Don't check svc1 because we can't reach it, but that's OK, it's been fenced.
Assert.Equal(HAServiceProtocol.HAServiceState.Active, svc2.state
);
}
public virtual void TestFailoverFromFaultyServiceFencingFailure()
{
DummyHAService svc1 = new DummyHAService(HAServiceProtocol.HAServiceState.Active,
svc1Addr);
Org.Mockito.Mockito.DoThrow(new ServiceFailedException("Failed!")).When(svc1.proxy
).TransitionToStandby(AnyReqInfo());
DummyHAService svc2 = new DummyHAService(HAServiceProtocol.HAServiceState.Standby
, svc2Addr);
svc1.fencer = svc2.fencer = TestNodeFencer.SetupFencer(typeof(TestNodeFencer.AlwaysFailFencer
).FullName);
TestNodeFencer.AlwaysFailFencer.fenceCalled = 0;
try
{
DoFailover(svc1, svc2, false, false);
NUnit.Framework.Assert.Fail("Failed over even though fencing failed");
}
catch (FailoverFailedException)
{
}
// Expected
Assert.Equal(1, TestNodeFencer.AlwaysFailFencer.fenceCalled);
NUnit.Framework.Assert.AreSame(svc1, TestNodeFencer.AlwaysFailFencer.fencedSvc);
Assert.Equal(HAServiceProtocol.HAServiceState.Active, svc1.state
);
Assert.Equal(HAServiceProtocol.HAServiceState.Standby, svc2.state
);
}
public virtual void TestFailoverAndFailback()
{
DummyHAService svc1 = new DummyHAService(HAServiceProtocol.HAServiceState.Active,
svc1Addr);
DummyHAService svc2 = new DummyHAService(HAServiceProtocol.HAServiceState.Standby
, svc2Addr);
svc1.fencer = svc2.fencer = TestNodeFencer.SetupFencer(typeof(TestNodeFencer.AlwaysSucceedFencer
).FullName);
TestNodeFencer.AlwaysSucceedFencer.fenceCalled = 0;
DoFailover(svc1, svc2, false, false);
Assert.Equal(0, TestNodeFencer.AlwaysSucceedFencer.fenceCalled
);
Assert.Equal(HAServiceProtocol.HAServiceState.Standby, svc1.state
);
Assert.Equal(HAServiceProtocol.HAServiceState.Active, svc2.state
);
TestNodeFencer.AlwaysSucceedFencer.fenceCalled = 0;
DoFailover(svc2, svc1, false, false);
Assert.Equal(0, TestNodeFencer.AlwaysSucceedFencer.fenceCalled
);
Assert.Equal(HAServiceProtocol.HAServiceState.Active, svc1.state
);
Assert.Equal(HAServiceProtocol.HAServiceState.Standby, svc2.state
);
}
public virtual void TestFailoverFromFaultyServiceSucceeds()
{
DummyHAService svc1 = new DummyHAService(HAServiceProtocol.HAServiceState.Active,
svc1Addr);
Org.Mockito.Mockito.DoThrow(new ServiceFailedException("Failed!")).When(svc1.proxy
).TransitionToStandby(AnyReqInfo());
DummyHAService svc2 = new DummyHAService(HAServiceProtocol.HAServiceState.Standby
, svc2Addr);
svc1.fencer = svc2.fencer = TestNodeFencer.SetupFencer(typeof(TestNodeFencer.AlwaysSucceedFencer
).FullName);
TestNodeFencer.AlwaysSucceedFencer.fenceCalled = 0;
try
{
DoFailover(svc1, svc2, false, false);
}
catch (FailoverFailedException)
{
NUnit.Framework.Assert.Fail("Faulty active prevented failover");
}
// svc1 still thinks it's active, that's OK, it was fenced
Assert.Equal(1, TestNodeFencer.AlwaysSucceedFencer.fenceCalled
);
NUnit.Framework.Assert.AreSame(svc1, TestNodeFencer.AlwaysSucceedFencer.fencedSvc
);
Assert.Equal(HAServiceProtocol.HAServiceState.Active, svc1.state
);
Assert.Equal(HAServiceProtocol.HAServiceState.Active, svc2.state
);
}
public virtual void TestFailoverToUnhealthyServiceFailsAndFailsback()
{
DummyHAService svc1 = new DummyHAService(HAServiceProtocol.HAServiceState.Active,
svc1Addr);
DummyHAService svc2 = new DummyHAService(HAServiceProtocol.HAServiceState.Standby
, svc2Addr);
Org.Mockito.Mockito.DoThrow(new HealthCheckFailedException("Failed!")).When(svc2.
proxy).MonitorHealth();
svc1.fencer = svc2.fencer = TestNodeFencer.SetupFencer(typeof(TestNodeFencer.AlwaysSucceedFencer
).FullName);
try
{
DoFailover(svc1, svc2, false, false);
NUnit.Framework.Assert.Fail("Failover to unhealthy service");
}
catch (FailoverFailedException)
{
}
// Expected
Assert.Equal(HAServiceProtocol.HAServiceState.Active, svc1.state
);
Assert.Equal(HAServiceProtocol.HAServiceState.Standby, svc2.state
);
}
public virtual void TestFailoverToUnreadyService()
{
DummyHAService svc1 = new DummyHAService(HAServiceProtocol.HAServiceState.Active,
svc1Addr);
DummyHAService svc2 = new DummyHAService(HAServiceProtocol.HAServiceState.Standby
, svc2Addr);
Org.Mockito.Mockito.DoReturn(StateNotReady).When(svc2.proxy).GetServiceStatus();
svc1.fencer = svc2.fencer = TestNodeFencer.SetupFencer(typeof(TestNodeFencer.AlwaysSucceedFencer
).FullName);
try
{
DoFailover(svc1, svc2, false, false);
NUnit.Framework.Assert.Fail("Can't failover to a service that's not ready");
}
catch (FailoverFailedException ffe)
{
// Expected
if (!ffe.Message.Contains("injected not ready"))
{
throw;
}
}
Assert.Equal(HAServiceProtocol.HAServiceState.Active, svc1.state
);
Assert.Equal(HAServiceProtocol.HAServiceState.Standby, svc2.state
);
// Forcing it means we ignore readyToBecomeActive
DoFailover(svc1, svc2, false, true);
Assert.Equal(HAServiceProtocol.HAServiceState.Standby, svc1.state
);
Assert.Equal(HAServiceProtocol.HAServiceState.Active, svc2.state
);
}
/// <summary>Wait for the given HA service to become the active lock holder.</summary>
/// <remarks>
/// Wait for the given HA service to become the active lock holder.
/// If the passed svc is null, waits for there to be no active
/// lock holder.
/// </remarks>
/// <exception cref="System.Exception"/>
public virtual void WaitForActiveLockHolder(int idx)
{
DummyHAService svc = idx == null ? null : svcs[idx];
ActiveStandbyElectorTestUtil.WaitForActiveLockData(ctx, zks, MiniZKFCCluster.DummyZKFC
.ScopedParentZnode, (idx == null) ? null : Ints.ToByteArray(svc.index));
}
public virtual void TestFailureToFenceOnFailbackFailsTheFailback()
{
DummyHAService svc1 = new DummyHAService(HAServiceProtocol.HAServiceState.Active,
svc1Addr);
DummyHAService svc2 = new DummyHAService(HAServiceProtocol.HAServiceState.Standby
, svc2Addr);
Org.Mockito.Mockito.DoThrow(new IOException("Failed!")).When(svc2.proxy).TransitionToActive
(AnyReqInfo());
svc1.fencer = svc2.fencer = TestNodeFencer.SetupFencer(typeof(TestNodeFencer.AlwaysFailFencer
).FullName);
TestNodeFencer.AlwaysFailFencer.fenceCalled = 0;
try
{
DoFailover(svc1, svc2, false, false);
NUnit.Framework.Assert.Fail("Failed over to service that won't transition to active"
);
}
catch (FailoverFailedException)
{
}
// Expected
// We did not fence svc1 because it cooperated and we didn't force it,
// we failed to failover so we fenced svc2, we failed to fence svc2
// so we did not failback to svc1, ie it's still standby.
Assert.Equal(HAServiceProtocol.HAServiceState.Standby, svc1.state
);
Assert.Equal(1, TestNodeFencer.AlwaysFailFencer.fenceCalled);
NUnit.Framework.Assert.AreSame(svc2, TestNodeFencer.AlwaysFailFencer.fencedSvc);
}
public virtual void TestFailoverWithoutPermission()
{
DummyHAService svc1 = new DummyHAService(HAServiceProtocol.HAServiceState.Active,
svc1Addr);
Org.Mockito.Mockito.DoThrow(new AccessControlException("Access denied")).When(svc1
.proxy).GetServiceStatus();
DummyHAService svc2 = new DummyHAService(HAServiceProtocol.HAServiceState.Standby
, svc2Addr);
Org.Mockito.Mockito.DoThrow(new AccessControlException("Access denied")).When(svc2
.proxy).GetServiceStatus();
svc1.fencer = svc2.fencer = TestNodeFencer.SetupFencer(typeof(TestNodeFencer.AlwaysSucceedFencer
).FullName);
try
{
DoFailover(svc1, svc2, false, false);
NUnit.Framework.Assert.Fail("Can't failover when access is denied");
}
catch (FailoverFailedException ffe)
{
Assert.True(ffe.InnerException.Message.Contains("Access denied"
));
}
}
/// <summary>
/// Test that the ZKFC successfully quits the election when it fails to
/// become active.
/// </summary>
/// <remarks>
/// Test that the ZKFC successfully quits the election when it fails to
/// become active. This allows the old node to successfully fail back.
/// </remarks>
/// <exception cref="System.Exception"/>
public virtual void TestBecomingActiveFails()
{
try
{
cluster.Start();
DummyHAService svc1 = cluster.GetService(1);
Log.Info("Making svc1 fail to become active");
cluster.SetFailToBecomeActive(1, true);
Log.Info("Faking svc0 unhealthy, should NOT successfully " + "failover to svc1");
cluster.SetHealthy(0, false);
cluster.WaitForHealthState(0, HealthMonitor.State.ServiceUnhealthy);
cluster.WaitForActiveLockHolder(null);
Org.Mockito.Mockito.Verify(svc1.proxy, Org.Mockito.Mockito.Timeout(2000).AtLeastOnce
()).TransitionToActive(Org.Mockito.Mockito.Any <HAServiceProtocol.StateChangeRequestInfo
>());
cluster.WaitForHAState(0, HAServiceProtocol.HAServiceState.Initializing);
cluster.WaitForHAState(1, HAServiceProtocol.HAServiceState.Standby);
Log.Info("Faking svc0 healthy again, should go back to svc0");
cluster.SetHealthy(0, true);
cluster.WaitForHAState(0, HAServiceProtocol.HAServiceState.Active);
cluster.WaitForHAState(1, HAServiceProtocol.HAServiceState.Standby);
cluster.WaitForActiveLockHolder(0);
// Ensure that we can fail back to svc1 once it it is able
// to become active (e.g the admin has restarted it)
Log.Info("Allowing svc1 to become active, expiring svc0");
svc1.failToBecomeActive = false;
cluster.ExpireAndVerifyFailover(0, 1);
}
finally
{
cluster.Stop();
}
}
/// <summary>
/// Test that, when the health monitor indicates bad health status,
/// failover is triggered.
/// </summary>
/// <remarks>
/// Test that, when the health monitor indicates bad health status,
/// failover is triggered. Also ensures that graceful active->standby
/// transition is used when possible, falling back to fencing when
/// the graceful approach fails.
/// </remarks>
/// <exception cref="System.Exception"/>
public virtual void TestAutoFailoverOnBadHealth()
{
try
{
cluster.Start();
DummyHAService svc1 = cluster.GetService(1);
Log.Info("Faking svc0 unhealthy, should failover to svc1");
cluster.SetHealthy(0, false);
Log.Info("Waiting for svc0 to enter initializing state");
cluster.WaitForHAState(0, HAServiceProtocol.HAServiceState.Initializing);
cluster.WaitForHAState(1, HAServiceProtocol.HAServiceState.Active);
Log.Info("Allowing svc0 to be healthy again, making svc1 unreachable " + "and fail to gracefully go to standby"
);
cluster.SetUnreachable(1, true);
cluster.SetHealthy(0, true);
// Should fail back to svc0 at this point
cluster.WaitForHAState(0, HAServiceProtocol.HAServiceState.Active);
// and fence svc1
Org.Mockito.Mockito.Verify(svc1.fencer).Fence(Org.Mockito.Mockito.Same(svc1));
}
finally
{
cluster.Stop();
}
}
public virtual void Release(DummyHAService oldHolder)
{
lock (this)
{
if (holder == oldHolder)
{
holder = null;
}
}
}
// expected
/// <summary>
/// Test that the ZKFC won't run if fencing is not configured for the
/// local service.
/// </summary>
/// <exception cref="System.Exception"/>
public virtual void TestFencingMustBeConfigured()
{
DummyHAService svc = Org.Mockito.Mockito.Spy(cluster.GetService(0));
Org.Mockito.Mockito.DoThrow(new BadFencingConfigurationException("no fencing")).When
(svc).CheckFencingConfigured();
// Format the base dir, should succeed
Assert.Equal(0, RunFC(svc, "-formatZK"));
// Try to run the actual FC, should fail without a fencer
Assert.Equal(ZKFailoverController.ErrCodeNoFencer, RunFC(svc));
}
/// <summary>
/// Test that automatic failover won't run against a target that hasn't
/// explicitly enabled the feature.
/// </summary>
/// <exception cref="System.Exception"/>
public virtual void TestWontRunWhenAutoFailoverDisabled()
{
DummyHAService svc = cluster.GetService(1);
svc = Org.Mockito.Mockito.Spy(svc);
Org.Mockito.Mockito.DoReturn(false).When(svc).IsAutoFailoverEnabled();
Assert.Equal(ZKFailoverController.ErrCodeAutoFailoverNotEnabled
, RunFC(svc, "-formatZK"));
Assert.Equal(ZKFailoverController.ErrCodeAutoFailoverNotEnabled
, RunFC(svc));
}
public virtual void Take(DummyHAService newHolder)
{
lock (this)
{
if (holder == null || holder == newHolder)
{
holder = newHolder;
}
else
{
violations++;
throw new InvalidOperationException("already held by: " + holder);
}
}
}
/// <summary>
/// Test that the various command lines for formatting the ZK directory
/// function correctly.
/// </summary>
/// <exception cref="System.Exception"/>
public virtual void TestFormatZK()
{
DummyHAService svc = cluster.GetService(1);
// Run without formatting the base dir,
// should barf
Assert.Equal(ZKFailoverController.ErrCodeNoParentZnode, RunFC(
svc));
// Format the base dir, should succeed
Assert.Equal(0, RunFC(svc, "-formatZK"));
// Should fail to format if already formatted
Assert.Equal(ZKFailoverController.ErrCodeFormatDenied, RunFC(svc
, "-formatZK", "-nonInteractive"));
// Unless '-force' is on
Assert.Equal(0, RunFC(svc, "-formatZK", "-force"));
}
/// <summary>
/// Test that, when the health monitor indicates bad health status,
/// failover is triggered.
/// </summary>
/// <remarks>
/// Test that, when the health monitor indicates bad health status,
/// failover is triggered. Also ensures that graceful active->standby
/// transition is used when possible, falling back to fencing when
/// the graceful approach fails.
/// </remarks>
/// <exception cref="System.Exception"/>
public virtual void TestAutoFailoverOnBadState()
{
try
{
cluster.Start();
DummyHAService svc0 = cluster.GetService(0);
Log.Info("Faking svc0 to change the state, should failover to svc1");
svc0.state = HAServiceProtocol.HAServiceState.Standby;
// Should fail back to svc0 at this point
cluster.WaitForHAState(1, HAServiceProtocol.HAServiceState.Active);
}
finally
{
cluster.Stop();
}
}
public virtual void SetupHM()
{
Configuration conf = new Configuration();
conf.SetInt(CommonConfigurationKeys.IpcClientConnectMaxRetriesKey, 1);
conf.SetInt(CommonConfigurationKeys.HaHmCheckIntervalKey, 50);
conf.SetInt(CommonConfigurationKeys.HaHmConnectRetryIntervalKey, 50);
conf.SetInt(CommonConfigurationKeys.HaHmSleepAfterDisconnectKey, 50);
svc = new DummyHAService(HAServiceProtocol.HAServiceState.Active, new IPEndPoint(
"0.0.0.0", 0), true);
hm = new _HealthMonitor_60(this, conf, svc);
Log.Info("Starting health monitor");
hm.Start();
Log.Info("Waiting for HEALTHY signal");
WaitForState(hm, HealthMonitor.State.ServiceHealthy);
}
public MiniZKFCCluster(Configuration conf, ZooKeeperServer zks)
{
this.conf = conf;
// Fast check interval so tests run faster
conf.SetInt(CommonConfigurationKeys.HaHmCheckIntervalKey, 50);
conf.SetInt(CommonConfigurationKeys.HaHmConnectRetryIntervalKey, 50);
conf.SetInt(CommonConfigurationKeys.HaHmSleepAfterDisconnectKey, 50);
svcs = new DummyHAService[2];
svcs[0] = new DummyHAService(HAServiceProtocol.HAServiceState.Initializing, new IPEndPoint
("svc1", 1234));
svcs[0].SetSharedResource(sharedResource);
svcs[1] = new DummyHAService(HAServiceProtocol.HAServiceState.Initializing, new IPEndPoint
("svc2", 1234));
svcs[1].SetSharedResource(sharedResource);
this.ctx = new MultithreadedTestUtil.TestContext();
this.zks = zks;
}
/// <exception cref="Org.Apache.Hadoop.HA.BadFencingConfigurationException"/>
public virtual bool TryFence(HAServiceTarget target, string args)
{
Log.Info("tryFence(" + target + ")");
DummyHAService svc = (DummyHAService)target;
lock (svc)
{
svc.fenceCount++;
}
if (svc.failToFence)
{
Log.Info("Injected failure to fence");
return(false);
}
svc.sharedResource.Release(svc);
return(true);
}
/// <summary>
/// Test that, if ACLs are specified in the configuration, that
/// it sets the ACLs when formatting the parent node.
/// </summary>
/// <exception cref="System.Exception"/>
public virtual void TestFormatSetsAcls()
{
// Format the base dir, should succeed
DummyHAService svc = cluster.GetService(1);
Assert.Equal(0, RunFC(svc, "-formatZK"));
ZooKeeper otherClient = CreateClient();
try
{
// client without auth should not be able to read it
Stat stat = new Stat();
otherClient.GetData(ZKFailoverController.ZkParentZnodeDefault, false, stat);
NUnit.Framework.Assert.Fail("Was able to read data without authenticating!");
}
catch (KeeperException.NoAuthException)
{
}
}
public virtual void TestFailoverToNonExistantServiceFails()
{
DummyHAService svc1 = new DummyHAService(HAServiceProtocol.HAServiceState.Active,
svc1Addr);
DummyHAService svc2 = Org.Mockito.Mockito.Spy(new DummyHAService(null, svc2Addr));
Org.Mockito.Mockito.DoThrow(new IOException("Failed to connect")).When(svc2).GetProxy
(Org.Mockito.Mockito.Any <Configuration>(), Org.Mockito.Mockito.AnyInt());
svc1.fencer = svc2.fencer = TestNodeFencer.SetupFencer(typeof(TestNodeFencer.AlwaysSucceedFencer
).FullName);
try
{
DoFailover(svc1, svc2, false, false);
NUnit.Framework.Assert.Fail("Failed over to a non-existant standby");
}
catch (FailoverFailedException)
{
}
// Expected
Assert.Equal(HAServiceProtocol.HAServiceState.Active, svc1.state
);
}
public virtual void TestFailoverFromActiveToActive()
{
DummyHAService svc1 = new DummyHAService(HAServiceProtocol.HAServiceState.Active,
svc1Addr);
DummyHAService svc2 = new DummyHAService(HAServiceProtocol.HAServiceState.Active,
svc2Addr);
svc1.fencer = svc2.fencer = TestNodeFencer.SetupFencer(typeof(TestNodeFencer.AlwaysSucceedFencer
).FullName);
try
{
DoFailover(svc1, svc2, false, false);
NUnit.Framework.Assert.Fail("Can't failover to an already active service");
}
catch (FailoverFailedException)
{
}
// Expected
Assert.Equal(HAServiceProtocol.HAServiceState.Active, svc1.state
);
Assert.Equal(HAServiceProtocol.HAServiceState.Active, svc2.state
);
}
public virtual void TestFormatOneClusterLeavesOtherClustersAlone()
{
DummyHAService svc = cluster.GetService(1);
MiniZKFCCluster.DummyZKFC zkfcInOtherCluster = new _DummyZKFC_116(conf, cluster.GetService
(1));
// Run without formatting the base dir,
// should barf
Assert.Equal(ZKFailoverController.ErrCodeNoParentZnode, RunFC(
svc));
// Format the base dir, should succeed
Assert.Equal(0, RunFC(svc, "-formatZK"));
// Run the other cluster without formatting, should barf because
// it uses a different parent znode
Assert.Equal(ZKFailoverController.ErrCodeNoParentZnode, zkfcInOtherCluster
.Run(new string[] { }));
// Should succeed in formatting the second cluster
Assert.Equal(0, zkfcInOtherCluster.Run(new string[] { "-formatZK" }));
// But should not have deleted the original base node from the first
// cluster
Assert.Equal(ZKFailoverController.ErrCodeFormatDenied, RunFC(svc
, "-formatZK", "-nonInteractive"));
}
public virtual void TestSelfFailoverFails()
{
DummyHAService svc1 = new DummyHAService(HAServiceProtocol.HAServiceState.Active,
svc1Addr);
DummyHAService svc2 = new DummyHAService(HAServiceProtocol.HAServiceState.Standby
, svc2Addr);
svc1.fencer = svc2.fencer = TestNodeFencer.SetupFencer(typeof(TestNodeFencer.AlwaysSucceedFencer
).FullName);
TestNodeFencer.AlwaysSucceedFencer.fenceCalled = 0;
try
{
DoFailover(svc1, svc1, false, false);
NUnit.Framework.Assert.Fail("Can't failover to yourself");
}
catch (FailoverFailedException)
{
}
// Expected
Assert.Equal(0, TestNodeFencer.AlwaysSucceedFencer.fenceCalled
);
Assert.Equal(HAServiceProtocol.HAServiceState.Active, svc1.state
);
try
{
DoFailover(svc2, svc2, false, false);
NUnit.Framework.Assert.Fail("Can't failover to yourself");
}
catch (FailoverFailedException)
{
}
// Expected
Assert.Equal(0, TestNodeFencer.AlwaysSucceedFencer.fenceCalled
);
Assert.Equal(HAServiceProtocol.HAServiceState.Standby, svc2.state
);
}
