/// <summary> /// Test that if ZooKeeper is not running, the correct error /// code is returned. /// </summary> /// <exception cref="System.Exception"/> public virtual void TestNoZK() { StopServer(); DummyHAService svc = cluster.GetService(1); Assert.Equal(ZKFailoverController.ErrCodeNoZk, RunFC(svc)); }
public virtual void TestFailbackToFaultyServiceFails() { DummyHAService svc1 = new DummyHAService(HAServiceProtocol.HAServiceState.Active, svc1Addr); Org.Mockito.Mockito.DoThrow(new ServiceFailedException("Failed!")).When(svc1.proxy ).TransitionToActive(AnyReqInfo()); DummyHAService svc2 = new DummyHAService(HAServiceProtocol.HAServiceState.Standby , svc2Addr); Org.Mockito.Mockito.DoThrow(new ServiceFailedException("Failed!")).When(svc2.proxy ).TransitionToActive(AnyReqInfo()); svc1.fencer = svc2.fencer = TestNodeFencer.SetupFencer(typeof(TestNodeFencer.AlwaysSucceedFencer ).FullName); try { DoFailover(svc1, svc2, false, false); NUnit.Framework.Assert.Fail("Failover to already active service"); } catch (FailoverFailedException) { } // Expected Assert.Equal(HAServiceProtocol.HAServiceState.Standby, svc1.state ); Assert.Equal(HAServiceProtocol.HAServiceState.Standby, svc2.state ); }
public DummyZKFCThread(MiniZKFCCluster _enclosing, MultithreadedTestUtil.TestContext ctx, DummyHAService svc) : base(ctx) { this._enclosing = _enclosing; this.zkfc = new MiniZKFCCluster.DummyZKFC(this._enclosing.conf, svc); }
public virtual void TestWeDontFailbackIfActiveWasFenced() { DummyHAService svc1 = new DummyHAService(HAServiceProtocol.HAServiceState.Active, svc1Addr); DummyHAService svc2 = new DummyHAService(HAServiceProtocol.HAServiceState.Standby , svc2Addr); Org.Mockito.Mockito.DoThrow(new ServiceFailedException("Failed!")).When(svc2.proxy ).TransitionToActive(AnyReqInfo()); svc1.fencer = svc2.fencer = TestNodeFencer.SetupFencer(typeof(TestNodeFencer.AlwaysSucceedFencer ).FullName); try { DoFailover(svc1, svc2, true, false); NUnit.Framework.Assert.Fail("Failed over to service that won't transition to active" ); } catch (FailoverFailedException) { } // Expected // We failed to failover and did not failback because we fenced // svc1 (we forced it), therefore svc1 and svc2 should be standby. Assert.Equal(HAServiceProtocol.HAServiceState.Standby, svc1.state ); Assert.Equal(HAServiceProtocol.HAServiceState.Standby, svc2.state ); }
public virtual void TestFencingFailureDuringFailover() { DummyHAService svc1 = new DummyHAService(HAServiceProtocol.HAServiceState.Active, svc1Addr); DummyHAService svc2 = new DummyHAService(HAServiceProtocol.HAServiceState.Standby , svc2Addr); svc1.fencer = svc2.fencer = TestNodeFencer.SetupFencer(typeof(TestNodeFencer.AlwaysFailFencer ).FullName); TestNodeFencer.AlwaysFailFencer.fenceCalled = 0; try { DoFailover(svc1, svc2, true, false); NUnit.Framework.Assert.Fail("Failed over even though fencing requested and failed" ); } catch (FailoverFailedException) { } // Expected // If fencing was requested and it failed we don't try to make // svc2 active anyway, and we don't failback to svc1. Assert.Equal(1, TestNodeFencer.AlwaysFailFencer.fenceCalled); NUnit.Framework.Assert.AreSame(svc1, TestNodeFencer.AlwaysFailFencer.fencedSvc); Assert.Equal(HAServiceProtocol.HAServiceState.Standby, svc1.state ); Assert.Equal(HAServiceProtocol.HAServiceState.Standby, svc2.state ); }
public virtual void TestFailoverFromNonExistantServiceWithFencer() { DummyHAService svc1 = Org.Mockito.Mockito.Spy(new DummyHAService(null, svc1Addr)); // Getting a proxy to a dead server will throw IOException on call, // not on creation of the proxy. HAServiceProtocol errorThrowingProxy = Org.Mockito.Mockito.Mock <HAServiceProtocol >(Org.Mockito.Mockito.WithSettings().DefaultAnswer(new ThrowsException(new IOException ("Could not connect to host"))).ExtraInterfaces(typeof(IDisposable))); Org.Mockito.Mockito.DoNothing().When((IDisposable)errorThrowingProxy).Close(); Org.Mockito.Mockito.DoReturn(errorThrowingProxy).When(svc1).GetProxy(Org.Mockito.Mockito .Any <Configuration>(), Org.Mockito.Mockito.AnyInt()); DummyHAService svc2 = new DummyHAService(HAServiceProtocol.HAServiceState.Standby , svc2Addr); svc1.fencer = svc2.fencer = TestNodeFencer.SetupFencer(typeof(TestNodeFencer.AlwaysSucceedFencer ).FullName); try { DoFailover(svc1, svc2, false, false); } catch (FailoverFailedException) { NUnit.Framework.Assert.Fail("Non-existant active prevented failover"); } // Verify that the proxy created to try to make it go to standby // gracefully used the right rpc timeout Org.Mockito.Mockito.Verify(svc1).GetProxy(Org.Mockito.Mockito.Any <Configuration>( ), Org.Mockito.Mockito.Eq(CommonConfigurationKeys.HaFcGracefulFenceTimeoutDefault )); // Don't check svc1 because we can't reach it, but that's OK, it's been fenced. Assert.Equal(HAServiceProtocol.HAServiceState.Active, svc2.state ); }
public virtual void TestFailoverFromFaultyServiceFencingFailure() { DummyHAService svc1 = new DummyHAService(HAServiceProtocol.HAServiceState.Active, svc1Addr); Org.Mockito.Mockito.DoThrow(new ServiceFailedException("Failed!")).When(svc1.proxy ).TransitionToStandby(AnyReqInfo()); DummyHAService svc2 = new DummyHAService(HAServiceProtocol.HAServiceState.Standby , svc2Addr); svc1.fencer = svc2.fencer = TestNodeFencer.SetupFencer(typeof(TestNodeFencer.AlwaysFailFencer ).FullName); TestNodeFencer.AlwaysFailFencer.fenceCalled = 0; try { DoFailover(svc1, svc2, false, false); NUnit.Framework.Assert.Fail("Failed over even though fencing failed"); } catch (FailoverFailedException) { } // Expected Assert.Equal(1, TestNodeFencer.AlwaysFailFencer.fenceCalled); NUnit.Framework.Assert.AreSame(svc1, TestNodeFencer.AlwaysFailFencer.fencedSvc); Assert.Equal(HAServiceProtocol.HAServiceState.Active, svc1.state ); Assert.Equal(HAServiceProtocol.HAServiceState.Standby, svc2.state ); }
public virtual void TestFailoverAndFailback() { DummyHAService svc1 = new DummyHAService(HAServiceProtocol.HAServiceState.Active, svc1Addr); DummyHAService svc2 = new DummyHAService(HAServiceProtocol.HAServiceState.Standby , svc2Addr); svc1.fencer = svc2.fencer = TestNodeFencer.SetupFencer(typeof(TestNodeFencer.AlwaysSucceedFencer ).FullName); TestNodeFencer.AlwaysSucceedFencer.fenceCalled = 0; DoFailover(svc1, svc2, false, false); Assert.Equal(0, TestNodeFencer.AlwaysSucceedFencer.fenceCalled ); Assert.Equal(HAServiceProtocol.HAServiceState.Standby, svc1.state ); Assert.Equal(HAServiceProtocol.HAServiceState.Active, svc2.state ); TestNodeFencer.AlwaysSucceedFencer.fenceCalled = 0; DoFailover(svc2, svc1, false, false); Assert.Equal(0, TestNodeFencer.AlwaysSucceedFencer.fenceCalled ); Assert.Equal(HAServiceProtocol.HAServiceState.Active, svc1.state ); Assert.Equal(HAServiceProtocol.HAServiceState.Standby, svc2.state ); }
public virtual void TestFailoverFromFaultyServiceSucceeds() { DummyHAService svc1 = new DummyHAService(HAServiceProtocol.HAServiceState.Active, svc1Addr); Org.Mockito.Mockito.DoThrow(new ServiceFailedException("Failed!")).When(svc1.proxy ).TransitionToStandby(AnyReqInfo()); DummyHAService svc2 = new DummyHAService(HAServiceProtocol.HAServiceState.Standby , svc2Addr); svc1.fencer = svc2.fencer = TestNodeFencer.SetupFencer(typeof(TestNodeFencer.AlwaysSucceedFencer ).FullName); TestNodeFencer.AlwaysSucceedFencer.fenceCalled = 0; try { DoFailover(svc1, svc2, false, false); } catch (FailoverFailedException) { NUnit.Framework.Assert.Fail("Faulty active prevented failover"); } // svc1 still thinks it's active, that's OK, it was fenced Assert.Equal(1, TestNodeFencer.AlwaysSucceedFencer.fenceCalled ); NUnit.Framework.Assert.AreSame(svc1, TestNodeFencer.AlwaysSucceedFencer.fencedSvc ); Assert.Equal(HAServiceProtocol.HAServiceState.Active, svc1.state ); Assert.Equal(HAServiceProtocol.HAServiceState.Active, svc2.state ); }
public virtual void TestFailoverToUnhealthyServiceFailsAndFailsback() { DummyHAService svc1 = new DummyHAService(HAServiceProtocol.HAServiceState.Active, svc1Addr); DummyHAService svc2 = new DummyHAService(HAServiceProtocol.HAServiceState.Standby , svc2Addr); Org.Mockito.Mockito.DoThrow(new HealthCheckFailedException("Failed!")).When(svc2. proxy).MonitorHealth(); svc1.fencer = svc2.fencer = TestNodeFencer.SetupFencer(typeof(TestNodeFencer.AlwaysSucceedFencer ).FullName); try { DoFailover(svc1, svc2, false, false); NUnit.Framework.Assert.Fail("Failover to unhealthy service"); } catch (FailoverFailedException) { } // Expected Assert.Equal(HAServiceProtocol.HAServiceState.Active, svc1.state ); Assert.Equal(HAServiceProtocol.HAServiceState.Standby, svc2.state ); }
public virtual void TestFailoverToUnreadyService() { DummyHAService svc1 = new DummyHAService(HAServiceProtocol.HAServiceState.Active, svc1Addr); DummyHAService svc2 = new DummyHAService(HAServiceProtocol.HAServiceState.Standby , svc2Addr); Org.Mockito.Mockito.DoReturn(StateNotReady).When(svc2.proxy).GetServiceStatus(); svc1.fencer = svc2.fencer = TestNodeFencer.SetupFencer(typeof(TestNodeFencer.AlwaysSucceedFencer ).FullName); try { DoFailover(svc1, svc2, false, false); NUnit.Framework.Assert.Fail("Can't failover to a service that's not ready"); } catch (FailoverFailedException ffe) { // Expected if (!ffe.Message.Contains("injected not ready")) { throw; } } Assert.Equal(HAServiceProtocol.HAServiceState.Active, svc1.state ); Assert.Equal(HAServiceProtocol.HAServiceState.Standby, svc2.state ); // Forcing it means we ignore readyToBecomeActive DoFailover(svc1, svc2, false, true); Assert.Equal(HAServiceProtocol.HAServiceState.Standby, svc1.state ); Assert.Equal(HAServiceProtocol.HAServiceState.Active, svc2.state ); }
/// <summary>Wait for the given HA service to become the active lock holder.</summary> /// <remarks> /// Wait for the given HA service to become the active lock holder. /// If the passed svc is null, waits for there to be no active /// lock holder. /// </remarks> /// <exception cref="System.Exception"/> public virtual void WaitForActiveLockHolder(int idx) { DummyHAService svc = idx == null ? null : svcs[idx]; ActiveStandbyElectorTestUtil.WaitForActiveLockData(ctx, zks, MiniZKFCCluster.DummyZKFC .ScopedParentZnode, (idx == null) ? null : Ints.ToByteArray(svc.index)); }
public virtual void TestFailureToFenceOnFailbackFailsTheFailback() { DummyHAService svc1 = new DummyHAService(HAServiceProtocol.HAServiceState.Active, svc1Addr); DummyHAService svc2 = new DummyHAService(HAServiceProtocol.HAServiceState.Standby , svc2Addr); Org.Mockito.Mockito.DoThrow(new IOException("Failed!")).When(svc2.proxy).TransitionToActive (AnyReqInfo()); svc1.fencer = svc2.fencer = TestNodeFencer.SetupFencer(typeof(TestNodeFencer.AlwaysFailFencer ).FullName); TestNodeFencer.AlwaysFailFencer.fenceCalled = 0; try { DoFailover(svc1, svc2, false, false); NUnit.Framework.Assert.Fail("Failed over to service that won't transition to active" ); } catch (FailoverFailedException) { } // Expected // We did not fence svc1 because it cooperated and we didn't force it, // we failed to failover so we fenced svc2, we failed to fence svc2 // so we did not failback to svc1, ie it's still standby. Assert.Equal(HAServiceProtocol.HAServiceState.Standby, svc1.state ); Assert.Equal(1, TestNodeFencer.AlwaysFailFencer.fenceCalled); NUnit.Framework.Assert.AreSame(svc2, TestNodeFencer.AlwaysFailFencer.fencedSvc); }
public virtual void TestFailoverWithoutPermission() { DummyHAService svc1 = new DummyHAService(HAServiceProtocol.HAServiceState.Active, svc1Addr); Org.Mockito.Mockito.DoThrow(new AccessControlException("Access denied")).When(svc1 .proxy).GetServiceStatus(); DummyHAService svc2 = new DummyHAService(HAServiceProtocol.HAServiceState.Standby , svc2Addr); Org.Mockito.Mockito.DoThrow(new AccessControlException("Access denied")).When(svc2 .proxy).GetServiceStatus(); svc1.fencer = svc2.fencer = TestNodeFencer.SetupFencer(typeof(TestNodeFencer.AlwaysSucceedFencer ).FullName); try { DoFailover(svc1, svc2, false, false); NUnit.Framework.Assert.Fail("Can't failover when access is denied"); } catch (FailoverFailedException ffe) { Assert.True(ffe.InnerException.Message.Contains("Access denied" )); } }
/// <summary> /// Test that the ZKFC successfully quits the election when it fails to /// become active. /// </summary> /// <remarks> /// Test that the ZKFC successfully quits the election when it fails to /// become active. This allows the old node to successfully fail back. /// </remarks> /// <exception cref="System.Exception"/> public virtual void TestBecomingActiveFails() { try { cluster.Start(); DummyHAService svc1 = cluster.GetService(1); Log.Info("Making svc1 fail to become active"); cluster.SetFailToBecomeActive(1, true); Log.Info("Faking svc0 unhealthy, should NOT successfully " + "failover to svc1"); cluster.SetHealthy(0, false); cluster.WaitForHealthState(0, HealthMonitor.State.ServiceUnhealthy); cluster.WaitForActiveLockHolder(null); Org.Mockito.Mockito.Verify(svc1.proxy, Org.Mockito.Mockito.Timeout(2000).AtLeastOnce ()).TransitionToActive(Org.Mockito.Mockito.Any <HAServiceProtocol.StateChangeRequestInfo >()); cluster.WaitForHAState(0, HAServiceProtocol.HAServiceState.Initializing); cluster.WaitForHAState(1, HAServiceProtocol.HAServiceState.Standby); Log.Info("Faking svc0 healthy again, should go back to svc0"); cluster.SetHealthy(0, true); cluster.WaitForHAState(0, HAServiceProtocol.HAServiceState.Active); cluster.WaitForHAState(1, HAServiceProtocol.HAServiceState.Standby); cluster.WaitForActiveLockHolder(0); // Ensure that we can fail back to svc1 once it it is able // to become active (e.g the admin has restarted it) Log.Info("Allowing svc1 to become active, expiring svc0"); svc1.failToBecomeActive = false; cluster.ExpireAndVerifyFailover(0, 1); } finally { cluster.Stop(); } }
/// <summary> /// Test that, when the health monitor indicates bad health status, /// failover is triggered. /// </summary> /// <remarks> /// Test that, when the health monitor indicates bad health status, /// failover is triggered. Also ensures that graceful active->standby /// transition is used when possible, falling back to fencing when /// the graceful approach fails. /// </remarks> /// <exception cref="System.Exception"/> public virtual void TestAutoFailoverOnBadHealth() { try { cluster.Start(); DummyHAService svc1 = cluster.GetService(1); Log.Info("Faking svc0 unhealthy, should failover to svc1"); cluster.SetHealthy(0, false); Log.Info("Waiting for svc0 to enter initializing state"); cluster.WaitForHAState(0, HAServiceProtocol.HAServiceState.Initializing); cluster.WaitForHAState(1, HAServiceProtocol.HAServiceState.Active); Log.Info("Allowing svc0 to be healthy again, making svc1 unreachable " + "and fail to gracefully go to standby" ); cluster.SetUnreachable(1, true); cluster.SetHealthy(0, true); // Should fail back to svc0 at this point cluster.WaitForHAState(0, HAServiceProtocol.HAServiceState.Active); // and fence svc1 Org.Mockito.Mockito.Verify(svc1.fencer).Fence(Org.Mockito.Mockito.Same(svc1)); } finally { cluster.Stop(); } }
public virtual void Release(DummyHAService oldHolder) { lock (this) { if (holder == oldHolder) { holder = null; } } }
// expected /// <summary> /// Test that the ZKFC won't run if fencing is not configured for the /// local service. /// </summary> /// <exception cref="System.Exception"/> public virtual void TestFencingMustBeConfigured() { DummyHAService svc = Org.Mockito.Mockito.Spy(cluster.GetService(0)); Org.Mockito.Mockito.DoThrow(new BadFencingConfigurationException("no fencing")).When (svc).CheckFencingConfigured(); // Format the base dir, should succeed Assert.Equal(0, RunFC(svc, "-formatZK")); // Try to run the actual FC, should fail without a fencer Assert.Equal(ZKFailoverController.ErrCodeNoFencer, RunFC(svc)); }
/// <summary> /// Test that automatic failover won't run against a target that hasn't /// explicitly enabled the feature. /// </summary> /// <exception cref="System.Exception"/> public virtual void TestWontRunWhenAutoFailoverDisabled() { DummyHAService svc = cluster.GetService(1); svc = Org.Mockito.Mockito.Spy(svc); Org.Mockito.Mockito.DoReturn(false).When(svc).IsAutoFailoverEnabled(); Assert.Equal(ZKFailoverController.ErrCodeAutoFailoverNotEnabled , RunFC(svc, "-formatZK")); Assert.Equal(ZKFailoverController.ErrCodeAutoFailoverNotEnabled , RunFC(svc)); }
public virtual void Take(DummyHAService newHolder) { lock (this) { if (holder == null || holder == newHolder) { holder = newHolder; } else { violations++; throw new InvalidOperationException("already held by: " + holder); } } }
/// <summary> /// Test that the various command lines for formatting the ZK directory /// function correctly. /// </summary> /// <exception cref="System.Exception"/> public virtual void TestFormatZK() { DummyHAService svc = cluster.GetService(1); // Run without formatting the base dir, // should barf Assert.Equal(ZKFailoverController.ErrCodeNoParentZnode, RunFC( svc)); // Format the base dir, should succeed Assert.Equal(0, RunFC(svc, "-formatZK")); // Should fail to format if already formatted Assert.Equal(ZKFailoverController.ErrCodeFormatDenied, RunFC(svc , "-formatZK", "-nonInteractive")); // Unless '-force' is on Assert.Equal(0, RunFC(svc, "-formatZK", "-force")); }
/// <summary> /// Test that, when the health monitor indicates bad health status, /// failover is triggered. /// </summary> /// <remarks> /// Test that, when the health monitor indicates bad health status, /// failover is triggered. Also ensures that graceful active->standby /// transition is used when possible, falling back to fencing when /// the graceful approach fails. /// </remarks> /// <exception cref="System.Exception"/> public virtual void TestAutoFailoverOnBadState() { try { cluster.Start(); DummyHAService svc0 = cluster.GetService(0); Log.Info("Faking svc0 to change the state, should failover to svc1"); svc0.state = HAServiceProtocol.HAServiceState.Standby; // Should fail back to svc0 at this point cluster.WaitForHAState(1, HAServiceProtocol.HAServiceState.Active); } finally { cluster.Stop(); } }
public virtual void SetupHM() { Configuration conf = new Configuration(); conf.SetInt(CommonConfigurationKeys.IpcClientConnectMaxRetriesKey, 1); conf.SetInt(CommonConfigurationKeys.HaHmCheckIntervalKey, 50); conf.SetInt(CommonConfigurationKeys.HaHmConnectRetryIntervalKey, 50); conf.SetInt(CommonConfigurationKeys.HaHmSleepAfterDisconnectKey, 50); svc = new DummyHAService(HAServiceProtocol.HAServiceState.Active, new IPEndPoint( "0.0.0.0", 0), true); hm = new _HealthMonitor_60(this, conf, svc); Log.Info("Starting health monitor"); hm.Start(); Log.Info("Waiting for HEALTHY signal"); WaitForState(hm, HealthMonitor.State.ServiceHealthy); }
public MiniZKFCCluster(Configuration conf, ZooKeeperServer zks) { this.conf = conf; // Fast check interval so tests run faster conf.SetInt(CommonConfigurationKeys.HaHmCheckIntervalKey, 50); conf.SetInt(CommonConfigurationKeys.HaHmConnectRetryIntervalKey, 50); conf.SetInt(CommonConfigurationKeys.HaHmSleepAfterDisconnectKey, 50); svcs = new DummyHAService[2]; svcs[0] = new DummyHAService(HAServiceProtocol.HAServiceState.Initializing, new IPEndPoint ("svc1", 1234)); svcs[0].SetSharedResource(sharedResource); svcs[1] = new DummyHAService(HAServiceProtocol.HAServiceState.Initializing, new IPEndPoint ("svc2", 1234)); svcs[1].SetSharedResource(sharedResource); this.ctx = new MultithreadedTestUtil.TestContext(); this.zks = zks; }
/// <exception cref="Org.Apache.Hadoop.HA.BadFencingConfigurationException"/> public virtual bool TryFence(HAServiceTarget target, string args) { Log.Info("tryFence(" + target + ")"); DummyHAService svc = (DummyHAService)target; lock (svc) { svc.fenceCount++; } if (svc.failToFence) { Log.Info("Injected failure to fence"); return(false); } svc.sharedResource.Release(svc); return(true); }
/// <summary> /// Test that, if ACLs are specified in the configuration, that /// it sets the ACLs when formatting the parent node. /// </summary> /// <exception cref="System.Exception"/> public virtual void TestFormatSetsAcls() { // Format the base dir, should succeed DummyHAService svc = cluster.GetService(1); Assert.Equal(0, RunFC(svc, "-formatZK")); ZooKeeper otherClient = CreateClient(); try { // client without auth should not be able to read it Stat stat = new Stat(); otherClient.GetData(ZKFailoverController.ZkParentZnodeDefault, false, stat); NUnit.Framework.Assert.Fail("Was able to read data without authenticating!"); } catch (KeeperException.NoAuthException) { } }
public virtual void TestFailoverToNonExistantServiceFails() { DummyHAService svc1 = new DummyHAService(HAServiceProtocol.HAServiceState.Active, svc1Addr); DummyHAService svc2 = Org.Mockito.Mockito.Spy(new DummyHAService(null, svc2Addr)); Org.Mockito.Mockito.DoThrow(new IOException("Failed to connect")).When(svc2).GetProxy (Org.Mockito.Mockito.Any <Configuration>(), Org.Mockito.Mockito.AnyInt()); svc1.fencer = svc2.fencer = TestNodeFencer.SetupFencer(typeof(TestNodeFencer.AlwaysSucceedFencer ).FullName); try { DoFailover(svc1, svc2, false, false); NUnit.Framework.Assert.Fail("Failed over to a non-existant standby"); } catch (FailoverFailedException) { } // Expected Assert.Equal(HAServiceProtocol.HAServiceState.Active, svc1.state ); }
public virtual void TestFailoverFromActiveToActive() { DummyHAService svc1 = new DummyHAService(HAServiceProtocol.HAServiceState.Active, svc1Addr); DummyHAService svc2 = new DummyHAService(HAServiceProtocol.HAServiceState.Active, svc2Addr); svc1.fencer = svc2.fencer = TestNodeFencer.SetupFencer(typeof(TestNodeFencer.AlwaysSucceedFencer ).FullName); try { DoFailover(svc1, svc2, false, false); NUnit.Framework.Assert.Fail("Can't failover to an already active service"); } catch (FailoverFailedException) { } // Expected Assert.Equal(HAServiceProtocol.HAServiceState.Active, svc1.state ); Assert.Equal(HAServiceProtocol.HAServiceState.Active, svc2.state ); }
public virtual void TestFormatOneClusterLeavesOtherClustersAlone() { DummyHAService svc = cluster.GetService(1); MiniZKFCCluster.DummyZKFC zkfcInOtherCluster = new _DummyZKFC_116(conf, cluster.GetService (1)); // Run without formatting the base dir, // should barf Assert.Equal(ZKFailoverController.ErrCodeNoParentZnode, RunFC( svc)); // Format the base dir, should succeed Assert.Equal(0, RunFC(svc, "-formatZK")); // Run the other cluster without formatting, should barf because // it uses a different parent znode Assert.Equal(ZKFailoverController.ErrCodeNoParentZnode, zkfcInOtherCluster .Run(new string[] { })); // Should succeed in formatting the second cluster Assert.Equal(0, zkfcInOtherCluster.Run(new string[] { "-formatZK" })); // But should not have deleted the original base node from the first // cluster Assert.Equal(ZKFailoverController.ErrCodeFormatDenied, RunFC(svc , "-formatZK", "-nonInteractive")); }
public virtual void TestSelfFailoverFails() { DummyHAService svc1 = new DummyHAService(HAServiceProtocol.HAServiceState.Active, svc1Addr); DummyHAService svc2 = new DummyHAService(HAServiceProtocol.HAServiceState.Standby , svc2Addr); svc1.fencer = svc2.fencer = TestNodeFencer.SetupFencer(typeof(TestNodeFencer.AlwaysSucceedFencer ).FullName); TestNodeFencer.AlwaysSucceedFencer.fenceCalled = 0; try { DoFailover(svc1, svc1, false, false); NUnit.Framework.Assert.Fail("Can't failover to yourself"); } catch (FailoverFailedException) { } // Expected Assert.Equal(0, TestNodeFencer.AlwaysSucceedFencer.fenceCalled ); Assert.Equal(HAServiceProtocol.HAServiceState.Active, svc1.state ); try { DoFailover(svc2, svc2, false, false); NUnit.Framework.Assert.Fail("Can't failover to yourself"); } catch (FailoverFailedException) { } // Expected Assert.Equal(0, TestNodeFencer.AlwaysSucceedFencer.fenceCalled ); Assert.Equal(HAServiceProtocol.HAServiceState.Standby, svc2.state ); }