/// <summary>Test that the ZKFC can gracefully cede its active status.</summary> /// <exception cref="System.Exception"/> public virtual void TestCedeActive() { try { cluster.Start(); MiniZKFCCluster.DummyZKFC zkfc = cluster.GetZkfc(0); // It should be in active to start. Assert.Equal(ActiveStandbyElector.State.Active, zkfc.GetElectorForTests ().GetStateForTests()); // Ask it to cede active for 3 seconds. It should respond promptly // (i.e. the RPC itself should not take 3 seconds!) ZKFCProtocol proxy = zkfc.GetLocalTarget().GetZKFCProxy(conf, 5000); long st = Time.Now(); proxy.CedeActive(3000); long et = Time.Now(); Assert.True("RPC to cedeActive took " + (et - st) + " ms", et - st < 1000); // Should be in "INIT" state since it's not in the election // at this point. Assert.Equal(ActiveStandbyElector.State.Init, zkfc.GetElectorForTests ().GetStateForTests()); // After the prescribed 3 seconds, should go into STANDBY state, // since the other node in the cluster would have taken ACTIVE. cluster.WaitForElectorState(0, ActiveStandbyElector.State.Standby); long et2 = Time.Now(); Assert.True("Should take ~3 seconds to rejoin. Only took " + (et2 - et) + "ms before rejoining.", et2 - et > 2800); } finally { cluster.Stop(); } }
/// <summary>Coordinate a graceful failover.</summary> /// <remarks> /// Coordinate a graceful failover. This proceeds in several phases: /// 1) Pre-flight checks: ensure that the local node is healthy, and /// thus a candidate for failover. /// 2) Determine the current active node. If it is the local node, no /// need to failover - return success. /// 3) Ask that node to yield from the election for a number of seconds. /// 4) Allow the normal election path to run in other threads. Wait until /// we either become unhealthy or we see an election attempt recorded by /// the normal code path. /// 5) Allow the old active to rejoin the election, so a future /// failback is possible. /// </remarks> /// <exception cref="Org.Apache.Hadoop.HA.ServiceFailedException"/> /// <exception cref="System.IO.IOException"/> /// <exception cref="System.Exception"/> private void DoGracefulFailover() { int timeout = FailoverController.GetGracefulFenceTimeout(conf) * 2; // Phase 1: pre-flight checks CheckEligibleForFailover(); // Phase 2: determine old/current active node. Check that we're not // ourselves active, etc. HAServiceTarget oldActive = GetCurrentActive(); if (oldActive == null) { // No node is currently active. So, if we aren't already // active ourselves by means of a normal election, then there's // probably something preventing us from becoming active. throw new ServiceFailedException("No other node is currently active."); } if (oldActive.GetAddress().Equals(localTarget.GetAddress())) { Log.Info("Local node " + localTarget + " is already active. " + "No need to failover. Returning success." ); return; } // Phase 3: ask the old active to yield from the election. Log.Info("Asking " + oldActive + " to cede its active state for " + timeout + "ms" ); ZKFCProtocol oldZkfc = oldActive.GetZKFCProxy(conf, timeout); oldZkfc.CedeActive(timeout); // Phase 4: wait for the normal election to make the local node // active. ZKFailoverController.ActiveAttemptRecord attempt = WaitForActiveAttempt(timeout + 60000); if (attempt == null) { // We didn't even make an attempt to become active. lock (this) { if (lastHealthState != HealthMonitor.State.ServiceHealthy) { throw new ServiceFailedException("Unable to become active. " + "Service became unhealthy while trying to failover." ); } } throw new ServiceFailedException("Unable to become active. " + "Local node did not get an opportunity to do so from ZooKeeper, " + "or the local node took too long to transition to active."); } // Phase 5. At this point, we made some attempt to become active. So we // can tell the old active to rejoin if it wants. This allows a quick // fail-back if we immediately crash. oldZkfc.CedeActive(-1); if (attempt.succeeded) { Log.Info("Successfully became active. " + attempt.status); } else { // Propagate failure string msg = "Failed to become active. " + attempt.status; throw new ServiceFailedException(msg); } }