Exemplo n.º 1
0
 /*
  * TODO:
  * we need to make sure that if we get fenced and then quickly restarted,
  * none of these calls will retry across the restart boundary
  * perhaps the solution is that, whenever the nn starts, it gets a unique
  * ID, and when we start becoming active, we record it, and then any future
  * calls use the same ID
  */
 /// <summary>Store the results of the last attempt to become active.</summary>
 /// <remarks>
 /// Store the results of the last attempt to become active.
 /// This is used so that, during manually initiated failover,
 /// we can report back the results of the attempt to become active
 /// to the initiator of the failover.
 /// </remarks>
 private void RecordActiveAttempt(ZKFailoverController.ActiveAttemptRecord record)
 {
     lock (activeAttemptRecordLock)
     {
         lastActiveAttemptRecord = record;
         Runtime.NotifyAll(activeAttemptRecordLock);
     }
 }
Exemplo n.º 2
0
        /// <summary>Coordinate a graceful failover.</summary>
        /// <remarks>
        /// Coordinate a graceful failover. This proceeds in several phases:
        /// 1) Pre-flight checks: ensure that the local node is healthy, and
        /// thus a candidate for failover.
        /// 2) Determine the current active node. If it is the local node, no
        /// need to failover - return success.
        /// 3) Ask that node to yield from the election for a number of seconds.
        /// 4) Allow the normal election path to run in other threads. Wait until
        /// we either become unhealthy or we see an election attempt recorded by
        /// the normal code path.
        /// 5) Allow the old active to rejoin the election, so a future
        /// failback is possible.
        /// </remarks>
        /// <exception cref="Org.Apache.Hadoop.HA.ServiceFailedException"/>
        /// <exception cref="System.IO.IOException"/>
        /// <exception cref="System.Exception"/>
        private void DoGracefulFailover()
        {
            int timeout = FailoverController.GetGracefulFenceTimeout(conf) * 2;

            // Phase 1: pre-flight checks
            CheckEligibleForFailover();
            // Phase 2: determine old/current active node. Check that we're not
            // ourselves active, etc.
            HAServiceTarget oldActive = GetCurrentActive();

            if (oldActive == null)
            {
                // No node is currently active. So, if we aren't already
                // active ourselves by means of a normal election, then there's
                // probably something preventing us from becoming active.
                throw new ServiceFailedException("No other node is currently active.");
            }
            if (oldActive.GetAddress().Equals(localTarget.GetAddress()))
            {
                Log.Info("Local node " + localTarget + " is already active. " + "No need to failover. Returning success."
                         );
                return;
            }
            // Phase 3: ask the old active to yield from the election.
            Log.Info("Asking " + oldActive + " to cede its active state for " + timeout + "ms"
                     );
            ZKFCProtocol oldZkfc = oldActive.GetZKFCProxy(conf, timeout);

            oldZkfc.CedeActive(timeout);
            // Phase 4: wait for the normal election to make the local node
            // active.
            ZKFailoverController.ActiveAttemptRecord attempt = WaitForActiveAttempt(timeout +
                                                                                    60000);
            if (attempt == null)
            {
                // We didn't even make an attempt to become active.
                lock (this)
                {
                    if (lastHealthState != HealthMonitor.State.ServiceHealthy)
                    {
                        throw new ServiceFailedException("Unable to become active. " + "Service became unhealthy while trying to failover."
                                                         );
                    }
                }
                throw new ServiceFailedException("Unable to become active. " + "Local node did not get an opportunity to do so from ZooKeeper, "
                                                 + "or the local node took too long to transition to active.");
            }
            // Phase 5. At this point, we made some attempt to become active. So we
            // can tell the old active to rejoin if it wants. This allows a quick
            // fail-back if we immediately crash.
            oldZkfc.CedeActive(-1);
            if (attempt.succeeded)
            {
                Log.Info("Successfully became active. " + attempt.status);
            }
            else
            {
                // Propagate failure
                string msg = "Failed to become active. " + attempt.status;
                throw new ServiceFailedException(msg);
            }
        }