private void DoFence(HAServiceTarget target)
{
Log.Info("Should fence: " + target);
bool gracefulWorked = new FailoverController(conf, HAServiceProtocol.RequestSource
.RequestByZkfc).TryGracefulFence(target);
if (gracefulWorked)
{
// It's possible that it's in standby but just about to go into active,
// no? Is there some race here?
Log.Info("Successfully transitioned " + target + " to standby " + "state without fencing"
);
return;
}
try
{
target.CheckFencingConfigured();
}
catch (BadFencingConfigurationException e)
{
Log.Error("Couldn't fence old active " + target, e);
RecordActiveAttempt(new ZKFailoverController.ActiveAttemptRecord(false, "Unable to fence old active"
));
throw new RuntimeException(e);
}
if (!target.GetFencer().Fence(target))
{
throw new RuntimeException("Unable to fence " + target);
}
}
/// <summary>Failover from service 1 to service 2.</summary>
/// <remarks>
/// Failover from service 1 to service 2. If the failover fails
/// then try to failback.
/// </remarks>
/// <param name="fromSvc">currently active service</param>
/// <param name="toSvc">service to make active</param>
/// <param name="forceFence">to fence fromSvc even if not strictly necessary</param>
/// <param name="forceActive">try to make toSvc active even if it is not ready</param>
/// <exception cref="FailoverFailedException">if the failover fails</exception>
/// <exception cref="Org.Apache.Hadoop.HA.FailoverFailedException"/>
public virtual void Failover(HAServiceTarget fromSvc, HAServiceTarget toSvc, bool
forceFence, bool forceActive)
{
Preconditions.CheckArgument(fromSvc.GetFencer() != null, "failover requires a fencer"
);
PreFailoverChecks(fromSvc, toSvc, forceActive);
// Try to make fromSvc standby
bool tryFence = true;
if (TryGracefulFence(fromSvc))
{
tryFence = forceFence;
}
// Fence fromSvc if it's required or forced by the user
if (tryFence)
{
if (!fromSvc.GetFencer().Fence(fromSvc))
{
throw new FailoverFailedException("Unable to fence " + fromSvc + ". Fencing failed."
);
}
}
// Try to make toSvc active
bool failed = false;
Exception cause = null;
try
{
HAServiceProtocolHelper.TransitionToActive(toSvc.GetProxy(conf, rpcTimeoutToNewActive
), CreateReqInfo());
}
catch (ServiceFailedException sfe)
{
Log.Error("Unable to make " + toSvc + " active (" + sfe.Message + "). Failing back."
);
failed = true;
cause = sfe;
}
catch (IOException ioe)
{
Log.Error("Unable to make " + toSvc + " active (unable to connect). Failing back."
, ioe);
failed = true;
cause = ioe;
}
// We failed to make toSvc active
if (failed)
{
string msg = "Unable to failover to " + toSvc;
// Only try to failback if we didn't fence fromSvc
if (!tryFence)
{
try
{
// Unconditionally fence toSvc in case it is still trying to
// become active, eg we timed out waiting for its response.
// Unconditionally force fromSvc to become active since it
// was previously active when we initiated failover.
Failover(toSvc, fromSvc, true, true);
}
catch (FailoverFailedException ffe)
{
msg += ". Failback to " + fromSvc + " failed (" + ffe.Message + ")";
Log.Fatal(msg);
}
}
throw new FailoverFailedException(msg, cause);
}
}