public virtual void TestFailoverFromNonExistantServiceWithFencer()
{
DummyHAService svc1 = Org.Mockito.Mockito.Spy(new DummyHAService(null, svc1Addr));
// Getting a proxy to a dead server will throw IOException on call,
// not on creation of the proxy.
HAServiceProtocol errorThrowingProxy = Org.Mockito.Mockito.Mock <HAServiceProtocol
>(Org.Mockito.Mockito.WithSettings().DefaultAnswer(new ThrowsException(new IOException
("Could not connect to host"))).ExtraInterfaces(typeof(IDisposable)));
Org.Mockito.Mockito.DoNothing().When((IDisposable)errorThrowingProxy).Close();
Org.Mockito.Mockito.DoReturn(errorThrowingProxy).When(svc1).GetProxy(Org.Mockito.Mockito
.Any <Configuration>(), Org.Mockito.Mockito.AnyInt());
DummyHAService svc2 = new DummyHAService(HAServiceProtocol.HAServiceState.Standby
, svc2Addr);
svc1.fencer = svc2.fencer = TestNodeFencer.SetupFencer(typeof(TestNodeFencer.AlwaysSucceedFencer
).FullName);
try
{
DoFailover(svc1, svc2, false, false);
}
catch (FailoverFailedException)
{
NUnit.Framework.Assert.Fail("Non-existant active prevented failover");
}
// Verify that the proxy created to try to make it go to standby
// gracefully used the right rpc timeout
Org.Mockito.Mockito.Verify(svc1).GetProxy(Org.Mockito.Mockito.Any <Configuration>(
), Org.Mockito.Mockito.Eq(CommonConfigurationKeys.HaFcGracefulFenceTimeoutDefault
));
// Don't check svc1 because we can't reach it, but that's OK, it's been fenced.
Assert.Equal(HAServiceProtocol.HAServiceState.Active, svc2.state
);
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="Org.Apache.Hadoop.HA.ServiceFailedException"/>
private int TransitionToActive(CommandLine cmd)
{
string[] argv = cmd.GetArgs();
if (argv.Length != 1)
{
errOut.WriteLine("transitionToActive: incorrect number of arguments");
PrintUsage(errOut, "-transitionToActive");
return(-1);
}
/* returns true if other target node is active or some exception occurred
* and forceActive was not set */
if (!cmd.HasOption(Forceactive))
{
if (IsOtherTargetNodeActive(argv[0], cmd.HasOption(Forceactive)))
{
return(-1);
}
}
HAServiceTarget target = ResolveTarget(argv[0]);
if (!CheckManualStateManagementOK(target))
{
return(-1);
}
HAServiceProtocol proto = target.GetProxy(GetConf(), 0);
HAServiceProtocolHelper.TransitionToActive(proto, CreateReqInfo());
return(0);
}
internal DummyHAService(HAServiceProtocol.HAServiceState state, IPEndPoint address
, bool testWithProtoBufRPC)
{
this.state = state;
this.testWithProtoBufRPC = testWithProtoBufRPC;
if (testWithProtoBufRPC)
{
this.address = StartAndGetRPCServerAddress(address);
}
else
{
this.address = address;
}
Configuration conf = new Configuration();
this.proxy = MakeMock(conf, CommonConfigurationKeys.HaHmRpcTimeoutDefault);
try
{
conf.Set(DummyFenceKey, typeof(DummyHAService.DummyFencer).FullName);
this.fencer = Org.Mockito.Mockito.Spy(NodeFencer.Create(conf, DummyFenceKey));
}
catch (BadFencingConfigurationException e)
{
throw new RuntimeException(e);
}
lock (instances)
{
instances.AddItem(this);
this.index = instances.Count;
}
}
/// <summary>Try to get the HA state of the node at the given address.</summary>
/// <remarks>
/// Try to get the HA state of the node at the given address. This
/// function is guaranteed to be "quick" -- ie it has a short timeout
/// and no retries. Its only purpose is to avoid fencing a node that
/// has already restarted.
/// </remarks>
internal virtual bool TryGracefulFence(HAServiceTarget svc)
{
HAServiceProtocol proxy = null;
try
{
proxy = svc.GetProxy(gracefulFenceConf, gracefulFenceTimeout);
proxy.TransitionToStandby(CreateReqInfo());
return(true);
}
catch (ServiceFailedException sfe)
{
Log.Warn("Unable to gracefully make " + svc + " standby (" + sfe.Message + ")");
}
catch (IOException ioe)
{
Log.Warn("Unable to gracefully make " + svc + " standby (unable to connect)", ioe
);
}
finally
{
if (proxy != null)
{
RPC.StopProxy(proxy);
}
}
return(false);
}
/// <exception cref="System.IO.IOException"/>
public override HAServiceProtocol GetProxy(Configuration conf, int timeout)
{
if (testWithProtoBufRPC)
{
proxy = MakeMock(conf, timeout);
}
return(proxy);
}
/// <exception cref="System.IO.IOException"/>
public static void TransitionToStandby(HAServiceProtocol svc, HAServiceProtocol.StateChangeRequestInfo
reqInfo)
{
try
{
svc.TransitionToStandby(reqInfo);
}
catch (RemoteException e)
{
throw e.UnwrapRemoteException(typeof(ServiceFailedException));
}
}
/// <exception cref="System.IO.IOException"/>
public static void MonitorHealth(HAServiceProtocol svc, HAServiceProtocol.StateChangeRequestInfo
reqInfo)
{
try
{
svc.MonitorHealth();
}
catch (RemoteException e)
{
throw e.UnwrapRemoteException(typeof(HealthCheckFailedException));
}
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="Org.Apache.Hadoop.HA.ServiceFailedException"/>
private int GetServiceState(CommandLine cmd)
{
string[] argv = cmd.GetArgs();
if (argv.Length != 1)
{
errOut.WriteLine("getServiceState: incorrect number of arguments");
PrintUsage(errOut, "-getServiceState");
return(-1);
}
HAServiceProtocol proto = ResolveTarget(argv[0]).GetProxy(GetConf(), rpcTimeoutForChecks
);
@out.WriteLine(proto.GetServiceStatus().GetState());
return(0);
}
private void TryConnect()
{
Preconditions.CheckState(proxy == null);
try
{
lock (this)
{
proxy = CreateProxy();
}
}
catch (IOException e)
{
Log.Warn("Could not connect to local service at " + targetToMonitor + ": " + e.Message
);
proxy = null;
EnterState(HealthMonitor.State.ServiceNotResponding);
}
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="Org.Apache.Hadoop.HA.ServiceFailedException"/>
private int TransitionToStandby(CommandLine cmd)
{
string[] argv = cmd.GetArgs();
if (argv.Length != 1)
{
errOut.WriteLine("transitionToStandby: incorrect number of arguments");
PrintUsage(errOut, "-transitionToStandby");
return(-1);
}
HAServiceTarget target = ResolveTarget(argv[0]);
if (!CheckManualStateManagementOK(target))
{
return(-1);
}
HAServiceProtocol proto = target.GetProxy(GetConf(), 0);
HAServiceProtocolHelper.TransitionToStandby(proto, CreateReqInfo());
return(0);
}
/// <exception cref="System.Exception"/>
private void DoHealthChecks()
{
while (shouldRun)
{
HAServiceStatus status = null;
bool healthy = false;
try
{
status = proxy.GetServiceStatus();
proxy.MonitorHealth();
healthy = true;
}
catch (Exception t)
{
if (IsHealthCheckFailedException(t))
{
Log.Warn("Service health check failed for " + targetToMonitor + ": " + t.Message);
EnterState(HealthMonitor.State.ServiceUnhealthy);
}
else
{
Log.Warn("Transport-level exception trying to monitor health of " + targetToMonitor
+ ": " + t.InnerException + " " + t.GetLocalizedMessage());
RPC.StopProxy(proxy);
proxy = null;
EnterState(HealthMonitor.State.ServiceNotResponding);
Thread.Sleep(sleepAfterDisconnectMillis);
return;
}
}
if (status != null)
{
SetLastServiceStatus(status);
}
if (healthy)
{
EnterState(HealthMonitor.State.ServiceHealthy);
}
Thread.Sleep(checkIntervalMillis);
}
}
/// <exception cref="System.IO.IOException"/>
/// <exception cref="Org.Apache.Hadoop.HA.ServiceFailedException"/>
private int CheckHealth(CommandLine cmd)
{
string[] argv = cmd.GetArgs();
if (argv.Length != 1)
{
errOut.WriteLine("checkHealth: incorrect number of arguments");
PrintUsage(errOut, "-checkHealth");
return(-1);
}
HAServiceProtocol proto = ResolveTarget(argv[0]).GetProxy(GetConf(), rpcTimeoutForChecks
);
try
{
HAServiceProtocolHelper.MonitorHealth(proto, CreateReqInfo());
}
catch (HealthCheckFailedException e)
{
errOut.WriteLine("Health check failed: " + e.GetLocalizedMessage());
return(-1);
}
return(0);
}
/// <summary>Checks whether other target node is active or not</summary>
/// <param name="targetNodeToActivate"/>
/// <returns>
/// true if other target node is active or some other exception
/// occurred and forceActive was set otherwise false
/// </returns>
/// <exception cref="System.IO.IOException"/>
private bool IsOtherTargetNodeActive(string targetNodeToActivate, bool forceActive
)
{
ICollection <string> targetIds = GetTargetIds(targetNodeToActivate);
targetIds.Remove(targetNodeToActivate);
foreach (string targetId in targetIds)
{
HAServiceTarget target = ResolveTarget(targetId);
if (!CheckManualStateManagementOK(target))
{
return(true);
}
try
{
HAServiceProtocol proto = target.GetProxy(GetConf(), 5000);
if (proto.GetServiceStatus().GetState() == HAServiceProtocol.HAServiceState.Active)
{
errOut.WriteLine("transitionToActive: Node " + targetId + " is already active");
PrintUsage(errOut, "-transitionToActive");
return(true);
}
}
catch (Exception e)
{
//If forceActive switch is false then return true
if (!forceActive)
{
errOut.WriteLine("Unexpected error occurred " + e.Message);
PrintUsage(errOut, "-transitionToActive");
return(true);
}
}
}
return(false);
}
