/// <summary>Wait for the given HA service to become the active lock holder.</summary>
/// <remarks>
/// Wait for the given HA service to become the active lock holder.
/// If the passed svc is null, waits for there to be no active
/// lock holder.
/// </remarks>
/// <exception cref="System.Exception"/>
public virtual void WaitForActiveLockHolder(int idx)
{
DummyHAService svc = idx == null ? null : svcs[idx];
ActiveStandbyElectorTestUtil.WaitForActiveLockData(ctx, zks, MiniZKFCCluster.DummyZKFC
.ScopedParentZnode, (idx == null) ? null : Ints.ToByteArray(svc.index));
}
/// <exception cref="System.Exception"/>
public virtual void TestHandleSessionExpiration()
{
ActiveStandbyElector.ActiveStandbyElectorCallback cb = cbs[0];
byte[] appData = appDatas[0];
ActiveStandbyElector elector = electors[0];
// Let the first elector become active
elector.EnsureParentZNode();
elector.JoinElection(appData);
ZooKeeperServer zks = GetServer(serverFactory);
ActiveStandbyElectorTestUtil.WaitForActiveLockData(null, zks, ParentDir, appData);
Org.Mockito.Mockito.Verify(cb, Org.Mockito.Mockito.Timeout(1000)).BecomeActive();
CheckFatalsAndReset();
Log.Info("========================== Expiring session");
zks.CloseSession(elector.GetZKSessionIdForTests());
// Should enter neutral mode when disconnected
Org.Mockito.Mockito.Verify(cb, Org.Mockito.Mockito.Timeout(1000)).EnterNeutralMode
();
// Should re-join the election and regain active
ActiveStandbyElectorTestUtil.WaitForActiveLockData(null, zks, ParentDir, appData);
Org.Mockito.Mockito.Verify(cb, Org.Mockito.Mockito.Timeout(1000)).BecomeActive();
CheckFatalsAndReset();
Log.Info("========================== Quitting election");
elector.QuitElection(false);
ActiveStandbyElectorTestUtil.WaitForActiveLockData(null, zks, ParentDir, null);
// Double check that we don't accidentally re-join the election
// due to receiving the "expired" event.
Thread.Sleep(1000);
Org.Mockito.Mockito.Verify(cb, Org.Mockito.Mockito.Never()).BecomeActive();
ActiveStandbyElectorTestUtil.WaitForActiveLockData(null, zks, ParentDir, null);
CheckFatalsAndReset();
}
/// <exception cref="System.Exception"/>
public virtual void TestDontJoinElectionOnDisconnectAndReconnect()
{
electors[0].EnsureParentZNode();
StopServer();
ActiveStandbyElectorTestUtil.WaitForElectorState(null, electors[0], ActiveStandbyElector.State
.Neutral);
StartServer();
WaitForServerUp(hostPort, ConnectionTimeout);
// Have to sleep to allow time for the clients to reconnect.
Thread.Sleep(2000);
Org.Mockito.Mockito.Verify(cbs[0], Org.Mockito.Mockito.Never()).BecomeActive();
Org.Mockito.Mockito.Verify(cbs[1], Org.Mockito.Mockito.Never()).BecomeActive();
CheckFatalsAndReset();
}
/// <exception cref="System.Exception"/>
public virtual void TestHandleSessionExpirationOfStandby()
{
// Let elector 0 be active
electors[0].EnsureParentZNode();
electors[0].JoinElection(appDatas[0]);
ZooKeeperServer zks = GetServer(serverFactory);
ActiveStandbyElectorTestUtil.WaitForActiveLockData(null, zks, ParentDir, appDatas
[0]);
Org.Mockito.Mockito.Verify(cbs[0], Org.Mockito.Mockito.Timeout(1000)).BecomeActive
();
CheckFatalsAndReset();
// Let elector 1 be standby
electors[1].JoinElection(appDatas[1]);
ActiveStandbyElectorTestUtil.WaitForElectorState(null, electors[1], ActiveStandbyElector.State
.Standby);
Log.Info("========================== Expiring standby's session");
zks.CloseSession(electors[1].GetZKSessionIdForTests());
// Should enter neutral mode when disconnected
Org.Mockito.Mockito.Verify(cbs[1], Org.Mockito.Mockito.Timeout(1000)).EnterNeutralMode
();
// Should re-join the election and go back to STANDBY
ActiveStandbyElectorTestUtil.WaitForElectorState(null, electors[1], ActiveStandbyElector.State
.Standby);
CheckFatalsAndReset();
Log.Info("========================== Quitting election");
electors[1].QuitElection(false);
// Double check that we don't accidentally re-join the election
// by quitting elector 0 and ensuring elector 1 doesn't become active
electors[0].QuitElection(false);
// due to receiving the "expired" event.
Thread.Sleep(1000);
Org.Mockito.Mockito.Verify(cbs[1], Org.Mockito.Mockito.Never()).BecomeActive();
ActiveStandbyElectorTestUtil.WaitForActiveLockData(null, zks, ParentDir, null);
CheckFatalsAndReset();
}
/// <summary>Wait for the given elector to enter the given elector state.</summary>
/// <param name="idx">the service index (0 or 1)</param>
/// <param name="state">the state to wait for</param>
/// <exception cref="System.Exception">
/// if it times out, or an exception occurs on one
/// of the ZKFC threads while waiting.
/// </exception>
public virtual void WaitForElectorState(int idx, ActiveStandbyElector.State state
)
{
ActiveStandbyElectorTestUtil.WaitForElectorState(ctx, GetElector(idx), state);
}
/// <summary>
/// the test creates 2 electors which try to become active using a real
/// zookeeper server.
/// </summary>
/// <remarks>
/// the test creates 2 electors which try to become active using a real
/// zookeeper server. It verifies that 1 becomes active and 1 becomes standby.
/// Upon becoming active the leader quits election and the test verifies that
/// the standby now becomes active.
/// </remarks>
/// <exception cref="System.Exception"/>
public virtual void TestActiveStandbyTransition()
{
Log.Info("starting test with parentDir:" + ParentDir);
NUnit.Framework.Assert.IsFalse(electors[0].ParentZNodeExists());
electors[0].EnsureParentZNode();
Assert.True(electors[0].ParentZNodeExists());
// First elector joins election, becomes active.
electors[0].JoinElection(appDatas[0]);
ActiveStandbyElectorTestUtil.WaitForActiveLockData(null, zkServer, ParentDir, appDatas
[0]);
Org.Mockito.Mockito.Verify(cbs[0], Org.Mockito.Mockito.Timeout(1000)).BecomeActive
();
CheckFatalsAndReset();
// Second elector joins election, becomes standby.
electors[1].JoinElection(appDatas[1]);
Org.Mockito.Mockito.Verify(cbs[1], Org.Mockito.Mockito.Timeout(1000)).BecomeStandby
();
CheckFatalsAndReset();
// First elector quits, second one should become active
electors[0].QuitElection(true);
ActiveStandbyElectorTestUtil.WaitForActiveLockData(null, zkServer, ParentDir, appDatas
[1]);
Org.Mockito.Mockito.Verify(cbs[1], Org.Mockito.Mockito.Timeout(1000)).BecomeActive
();
CheckFatalsAndReset();
// First one rejoins, becomes standby, second one stays active
electors[0].JoinElection(appDatas[0]);
Org.Mockito.Mockito.Verify(cbs[0], Org.Mockito.Mockito.Timeout(1000)).BecomeStandby
();
CheckFatalsAndReset();
// Second one expires, first one becomes active
electors[1].PreventSessionReestablishmentForTests();
try
{
zkServer.CloseSession(electors[1].GetZKSessionIdForTests());
ActiveStandbyElectorTestUtil.WaitForActiveLockData(null, zkServer, ParentDir, appDatas
[0]);
Org.Mockito.Mockito.Verify(cbs[1], Org.Mockito.Mockito.Timeout(1000)).EnterNeutralMode
();
Org.Mockito.Mockito.Verify(cbs[0], Org.Mockito.Mockito.Timeout(1000)).FenceOldActive
(AdditionalMatchers.AryEq(appDatas[1]));
Org.Mockito.Mockito.Verify(cbs[0], Org.Mockito.Mockito.Timeout(1000)).BecomeActive
();
}
finally
{
electors[1].AllowSessionReestablishmentForTests();
}
// Second one eventually reconnects and becomes standby
Org.Mockito.Mockito.Verify(cbs[1], Org.Mockito.Mockito.Timeout(5000)).BecomeStandby
();
CheckFatalsAndReset();
// First one expires, second one should become active
electors[0].PreventSessionReestablishmentForTests();
try
{
zkServer.CloseSession(electors[0].GetZKSessionIdForTests());
ActiveStandbyElectorTestUtil.WaitForActiveLockData(null, zkServer, ParentDir, appDatas
[1]);
Org.Mockito.Mockito.Verify(cbs[0], Org.Mockito.Mockito.Timeout(1000)).EnterNeutralMode
();
Org.Mockito.Mockito.Verify(cbs[1], Org.Mockito.Mockito.Timeout(1000)).FenceOldActive
(AdditionalMatchers.AryEq(appDatas[0]));
Org.Mockito.Mockito.Verify(cbs[1], Org.Mockito.Mockito.Timeout(1000)).BecomeActive
();
}
finally
{
electors[0].AllowSessionReestablishmentForTests();
}
CheckFatalsAndReset();
}
