private void startConnect(ResolvedEndPoint addr)
{
state.Value = (int)ZooKeeper.States.CONNECTING;
logStartConnect(addr);
clientCnxnSocket.connect(addr);
}
private async Task startConnect()
{
state.Value = ZooKeeper.States.CONNECTING;
ResolvedEndPoint addr;
if (rwServerAddress != null)
{
addr = rwServerAddress;
rwServerAddress = null;
}
else
{
addr = await hostProvider.next(1000).ConfigureAwait(false);
}
logStartConnect(addr);
clientCnxnSocket.connect(addr);
}
private void AssertState(DynamicHostProvider dynamicHostProvider, int currentIndex, ResolvedEndPoint lastIP, bool resolvingInBackground, bool firstDnsTry = false)
{
Xunit.Assert.Equal(currentIndex, dynamicHostProvider.CurrentIndex);
Xunit.Assert.Equal(lastIP, dynamicHostProvider.LastIP);
Xunit.Assert.Equal(firstDnsTry, dynamicHostProvider.FirstDnsTry);
Xunit.Assert.Equal(resolvingInBackground, dynamicHostProvider.ResolvingInBackground);
}
public async Task TestAfterFirstDnsSuccess()
{
var fakeDnsResolver = new FakeDnsResolver();
var resolvedEndPoints = new List <ResolvedEndPoint>();
//adding a resolved address to dns resolution
var resolved1 = new ResolvedEndPoint(IPAddress.Loopback, 999);
resolvedEndPoints.Add(resolved1);
fakeDnsResolver.ResolveFunc = async() =>
{
await Task.Delay(200);
return(resolvedEndPoints);
};
var dynamicHostProvider = getHostProvider(1, fakeDnsResolver);
await dynamicHostProvider.next(0);
await dynamicHostProvider.next(0);
AssertState(dynamicHostProvider, currentIndex: 0, lastIP: resolved1, resolvingInBackground: false);
//clearing resolveEndPoints in order to make sure background resolving is ignored when it fails
resolvedEndPoints.Clear();
var sw = new Stopwatch();
sw.Restart();
//this should start a background dns resolving task that's expected to fail
var next = await dynamicHostProvider.next(100);
Xunit.Assert.Equal(next, resolved1);
AssertState(dynamicHostProvider, currentIndex: 0, lastIP: resolved1, resolvingInBackground: true);
Assert.assertTrue($"is {sw.ElapsedMilliseconds}", sw.ElapsedMilliseconds >= 99 && sw.ElapsedMilliseconds <= 200);
//the resolving runs in the background, we make sure we can still get the last ip
sw.Restart();
next = await dynamicHostProvider.next(10);
Xunit.Assert.Equal(next, resolved1);
AssertState(dynamicHostProvider, currentIndex: 0, lastIP: resolved1, resolvingInBackground: true);
Assert.assertTrue($"is {sw.ElapsedMilliseconds}", sw.ElapsedMilliseconds >= 9 && sw.ElapsedMilliseconds <= 200);
await Task.Delay(300);
//background resolving should have ended by now
AssertState(dynamicHostProvider, currentIndex: 0, lastIP: resolved1, resolvingInBackground: false);
//add a new ip to the returned list (we only have one now) this doesn't yet affect anything yet
var resolved2 = new ResolvedEndPoint(IPAddress.Loopback, 998);
resolvedEndPoints.Add(resolved2);
sw.Restart();
//make sure we still have the last known ip available
next = await dynamicHostProvider.next(100);
Xunit.Assert.Equal(next, resolved1);
//we should start another background dns resolving which should return 'resolve2'
AssertState(dynamicHostProvider, currentIndex: 0, lastIP: resolved1, resolvingInBackground: true);
Assert.assertTrue($"is {sw.ElapsedMilliseconds}", sw.ElapsedMilliseconds >= 99);
await Task.Delay(300);
//background resolving should have ended by now
AssertState(dynamicHostProvider, currentIndex: 0, lastIP: resolved1, resolvingInBackground: false);
sw.Restart();
//we expect to get 'resolve2' without the timeout since it's different than 'resolve1'
next = await dynamicHostProvider.next(1000);
Xunit.Assert.Equal(next, resolved2);
AssertState(dynamicHostProvider, currentIndex: 0, lastIP: resolved2, resolvingInBackground: false);
Assert.assertTrue($"is {sw.ElapsedMilliseconds}", sw.ElapsedMilliseconds <= 25);
//add back 'resolved1' to the returned list (the list is: 'resolved1','resolved2')
resolvedEndPoints.Add(resolved1);
sw.Restart();
//make sure we still have the last known ip available
next = await dynamicHostProvider.next(100);
Xunit.Assert.Equal(next, resolved2);
//we should start another background dns resolving which should return 'resolve1,resolve2'
AssertState(dynamicHostProvider, currentIndex: 0, lastIP: resolved2, resolvingInBackground: true);
Assert.assertTrue($"is {sw.ElapsedMilliseconds}", sw.ElapsedMilliseconds >= 99);
await Task.Delay(300);
//background resolving should have ended by now
AssertState(dynamicHostProvider, currentIndex: 0, lastIP: resolved2, resolvingInBackground: false);
sw.Restart();
//we expect to get 'resolve1' without the timeout since 'resolve2' was returned after a timeout
next = await dynamicHostProvider.next(1000);
Xunit.Assert.Equal(next, resolved1);
AssertState(dynamicHostProvider, currentIndex: 0, lastIP: resolved2, resolvingInBackground: false);
Assert.assertTrue($"is {sw.ElapsedMilliseconds}", sw.ElapsedMilliseconds <= 25);
sw.Restart();
//we expect to get 'resolve2' with the timeout since it's the LastIP
next = await dynamicHostProvider.next(400);
Xunit.Assert.Equal(next, resolved2);
AssertState(dynamicHostProvider, currentIndex: 1, lastIP: resolved2, resolvingInBackground: false);
Assert.assertTrue($"is {sw.ElapsedMilliseconds}", sw.ElapsedMilliseconds >= 399);
sw.Restart();
//we expect to get 'resolve1' without the timeout since 'resolve2' was returned after a timeout
next = await dynamicHostProvider.next(1000);
Xunit.Assert.Equal(next, resolved1);
AssertState(dynamicHostProvider, currentIndex: 0, lastIP: resolved2, resolvingInBackground: false);
Assert.assertTrue($"is {sw.ElapsedMilliseconds}", sw.ElapsedMilliseconds <= 25);
//notify we have successfully connected with 'resolve1'
dynamicHostProvider.onConnected();
sw.Restart();
//we expect to get 'resolve2' without the timeout since it's NOT the LastIP
next = await dynamicHostProvider.next(1000);
Xunit.Assert.Equal(next, resolved2);
AssertState(dynamicHostProvider, currentIndex: 1, lastIP: resolved1, resolvingInBackground: false);
Assert.assertTrue($"is {sw.ElapsedMilliseconds}", sw.ElapsedMilliseconds <= 25);
}
public async Task TestBeforeFirstDnsSuccess()
{
var fakeDnsResolver = new FakeDnsResolver();
var resolvedEndPoints = new List <ResolvedEndPoint>();
fakeDnsResolver.ResolveFunc = () => Task.FromResult((IEnumerable <ResolvedEndPoint>)resolvedEndPoints);
var sw = new Stopwatch();
sw.Start();
var dynamicHostProvider = getHostProvider(1, fakeDnsResolver);
//before calling next() for the first time
AssertState(dynamicHostProvider, currentIndex: -1, lastIP: null, resolvingInBackground: false,
firstDnsTry: true);
//expected to fail since no IPs were returned
await Xunit.Assert.ThrowsAsync <SocketException>(() => dynamicHostProvider.next(1000));
//FirstDnsTry is now false
AssertState(dynamicHostProvider, currentIndex: -1, lastIP: null, resolvingInBackground: false,
firstDnsTry: false);
//should have ignored given timeouts, since it was the first dns attempt
Assert.assertTrue($"is {sw.ElapsedMilliseconds}", sw.ElapsedMilliseconds <= 25);
sw.Restart();
//again the dns resolution returns an empty list, we expected to fail
await Xunit.Assert.ThrowsAsync <SocketException>(() => dynamicHostProvider.next(1000));
AssertState(dynamicHostProvider, currentIndex: -1, lastIP: null, resolvingInBackground: false);
//we expect the timeout to be used, since it's not the first dns attempt
Assert.assertTrue($"is {sw.ElapsedMilliseconds}", sw.ElapsedMilliseconds >= 990);
//adding a resolved address to dns resolution
var resolvedEndPoint1 = new ResolvedEndPoint(IPAddress.Loopback, 999);
resolvedEndPoints.Add(resolvedEndPoint1);
sw.Restart();
//this is the first successful dns attempt, after a failed one.
var next = await dynamicHostProvider.next(1000);
Xunit.Assert.Equal(resolvedEndPoint1, next);
AssertState(dynamicHostProvider, currentIndex: -1, lastIP: null, resolvingInBackground: false);
//we expect the timeout to be used, since it's not the first dns attempt
Assert.assertTrue($"is {sw.ElapsedMilliseconds}", sw.ElapsedMilliseconds >= 990);
sw.Restart();
//we asked for the next, knowing it would be the same address
next = await dynamicHostProvider.next(1000);
Xunit.Assert.Equal(next, resolvedEndPoint1);
AssertState(dynamicHostProvider, currentIndex: 0, lastIP: resolvedEndPoint1, resolvingInBackground: false);
//we ignore the timeout because it's the first time we're in the main loop
Assert.assertTrue($"is {sw.ElapsedMilliseconds}", sw.ElapsedMilliseconds <= 10);
sw.Restart();
//Creating a new host provider, this time the dns resolve will succeed on first try
dynamicHostProvider = getHostProvider(1, fakeDnsResolver);
next = await dynamicHostProvider.next(1000);
Xunit.Assert.Equal(next, resolvedEndPoint1);
AssertState(dynamicHostProvider, currentIndex: -1, lastIP: null, resolvingInBackground: false);
next = await dynamicHostProvider.next(1000);
Xunit.Assert.Equal(next, resolvedEndPoint1);
AssertState(dynamicHostProvider, currentIndex: 0, lastIP: resolvedEndPoint1, resolvingInBackground: false);
//we ignored the timeout since we have succeeded in resolving an address
Assert.assertTrue($"is {sw.ElapsedMilliseconds}", sw.ElapsedMilliseconds <= 25);
sw.Restart();
//we ask for the next address, but this time we expect the timeout. bcz we looped
next = await dynamicHostProvider.next(1000);
Xunit.Assert.Equal(next, resolvedEndPoint1);
AssertState(dynamicHostProvider, currentIndex: 0, lastIP: resolvedEndPoint1, resolvingInBackground: false);
Assert.assertTrue($"is {sw.ElapsedMilliseconds}", sw.ElapsedMilliseconds >= 990);
}
private async Task pingRwServer()
{
string result = null;
ResolvedEndPoint addr = await hostProvider.next(0).ConfigureAwait(false);
LOG.info("Checking server " + addr + " for being r/w." +
" Timeout " + pingRwTimeout);
TcpClient sock = null;
StreamReader br = null;
try {
sock = new TcpClient();
sock.LingerState = new LingerOption(false, 0);
sock.SendTimeout = 1000;
sock.NoDelay = true;
await sock.ConnectAsync(addr.Address, addr.Port).ConfigureAwait(false);
var networkStream = sock.GetStream();
await networkStream.WriteAsync("isro".UTF8getBytes(), 0, 4).ConfigureAwait(false);
await networkStream.FlushAsync().ConfigureAwait(false);
br = new StreamReader(networkStream);
result = await br.ReadLineAsync().ConfigureAwait(false);
}
catch (Exception e) {
var se = e.InnerException as SocketException;
if (se != null && se.SocketErrorCode == SocketError.ConnectionRefused)
{
// ignore, this just means server is not up
}
else
{
// some unexpected error, warn about it
LOG.warn("Exception while seeking for r/w server ", e);
}
}
if (sock != null)
{
try {
((IDisposable)sock).Dispose();
}
catch (Exception e) {
LOG.warn("Unexpected exception", e);
}
}
if (br != null)
{
try {
br.Dispose();
}
catch (Exception e) {
LOG.warn("Unexpected exception", e);
}
}
if ("rw" == result)
{
pingRwTimeout = minPingRwTimeout;
// save the found address so that it's used during the next
// connection attempt
rwServerAddress = addr;
throw new RWServerFoundException("Majority server found at " + addr);
}
}
private static void logStartConnect(ResolvedEndPoint addr)
{
string msg = "Opening socket connection to server " + addr;
LOG.info(msg);
}
private async Task startSendTask()
{
try {
clientCnxnSocket.introduce(sessionId);
clientCnxnSocket.updateNow();
clientCnxnSocket.updateLastSendAndHeard();
int to;
long lastPingRwServer = TimeHelper.ElapsedMiliseconds;
const int MAX_SEND_PING_INTERVAL = 10000; //10 seconds
ResolvedEndPoint serverAddress = null;
while (getState().isAlive())
{
try {
if (!clientCnxnSocket.isConnected())
{
if (!isFirstConnect)
{
await Task.Delay(r.Next(1000)).ConfigureAwait(false);
}
else
{
await hostProvider.next(1000).ConfigureAwait(false);
}
// don't re-establish connection if we are closing
if (closing.Value || !getState().isAlive())
{
break;
}
if (rwServerAddress != null)
{
serverAddress = rwServerAddress;
rwServerAddress = null;
}
else
{
serverAddress = await hostProvider.next(1000).ConfigureAwait(false);
}
startConnect(serverAddress);
clientCnxnSocket.updateLastSendAndHeard();
}
if (getState().isConnected())
{
to = readTimeout - clientCnxnSocket.getIdleRecv();
}
else
{
to = connectTimeout - clientCnxnSocket.getIdleRecv();
}
if (to <= 0)
{
string warnInfo;
warnInfo = "Client session timed out, have not heard from server in "
+ clientCnxnSocket.getIdleRecv()
+ "ms"
+ " for sessionid 0x"
+ clientCnxnSocket.sessionId.ToHexString();
LOG.warn(warnInfo);
throw new SessionTimeoutException(warnInfo);
}
if (getState().isConnected())
{
// 1000(1 second) is to prevent race condition missing to send the second ping
// also make sure not to send too many pings when readTimeout is small
int timeToNextPing = readTimeout / 2 - clientCnxnSocket.getIdleSend() -
((clientCnxnSocket.getIdleSend() > 1000) ? 1000 : 0);
// send a ping request either time is due or no packet sent out within MAX_SEND_PING_INTERVAL
if (timeToNextPing <= 0 || clientCnxnSocket.getIdleSend() > MAX_SEND_PING_INTERVAL)
{
sendPing();
clientCnxnSocket.updateLastSend();
}
else
{
if (timeToNextPing < to)
{
to = timeToNextPing;
}
}
}
// If we are in read-only mode, seek for read/write server
if (getState() == ZooKeeper.States.CONNECTEDREADONLY)
{
long now = TimeHelper.ElapsedMiliseconds;
int idlePingRwServer = (int)(now - lastPingRwServer);
if (idlePingRwServer >= pingRwTimeout)
{
lastPingRwServer = now;
idlePingRwServer = 0;
pingRwTimeout =
Math.Min(2 * pingRwTimeout, maxPingRwTimeout);
await pingRwServer().ConfigureAwait(false);
}
to = Math.Min(to, pingRwTimeout - idlePingRwServer);
}
if (to > 0 && !clientCnxnSocket.somethingIsPending.IsCompleted)
{
timer.Change(to, Timeout.Infinite);
await clientCnxnSocket.somethingIsPending;
timer.Change(Timeout.Infinite, Timeout.Infinite);
}
clientCnxnSocket.doTransport();
} catch (Exception e) {
if (closing.Value)
{
if (LOG.isDebugEnabled())
{
// closing so this is expected
LOG.debug("An exception was thrown while closing send task for session 0x" +
getSessionId().ToHexString(), e);
}
break;
}
// this is ugly, you have a better way speak up
if (e is SessionExpiredException)
{
LOG.info("closing socket connection", e);
}
else if (e is SessionTimeoutException)
{
LOG.info(RETRY_CONN_MSG, e);
}
else if (e is EndOfStreamException)
{
LOG.info(RETRY_CONN_MSG, e);
}
else if (e is RWServerFoundException)
{
LOG.info(e);
}
else if (e is SocketException)
{
LOG.info(string.Format("Socket error occurred: {0}", serverAddress), e);
}
else
{
LOG.warn(string.Format("Session 0x{0} for server {1}, unexpected error{2}",
getSessionId().ToHexString(),
serverAddress,
RETRY_CONN_MSG),
e);
}
cleanup();
if (getState().isAlive())
{
queueEvent(new WatchedEvent(
Watcher.Event.EventType.None,
Watcher.Event.KeeperState.Disconnected, null));
}
clientCnxnSocket.updateNow();
clientCnxnSocket.updateLastSendAndHeard();
}
}
cleanup();
clientCnxnSocket.close();
if (getState().isAlive())
{
queueEvent(new WatchedEvent(Watcher.Event.EventType.None,
Watcher.Event.KeeperState.Disconnected, null));
}
}
catch (Exception e) {
LOG.warn("Exception occurred in SendTask", e);
}
LOG.debug("SendTask exited loop for session: 0x"
+ getSessionId().ToHexString());
}
