internal void onConnClose(Conn c)
{
bool hasRDYRetryTimer = false;
string connAddr = c.ToString();
// remove this connections RDY count from the consumer's total
long rdyCount = c.RDY;
Interlocked.Add(ref _totalRdyCount, rdyCount * -1);
_rdyRetryMtx.EnterWriteLock();
try
{
Timer timer;
if (_rdyRetryTimers.TryGetValue(connAddr, out timer))
{
// stop any pending retry of an old RDY update
timer.Stop();
_rdyRetryTimers.Remove(connAddr);
hasRDYRetryTimer = true;
}
}
finally
{
_rdyRetryMtx.ExitWriteLock();
}
int left;
_mtx.EnterWriteLock();
try
{
_connections.Remove(connAddr);
left = _connections.Count;
}
finally
{
_mtx.ExitWriteLock();
}
var connsAlivelogLevel = (_stopFlag == 1 ? LogLevel.Info : LogLevel.Warning);
log(connsAlivelogLevel, string.Format("there are {0} connections left alive", left));
if ((hasRDYRetryTimer || rdyCount > 0) &&
(left == getMaxInFlight() || inBackoff()))
{
// we're toggling out of (normal) redistribution cases and this conn
// had a RDY count...
//
// trigger RDY redistribution to make sure this RDY is moved
// to a new connection
_needRdyRedistributed = 1;
}
if (_stopFlag == 1)
{
if (left == 0)
{
stopHandlers();
}
return;
}
int numLookupd;
bool reconnect;
_mtx.EnterReadLock();
try
{
numLookupd = _lookupdHTTPAddrs.Count;
reconnect = _nsqdTCPAddrs.Contains(connAddr);
}
finally
{
_mtx.ExitReadLock();
}
if (numLookupd > 0)
{
// trigger a poll of the lookupd
Select
.CaseSend(_lookupdRecheckChan, 1)
.Default(func: null);
}
else if (reconnect)
{
// there are no lookupd and we still have this nsqd TCP address in our list...
// try to reconnect after a bit
GoFunc.Run(() =>
{
while (true)
{
// TODO: PR go-nsq: do they need .Seconds() on their r.log string?
// https://github.com/bitly/go-nsq/blob/667c739c212e55a5ddde2a33d4be2b9376d2c7e5/consumer.go#L731
log(LogLevel.Info, string.Format("({0}) re-connecting in {1:0.0000} seconds...", connAddr,
_config.LookupdPollInterval.TotalSeconds));
Thread.Sleep(_config.LookupdPollInterval);
if (_stopFlag == 1)
{
break;
}
_mtx.EnterReadLock();
reconnect = _nsqdTCPAddrs.Contains(connAddr);
_mtx.ExitReadLock();
if (!reconnect)
{
log(LogLevel.Warning, string.Format("({0}) skipped reconnect after removal...", connAddr));
return;
}
try
{
ConnectToNsqd(connAddr);
}
catch (Exception ex)
{
log(LogLevel.Error, string.Format("({0}) error connecting to nsqd - {1}", connAddr, ex));
continue;
// TODO: PR go-nsq if we get DialTimeout this loop stops. check other exceptions.
}
break;
}
}, string.Format("onConnClose:reconnect: {0}/{1}", _topic, _channel));
}
}
private Exception updateRDY(Conn c, long count)
{
try
{
if (c.IsClosing)
{
throw new ErrClosing();
}
// never exceed the nsqd's configured max RDY count
if (count > c.MaxRDY)
count = c.MaxRDY;
string connAddr = c.ToString();
// stop any pending retry of an old RDY update
_rdyRetryMtx.EnterWriteLock();
try
{
Timer timer;
if (_rdyRetryTimers.TryGetValue(connAddr, out timer))
{
timer.Stop();
_rdyRetryTimers.Remove(connAddr);
}
}
finally
{
_rdyRetryMtx.ExitWriteLock();
}
// never exceed our global max in flight. truncate if possible.
// this could help a new connection get partial max-in-flight
long rdyCount = c.RDY;
long maxPossibleRdy = getMaxInFlight() - _totalRdyCount + rdyCount;
if (maxPossibleRdy > 0 && maxPossibleRdy < count)
{
count = maxPossibleRdy;
}
else if (maxPossibleRdy <= 0 && count > 0)
{
// TODO: PR go-nsq: add "else" for clarity
if (rdyCount == 0)
{
// we wanted to exit a zero RDY count but we couldn't send it...
// in order to prevent eternal starvation we reschedule this attempt
// (if any other RDY update succeeds this timer will be stopped)
_rdyRetryMtx.EnterWriteLock();
try
{
_rdyRetryTimers[connAddr] = Time.AfterFunc(TimeSpan.FromSeconds(5),
() => updateRDY(c, count));
}
finally
{
_rdyRetryMtx.ExitWriteLock();
}
}
throw new ErrOverMaxInFlight();
}
sendRDY(c, count);
}
catch (Exception ex)
{
// NOTE: errors intentionally not rethrown
log(ex is ErrClosing ? LogLevel.Warning : LogLevel.Error,
string.Format("({0}) error in updateRDY {1} - {2}", c, count, ex));
return ex;
}
return null;
}