internal void onConnClose(Conn c) { bool hasRDYRetryTimer = false; string connAddr = c.ToString(); // remove this connections RDY count from the consumer's total long rdyCount = c.RDY; Interlocked.Add(ref _totalRdyCount, rdyCount * -1); _rdyRetryMtx.EnterWriteLock(); try { Timer timer; if (_rdyRetryTimers.TryGetValue(connAddr, out timer)) { // stop any pending retry of an old RDY update timer.Stop(); _rdyRetryTimers.Remove(connAddr); hasRDYRetryTimer = true; } } finally { _rdyRetryMtx.ExitWriteLock(); } int left; _mtx.EnterWriteLock(); try { _connections.Remove(connAddr); left = _connections.Count; } finally { _mtx.ExitWriteLock(); } var connsAlivelogLevel = (_stopFlag == 1 ? LogLevel.Info : LogLevel.Warning); log(connsAlivelogLevel, string.Format("there are {0} connections left alive", left)); if ((hasRDYRetryTimer || rdyCount > 0) && (left == getMaxInFlight() || inBackoff())) { // we're toggling out of (normal) redistribution cases and this conn // had a RDY count... // // trigger RDY redistribution to make sure this RDY is moved // to a new connection _needRdyRedistributed = 1; } if (_stopFlag == 1) { if (left == 0) { stopHandlers(); } return; } int numLookupd; bool reconnect; _mtx.EnterReadLock(); try { numLookupd = _lookupdHTTPAddrs.Count; reconnect = _nsqdTCPAddrs.Contains(connAddr); } finally { _mtx.ExitReadLock(); } if (numLookupd > 0) { // trigger a poll of the lookupd Select .CaseSend(_lookupdRecheckChan, 1) .Default(func: null); } else if (reconnect) { // there are no lookupd and we still have this nsqd TCP address in our list... // try to reconnect after a bit GoFunc.Run(() => { while (true) { // TODO: PR go-nsq: do they need .Seconds() on their r.log string? // https://github.com/bitly/go-nsq/blob/667c739c212e55a5ddde2a33d4be2b9376d2c7e5/consumer.go#L731 log(LogLevel.Info, string.Format("({0}) re-connecting in {1:0.0000} seconds...", connAddr, _config.LookupdPollInterval.TotalSeconds)); Thread.Sleep(_config.LookupdPollInterval); if (_stopFlag == 1) { break; } _mtx.EnterReadLock(); reconnect = _nsqdTCPAddrs.Contains(connAddr); _mtx.ExitReadLock(); if (!reconnect) { log(LogLevel.Warning, string.Format("({0}) skipped reconnect after removal...", connAddr)); return; } try { ConnectToNsqd(connAddr); } catch (Exception ex) { log(LogLevel.Error, string.Format("({0}) error connecting to nsqd - {1}", connAddr, ex)); continue; // TODO: PR go-nsq if we get DialTimeout this loop stops. check other exceptions. } break; } }, string.Format("onConnClose:reconnect: {0}/{1}", _topic, _channel)); } }
private Exception updateRDY(Conn c, long count) { try { if (c.IsClosing) { throw new ErrClosing(); } // never exceed the nsqd's configured max RDY count if (count > c.MaxRDY) count = c.MaxRDY; string connAddr = c.ToString(); // stop any pending retry of an old RDY update _rdyRetryMtx.EnterWriteLock(); try { Timer timer; if (_rdyRetryTimers.TryGetValue(connAddr, out timer)) { timer.Stop(); _rdyRetryTimers.Remove(connAddr); } } finally { _rdyRetryMtx.ExitWriteLock(); } // never exceed our global max in flight. truncate if possible. // this could help a new connection get partial max-in-flight long rdyCount = c.RDY; long maxPossibleRdy = getMaxInFlight() - _totalRdyCount + rdyCount; if (maxPossibleRdy > 0 && maxPossibleRdy < count) { count = maxPossibleRdy; } else if (maxPossibleRdy <= 0 && count > 0) { // TODO: PR go-nsq: add "else" for clarity if (rdyCount == 0) { // we wanted to exit a zero RDY count but we couldn't send it... // in order to prevent eternal starvation we reschedule this attempt // (if any other RDY update succeeds this timer will be stopped) _rdyRetryMtx.EnterWriteLock(); try { _rdyRetryTimers[connAddr] = Time.AfterFunc(TimeSpan.FromSeconds(5), () => updateRDY(c, count)); } finally { _rdyRetryMtx.ExitWriteLock(); } } throw new ErrOverMaxInFlight(); } sendRDY(c, count); } catch (Exception ex) { // NOTE: errors intentionally not rethrown log(ex is ErrClosing ? LogLevel.Warning : LogLevel.Error, string.Format("({0}) error in updateRDY {1} - {2}", c, count, ex)); return ex; } return null; }