void IConnDelegate.OnResume(Conn c) { onConnResume(); }
internal void onConnClose(Conn c) { bool hasRDYRetryTimer = false; string connAddr = c.ToString(); // remove this connections RDY count from the consumer's total long rdyCount = c.RDY; Interlocked.Add(ref _totalRdyCount, rdyCount * -1); _rdyRetryMtx.EnterWriteLock(); try { Timer timer; if (_rdyRetryTimers.TryGetValue(connAddr, out timer)) { // stop any pending retry of an old RDY update timer.Stop(); _rdyRetryTimers.Remove(connAddr); hasRDYRetryTimer = true; } } finally { _rdyRetryMtx.ExitWriteLock(); } int left; _mtx.EnterWriteLock(); try { _connections.Remove(connAddr); left = _connections.Count; } finally { _mtx.ExitWriteLock(); } var connsAlivelogLevel = (_stopFlag == 1 ? LogLevel.Info : LogLevel.Warning); log(connsAlivelogLevel, string.Format("there are {0} connections left alive", left)); if ((hasRDYRetryTimer || rdyCount > 0) && (left == getMaxInFlight() || inBackoff())) { // we're toggling out of (normal) redistribution cases and this conn // had a RDY count... // // trigger RDY redistribution to make sure this RDY is moved // to a new connection _needRdyRedistributed = 1; } if (_stopFlag == 1) { if (left == 0) { stopHandlers(); } return; } int numLookupd; bool reconnect; _mtx.EnterReadLock(); try { numLookupd = _lookupdHTTPAddrs.Count; reconnect = _nsqdTCPAddrs.Contains(connAddr); } finally { _mtx.ExitReadLock(); } if (numLookupd > 0) { // trigger a poll of the lookupd Select .CaseSend(_lookupdRecheckChan, 1) .Default(func: null); } else if (reconnect) { // there are no lookupd and we still have this nsqd TCP address in our list... // try to reconnect after a bit GoFunc.Run(() => { while (true) { // TODO: PR go-nsq: do they need .Seconds() on their r.log string? // https://github.com/bitly/go-nsq/blob/667c739c212e55a5ddde2a33d4be2b9376d2c7e5/consumer.go#L731 log(LogLevel.Info, string.Format("({0}) re-connecting in {1:0.0000} seconds...", connAddr, _config.LookupdPollInterval.TotalSeconds)); Thread.Sleep(_config.LookupdPollInterval); if (_stopFlag == 1) { break; } _mtx.EnterReadLock(); reconnect = _nsqdTCPAddrs.Contains(connAddr); _mtx.ExitReadLock(); if (!reconnect) { log(LogLevel.Warning, string.Format("({0}) skipped reconnect after removal...", connAddr)); return; } try { ConnectToNsqd(connAddr); } catch (Exception ex) { log(LogLevel.Error, string.Format("({0}) error connecting to nsqd - {1}", connAddr, ex)); continue; // TODO: PR go-nsq if we get DialTimeout this loop stops. check other exceptions. } break; } }, string.Format("onConnClose:reconnect: {0}/{1}", _topic, _channel)); } }
private void sendRDY(Conn c, long count) { if (count == 0 && c.LastRDY == 0) { // no need to send. It's already that RDY count return; } Interlocked.Add(ref _totalRdyCount, -c.RDY + count); c.SetRDY(count); try { c.WriteCommand(Command.Ready(count)); } catch (Exception ex) { log(LogLevel.Error, string.Format("({0}) error sending RDY {1} - {2}", c, count, ex)); throw; } }
void IConnDelegate.OnResponse(Conn c, byte[] data) { onConnResponse(c, data); }
internal void onConnMessageFinished(Conn c, Message msg) { Interlocked.Increment(ref _messagesFinished); }
internal void onConnResponse(Conn c, byte[] data) { if (CLOSE_WAIT_BYTES.SequenceEqual(data)) { // server is ready for us to close (it ack'd our StartClose) // we can assume we will not receive any more messages over this channel // (but we can still write back responses) log(LogLevel.Info, string.Format("({0}) received CLOSE_WAIT from nsqd", c)); c.Close(); } }
void IConnDelegate.OnHeartbeat(Conn c) { onConnHeartbeat(c); }
internal void onConnIOError(Conn c, Exception err) { c.Close(); }
void IConnDelegate.OnContinue(Conn c) { onConnContinue(); }
void IConnDelegate.OnError(Conn c, byte[] data) { onConnError(c, data); }
void IConnDelegate.OnClose(Conn c) { onConnClose(c); }
void IConnDelegate.OnBackoff(Conn c) { onConnBackoff(); }
private Exception updateRDY(Conn c, long count) { try { if (c.IsClosing) { throw new ErrClosing(); } // never exceed the nsqd's configured max RDY count if (count > c.MaxRDY) count = c.MaxRDY; string connAddr = c.ToString(); // stop any pending retry of an old RDY update _rdyRetryMtx.EnterWriteLock(); try { Timer timer; if (_rdyRetryTimers.TryGetValue(connAddr, out timer)) { timer.Stop(); _rdyRetryTimers.Remove(connAddr); } } finally { _rdyRetryMtx.ExitWriteLock(); } // never exceed our global max in flight. truncate if possible. // this could help a new connection get partial max-in-flight long rdyCount = c.RDY; long maxPossibleRdy = getMaxInFlight() - _totalRdyCount + rdyCount; if (maxPossibleRdy > 0 && maxPossibleRdy < count) { count = maxPossibleRdy; } else if (maxPossibleRdy <= 0 && count > 0) { // TODO: PR go-nsq: add "else" for clarity if (rdyCount == 0) { // we wanted to exit a zero RDY count but we couldn't send it... // in order to prevent eternal starvation we reschedule this attempt // (if any other RDY update succeeds this timer will be stopped) _rdyRetryMtx.EnterWriteLock(); try { _rdyRetryTimers[connAddr] = Time.AfterFunc(TimeSpan.FromSeconds(5), () => updateRDY(c, count)); } finally { _rdyRetryMtx.ExitWriteLock(); } } throw new ErrOverMaxInFlight(); } sendRDY(c, count); } catch (Exception ex) { // NOTE: errors intentionally not rethrown log(ex is ErrClosing ? LogLevel.Warning : LogLevel.Error, string.Format("({0}) error in updateRDY {1} - {2}", c, count, ex)); return ex; } return null; }
internal void onConnError(Conn c, byte[] data) { }
void IConnDelegate.OnIOError(Conn c, Exception err) { onConnIOError(c, err); }
internal void onConnHeartbeat(Conn c) { }
void IConnDelegate.OnMessage(Conn c, Message m) { onConnMessage(c, m); }
internal void onConnMessage(Conn c, Message msg) { Interlocked.Decrement(ref _totalRdyCount); Interlocked.Increment(ref _messagesReceived); _incomingMessages.Send(msg); maybeUpdateRDY(c); }
void IConnDelegate.OnMessageFinished(Conn c, Message m) { onConnMessageFinished(c, m); }
internal void onConnMessageRequeued(Conn c, Message msg) { Interlocked.Increment(ref _messagesRequeued); }
void IConnDelegate.OnMessageRequeued(Conn c, Message m) { onConnMessageRequeued(c, m); }
private void connectToNsqd(string addr) { if (string.IsNullOrEmpty(addr)) throw new ArgumentNullException("addr"); if (_stopFlag == 1) { throw new Exception("consumer stopped"); } if (_runningHandlers == 0) { throw new Exception("no handlers"); } _connectedFlag = 1; var conn = new Conn(addr, _config, this); // TODO: Check log format conn.SetLogger(_logger, string.Format("C{0} [{1}/{2}] ({{0}})", _id, _topic, _channel)); _mtx.EnterWriteLock(); try { bool pendingOk = _pendingConnections.ContainsKey(addr); bool ok = _connections.ContainsKey(addr); if (pendingOk || ok) { return; } _pendingConnections[addr] = conn; if (!_nsqdTCPAddrs.Contains(addr)) _nsqdTCPAddrs.Add(addr); } finally { _mtx.ExitWriteLock(); } log(LogLevel.Info, string.Format("({0}) connecting to nsqd", addr)); var cleanupConnection = new Action(() => { _mtx.EnterWriteLock(); try { _pendingConnections.Remove(addr); } finally { _mtx.ExitWriteLock(); } }); IdentifyResponse resp; try { resp = conn.Connect(); } catch (Exception) { cleanupConnection(); throw; } if (resp != null) { if (resp.MaxRdyCount < getMaxInFlight()) { log(LogLevel.Warning, string.Format( "({0}) max RDY count {1} < consumer max in flight {2}, truncation possible", conn, resp.MaxRdyCount, getMaxInFlight())); } } var cmd = Command.Subscribe(_topic, _channel); try { conn.WriteCommand(cmd); } catch (Exception ex) { cleanupConnection(); throw new Exception(string.Format("[{0}] failed to subscribe to {1}:{2} - {3}", conn, _topic, _channel, ex)); } _mtx.EnterWriteLock(); try { _pendingConnections.Remove(addr); _connections[addr] = conn; } finally { _mtx.ExitWriteLock(); } // pre-emptive signal to existing connections to lower their RDY count _perConnMaxInFlightOverride = 0; foreach (var c in conns()) { maybeUpdateRDY(c); } }
private void maybeUpdateRDY(Conn conn) { var isInBackoff = inBackoff(); var isInBackoffTimeout = inBackoffTimeout(); if (isInBackoff || isInBackoffTimeout) { log(LogLevel.Debug, string.Format("({0}) skip sending RDY inBackoff:{1} || inBackoffTimeout:{2}", conn, isInBackoff, isInBackoffTimeout)); return; } long remain = conn.RDY; long lastRdyCount = conn.LastRDY; long count = perConnMaxInFlight(); // refill when at 1, or at 25%, or if connections have changed and we're imbalanced if (remain <= 1 || remain < (lastRdyCount / 4) || (count > 0 && count < remain)) { log(LogLevel.Debug, string.Format("({0}) sending RDY {1} ({2} remain from last RDY {3})", conn, count, remain, lastRdyCount)); updateRDY(conn, count); } else { log(LogLevel.Debug, string.Format("({0}) skip sending RDY {1} ({2} remain out of last RDY {3})", conn, count, remain, lastRdyCount)); } }