void IConnDelegate.OnResume(Conn c)
{
onConnResume();
}
internal void onConnClose(Conn c)
{
bool hasRDYRetryTimer = false;
string connAddr = c.ToString();
// remove this connections RDY count from the consumer's total
long rdyCount = c.RDY;
Interlocked.Add(ref _totalRdyCount, rdyCount * -1);
_rdyRetryMtx.EnterWriteLock();
try
{
Timer timer;
if (_rdyRetryTimers.TryGetValue(connAddr, out timer))
{
// stop any pending retry of an old RDY update
timer.Stop();
_rdyRetryTimers.Remove(connAddr);
hasRDYRetryTimer = true;
}
}
finally
{
_rdyRetryMtx.ExitWriteLock();
}
int left;
_mtx.EnterWriteLock();
try
{
_connections.Remove(connAddr);
left = _connections.Count;
}
finally
{
_mtx.ExitWriteLock();
}
var connsAlivelogLevel = (_stopFlag == 1 ? LogLevel.Info : LogLevel.Warning);
log(connsAlivelogLevel, string.Format("there are {0} connections left alive", left));
if ((hasRDYRetryTimer || rdyCount > 0) &&
(left == getMaxInFlight() || inBackoff()))
{
// we're toggling out of (normal) redistribution cases and this conn
// had a RDY count...
//
// trigger RDY redistribution to make sure this RDY is moved
// to a new connection
_needRdyRedistributed = 1;
}
if (_stopFlag == 1)
{
if (left == 0)
{
stopHandlers();
}
return;
}
int numLookupd;
bool reconnect;
_mtx.EnterReadLock();
try
{
numLookupd = _lookupdHTTPAddrs.Count;
reconnect = _nsqdTCPAddrs.Contains(connAddr);
}
finally
{
_mtx.ExitReadLock();
}
if (numLookupd > 0)
{
// trigger a poll of the lookupd
Select
.CaseSend(_lookupdRecheckChan, 1)
.Default(func: null);
}
else if (reconnect)
{
// there are no lookupd and we still have this nsqd TCP address in our list...
// try to reconnect after a bit
GoFunc.Run(() =>
{
while (true)
{
// TODO: PR go-nsq: do they need .Seconds() on their r.log string?
// https://github.com/bitly/go-nsq/blob/667c739c212e55a5ddde2a33d4be2b9376d2c7e5/consumer.go#L731
log(LogLevel.Info, string.Format("({0}) re-connecting in {1:0.0000} seconds...", connAddr,
_config.LookupdPollInterval.TotalSeconds));
Thread.Sleep(_config.LookupdPollInterval);
if (_stopFlag == 1)
{
break;
}
_mtx.EnterReadLock();
reconnect = _nsqdTCPAddrs.Contains(connAddr);
_mtx.ExitReadLock();
if (!reconnect)
{
log(LogLevel.Warning, string.Format("({0}) skipped reconnect after removal...", connAddr));
return;
}
try
{
ConnectToNsqd(connAddr);
}
catch (Exception ex)
{
log(LogLevel.Error, string.Format("({0}) error connecting to nsqd - {1}", connAddr, ex));
continue;
// TODO: PR go-nsq if we get DialTimeout this loop stops. check other exceptions.
}
break;
}
}, string.Format("onConnClose:reconnect: {0}/{1}", _topic, _channel));
}
}
private void sendRDY(Conn c, long count)
{
if (count == 0 && c.LastRDY == 0)
{
// no need to send. It's already that RDY count
return;
}
Interlocked.Add(ref _totalRdyCount, -c.RDY + count);
c.SetRDY(count);
try
{
c.WriteCommand(Command.Ready(count));
}
catch (Exception ex)
{
log(LogLevel.Error, string.Format("({0}) error sending RDY {1} - {2}", c, count, ex));
throw;
}
}
void IConnDelegate.OnResponse(Conn c, byte[] data)
{
onConnResponse(c, data);
}
internal void onConnMessageFinished(Conn c, Message msg)
{
Interlocked.Increment(ref _messagesFinished);
}
internal void onConnResponse(Conn c, byte[] data)
{
if (CLOSE_WAIT_BYTES.SequenceEqual(data))
{
// server is ready for us to close (it ack'd our StartClose)
// we can assume we will not receive any more messages over this channel
// (but we can still write back responses)
log(LogLevel.Info, string.Format("({0}) received CLOSE_WAIT from nsqd", c));
c.Close();
}
}
void IConnDelegate.OnHeartbeat(Conn c)
{
onConnHeartbeat(c);
}
internal void onConnIOError(Conn c, Exception err)
{
c.Close();
}
void IConnDelegate.OnContinue(Conn c)
{
onConnContinue();
}
void IConnDelegate.OnError(Conn c, byte[] data)
{
onConnError(c, data);
}
void IConnDelegate.OnClose(Conn c)
{
onConnClose(c);
}
void IConnDelegate.OnBackoff(Conn c)
{
onConnBackoff();
}
private Exception updateRDY(Conn c, long count)
{
try
{
if (c.IsClosing)
{
throw new ErrClosing();
}
// never exceed the nsqd's configured max RDY count
if (count > c.MaxRDY)
count = c.MaxRDY;
string connAddr = c.ToString();
// stop any pending retry of an old RDY update
_rdyRetryMtx.EnterWriteLock();
try
{
Timer timer;
if (_rdyRetryTimers.TryGetValue(connAddr, out timer))
{
timer.Stop();
_rdyRetryTimers.Remove(connAddr);
}
}
finally
{
_rdyRetryMtx.ExitWriteLock();
}
// never exceed our global max in flight. truncate if possible.
// this could help a new connection get partial max-in-flight
long rdyCount = c.RDY;
long maxPossibleRdy = getMaxInFlight() - _totalRdyCount + rdyCount;
if (maxPossibleRdy > 0 && maxPossibleRdy < count)
{
count = maxPossibleRdy;
}
else if (maxPossibleRdy <= 0 && count > 0)
{
// TODO: PR go-nsq: add "else" for clarity
if (rdyCount == 0)
{
// we wanted to exit a zero RDY count but we couldn't send it...
// in order to prevent eternal starvation we reschedule this attempt
// (if any other RDY update succeeds this timer will be stopped)
_rdyRetryMtx.EnterWriteLock();
try
{
_rdyRetryTimers[connAddr] = Time.AfterFunc(TimeSpan.FromSeconds(5),
() => updateRDY(c, count));
}
finally
{
_rdyRetryMtx.ExitWriteLock();
}
}
throw new ErrOverMaxInFlight();
}
sendRDY(c, count);
}
catch (Exception ex)
{
// NOTE: errors intentionally not rethrown
log(ex is ErrClosing ? LogLevel.Warning : LogLevel.Error,
string.Format("({0}) error in updateRDY {1} - {2}", c, count, ex));
return ex;
}
return null;
}
internal void onConnError(Conn c, byte[] data)
{
}
void IConnDelegate.OnIOError(Conn c, Exception err)
{
onConnIOError(c, err);
}
internal void onConnHeartbeat(Conn c)
{
}
void IConnDelegate.OnMessage(Conn c, Message m)
{
onConnMessage(c, m);
}
internal void onConnMessage(Conn c, Message msg)
{
Interlocked.Decrement(ref _totalRdyCount);
Interlocked.Increment(ref _messagesReceived);
_incomingMessages.Send(msg);
maybeUpdateRDY(c);
}
void IConnDelegate.OnMessageFinished(Conn c, Message m)
{
onConnMessageFinished(c, m);
}
internal void onConnMessageRequeued(Conn c, Message msg)
{
Interlocked.Increment(ref _messagesRequeued);
}
void IConnDelegate.OnMessageRequeued(Conn c, Message m)
{
onConnMessageRequeued(c, m);
}
private void connectToNsqd(string addr)
{
if (string.IsNullOrEmpty(addr))
throw new ArgumentNullException("addr");
if (_stopFlag == 1)
{
throw new Exception("consumer stopped");
}
if (_runningHandlers == 0)
{
throw new Exception("no handlers");
}
_connectedFlag = 1;
var conn = new Conn(addr, _config, this);
// TODO: Check log format
conn.SetLogger(_logger, string.Format("C{0} [{1}/{2}] ({{0}})", _id, _topic, _channel));
_mtx.EnterWriteLock();
try
{
bool pendingOk = _pendingConnections.ContainsKey(addr);
bool ok = _connections.ContainsKey(addr);
if (pendingOk || ok)
{
return;
}
_pendingConnections[addr] = conn;
if (!_nsqdTCPAddrs.Contains(addr))
_nsqdTCPAddrs.Add(addr);
}
finally
{
_mtx.ExitWriteLock();
}
log(LogLevel.Info, string.Format("({0}) connecting to nsqd", addr));
var cleanupConnection = new Action(() =>
{
_mtx.EnterWriteLock();
try
{
_pendingConnections.Remove(addr);
}
finally
{
_mtx.ExitWriteLock();
}
});
IdentifyResponse resp;
try
{
resp = conn.Connect();
}
catch (Exception)
{
cleanupConnection();
throw;
}
if (resp != null)
{
if (resp.MaxRdyCount < getMaxInFlight())
{
log(LogLevel.Warning, string.Format(
"({0}) max RDY count {1} < consumer max in flight {2}, truncation possible",
conn, resp.MaxRdyCount, getMaxInFlight()));
}
}
var cmd = Command.Subscribe(_topic, _channel);
try
{
conn.WriteCommand(cmd);
}
catch (Exception ex)
{
cleanupConnection();
throw new Exception(string.Format("[{0}] failed to subscribe to {1}:{2} - {3}",
conn, _topic, _channel, ex));
}
_mtx.EnterWriteLock();
try
{
_pendingConnections.Remove(addr);
_connections[addr] = conn;
}
finally
{
_mtx.ExitWriteLock();
}
// pre-emptive signal to existing connections to lower their RDY count
_perConnMaxInFlightOverride = 0;
foreach (var c in conns())
{
maybeUpdateRDY(c);
}
}
private void maybeUpdateRDY(Conn conn)
{
var isInBackoff = inBackoff();
var isInBackoffTimeout = inBackoffTimeout();
if (isInBackoff || isInBackoffTimeout)
{
log(LogLevel.Debug, string.Format("({0}) skip sending RDY inBackoff:{1} || inBackoffTimeout:{2}",
conn, isInBackoff, isInBackoffTimeout));
return;
}
long remain = conn.RDY;
long lastRdyCount = conn.LastRDY;
long count = perConnMaxInFlight();
// refill when at 1, or at 25%, or if connections have changed and we're imbalanced
if (remain <= 1 || remain < (lastRdyCount / 4) || (count > 0 && count < remain))
{
log(LogLevel.Debug, string.Format("({0}) sending RDY {1} ({2} remain from last RDY {3})",
conn, count, remain, lastRdyCount));
updateRDY(conn, count);
}
else
{
log(LogLevel.Debug, string.Format("({0}) skip sending RDY {1} ({2} remain out of last RDY {3})",
conn, count, remain, lastRdyCount));
}
}
