private void HeartBeatTimer_Elapsed(object sender, NetMQTimerEventArgs e)
{
//check for last heartbeat from server, set to throw on new send if exceeds certain threshold
if (null != _session && null != _session.Crypto)
{
if ((DateTime.UtcNow - _session.LastHeartBeat).TotalMilliseconds
> _heartBeatMs * _maxSkippedHeartBeatReplies)
{
_throwOnSend = true; //do not allow send
_hostDead = true;
_logger.Debug("Heartbeat from server skipped {0} time. Host is dead.",
_maxSkippedHeartBeatReplies);
}
else
{
_logger.Debug("Heartbeat sent.");
HeartBeatsSentCount++;
_sendQueue.Enqueue(new List <byte[]> {
MessageHeader.HeartBeat
});
}
}
else
{
_throwOnSend = true; //do not allow send
}
}
/// <summary>
/// Sends out requests for historical data and raises an event when it's received.
/// </summary>
private void HistoricalDataTimerElapsed(object sender, NetMQTimerEventArgs e)
{
if (!ClientRunningAndIsConnected)
{
return;
}
// TODO: Solve issue with _poller and socket in Disconnect method and here
while (!historicalDataRequests.IsEmpty)
{
if (historicalDataRequests.TryDequeue(out HistoricalDataRequest request))
{
using (MemoryStream ms = new MemoryStream())
{
byte[] buffer = MyUtils.ProtoBufSerialize(request, ms);
lock (historicalDataSocketLock)
{
if (PollerRunning && historicalDataSocket != null)
{
historicalDataSocket.SendMoreFrame(BitConverter.GetBytes((byte)DataRequestMessageType.HistRequest));
historicalDataSocket.SendFrame(buffer);
}
else
{
return;
}
}
}
}
}
}
/// <summary>
/// Sends out requests for historical data and raises an event when it's received.
/// </summary>
private void HistoricalDataTimerElapsed(object sender, NetMQTimerEventArgs e)
{
if (!Connected)
{
return;
}
// TODO: Solve issue with _poller and socket in Disconnect method and here
while (!_historicalDataRequests.IsEmpty)
{
HistoricalDataRequest request;
if (_historicalDataRequests.TryDequeue(out request))
{
using (var ms = new MemoryStream())
{
var buffer = MyUtils.ProtoBufSerialize(request, ms);
lock (_historicalDataSocketLock)
{
if (PollerRunning && _historicalDataSocket != null)
{
_historicalDataSocket.SendMoreFrame(MessageType.HistRequest);
_historicalDataSocket.SendFrame(buffer);
}
else
{
return;
}
}
}
}
}
}
private void TimeoutElapsed(object sender, NetMQTimerEventArgs e)
{
// no need to reconnect, the client would be recreated because of RX
// because of RX internal stuff invoking on the poller thread block the entire application, so calling on Thread Pool
Task.Run(() => subject.OnError(new Exception("Disconnected from server")));
}
private void OnTimeoutTimer(object sender, NetMQTimerEventArgs e)
{
// dispose the current subscriber socket and try to connect
m_poller.RemoveSocket(m_subscriber);
m_subscriber.Dispose();
m_subscriber = null;
Connect();
}
private void Timer_Elapsed(object sender, NetMQTimerEventArgs e, byte[][] data)
{
try
{
_socket.SendData(data);
}
catch (Exception ex) { throw new NotImplementedException($"Timer_Elapsed: ZMQName {_socket.ZMQName} ---> {ex.Message}\r\n{ex.StackTrace}\r\n{ex.Source}"); }
}
private void OnTimer(object sender, NetMQTimerEventArgs e)
{
Action action;
while (_pollerTasks.TryDequeue(out action))
{
action();
}
}
private void OnTimeoutElapsed(NetMQTimerEventArgs args)
{
_parent._requiresInitialisation = 1;
var status = _borgConnected
? (ConnectionStatus.Offline | ConnectionStatus.Connected)
: (ConnectionStatus.Offline | ConnectionStatus.Disconnected);
_subject.OnNext(status);
}
/// <summary>
/// Sends heartbeat messages so we know that the server is still up.
/// </summary>
private void HeartBeatTimerElapsed(object sender, NetMQTimerEventArgs e)
{
lock (_realTimeRequestSocketLock) {
if (PollerRunning && _realTimeRequestSocket != null)
{
_realTimeRequestSocket.SendMoreFrame(string.Empty);
_realTimeRequestSocket.SendFrame("PING");
}
}
}
private void TimeoutElapsed(object sender, NetMQTimerEventArgs e)
{
// no need to reconnect, the client would be recreated because of RX
// because of RX internal stuff invoking on the poller thread block the entire application, so calling on Thread Pool
Task.Run(() =>
{
parent.requiresInitialisation = 1;
subject.OnNext(new ConnectionInfo(ConnectionStatus.Closed, this.address));
});
}
/// <summary>
/// Sends heartbeat messages so we know that the server is still up.
/// </summary>
private void HeartBeatTimerElapsed(object sender, NetMQTimerEventArgs e)
{
lock (realTimeRequestSocketLock)
{
if (PollerRunning && realTimeRequestSocket != null)
{
realTimeRequestSocket.SendMoreFrame(string.Empty);
realTimeRequestSocket.SendFrame(BitConverter.GetBytes((byte)DataRequestMessageType.Ping));
}
}
}
private void OnTimeoutTimer(object sender, NetMQTimerEventArgs e)
{
// dispose the current subscriber socket and try to connect
_actor.ReceiveReady -= OnActorMessage;
_poller.Remove(_actor);
_poller.Remove(_subscriber);
_subscriber.Options.Linger = TimeSpan.Zero;
_subscriber.Dispose();
_subscriber = null;
Connect();
}
private void ClearDeadNodes(object sender, NetMQTimerEventArgs e)
{
// create an array with the dead nodes
var deadNodes = m_nodes.
Where(n => DateTime.Now > n.Value + m_deadNodeTimeout)
.Select(n => n.Key).ToArray();
// remove all the dead nodes from the nodes list and disconnect from the publisher
foreach (var node in deadNodes)
{
m_nodes.Remove(node);
m_publisher.Disconnect(node.Address);
m_shim.SendMoreFrame(RemovedNodeCommand).SendFrame(node.Address);
}
}
private void SessionCleanupTimer_Elapsed(object sender, NetMQTimerEventArgs e)
{
//remove timed out sessions
var now = DateTime.UtcNow;
var timedOutKeys = (from n in _sessions
where (now - n.Value.LastMessageReceived).TotalMinutes > _sessionTimeoutMins
select n.Key).ToArray();
if (timedOutKeys.Length > 0)
{
foreach (var key in timedOutKeys)
{
HostSession session;
_sessions.TryRemove(key, out session);
}
_logger.Debug("Protocol sessions cleaned up: Count {0}.", timedOutKeys.Length);
}
}
private void Cleanup(object sender, NetMQTimerEventArgs e)
{
_logger?.LogDebug($"{_id}: Running peer cleanup on thread {Thread.CurrentThread.Name}");
var unresponsivePeers = _info.Where(n => DateTimeOffset.Now > n.Value + _peerLifetime)
.Select(n => n.Key);
foreach (var peer in unresponsivePeers)
{
_info.Remove(peer);
_publisher.Disconnect(peer.Address);
_shim.SendMoreFrame("R").SendFrame(peer.Address);
_logger?.LogInformation($"{_id}: Removed unresponsive peer at '{{PeerAddress}}'", peer.Address);
_hub?.Clients.All.SendAsync("ReceiveMessage", "warning",
$"Removing unresponsive peer at '{peer.Address}'");
}
}
private void OnReapTimerElapsed(object sender, NetMQTimerEventArgs e)
{
// Ping all peers and reap any expired ones
// Don't remove them during the foreach loop
var peersToRemove = new List <ZyrePeer>();
foreach (var peer in _peers.Values)
{
var isToBeRemoved = PingPeer(peer);
if (isToBeRemoved)
{
peersToRemove.Add(peer);
}
}
foreach (var peer in peersToRemove)
{
RemovePeer(peer);
}
}
private void heartbeatTimer_Elapsed(object sender, NetMQTimerEventArgs e)
{
// Send heartbeats to idle workers if it's time
if (DateTime.Now >= _nextHeartbeatAt)
{
var expired = _workerQueue.Where(w => w.Expiry < DateTime.Now).ToList();
expired.ForEach(w => HandleExpiredWorker(w));
foreach (var worker in _workerQueue)
{
var heartbeatMessage = new NetMQMessage();
heartbeatMessage.Append(new NetMQFrame(worker.Address));
heartbeatMessage.Append(new NetMQFrame(Encoding.Unicode.GetBytes(Paranoid.PPP_HEARTBEAT)));
_backend.SendMessage(heartbeatMessage);
}
_nextHeartbeatAt = DateTime.Now.AddMilliseconds(Paranoid.HEARTBEAT_INTERVAL_MS);
}
}
private void ClearDeadNodes(object sender, NetMQTimerEventArgs e)
{
// create an array with the dead nodes
var deadNodes = _nodes.
Where(n => DateTime.Now > n.Value + m_deadNodeTimeout)
.Select(n => n.Key).ToArray();
if (!deadNodes.Any())
{
return;
}
// remove all the dead nodes from the nodes list and disconnect from the publisher
foreach (var node in deadNodes)
{
_nodes.Remove(node);
OnNodeDisConnected(node);
}
}
/// <summary>
///
/// </summary>
/// <param name="sender"></param>
/// <param name="e"></param>
private void SendQueue(object sender, NetMQTimerEventArgs e)
{
var i = m_sendQueue.Count;
while (i > 0)
{
NetMQMessage message;
if (m_sendQueue.TryDequeue(out message))
{
m_client.TrySendMultipartMessage(message);
}
i--;
}
if (m_connected && (m_receiveTimer.ElapsedMilliseconds - m_lastReceived > DISCONNECTION_TIMEOUT))
{
SetDisconnected();
SendGreet();
}
}
private void heartbeatTimer_Elapsed(object sender, NetMQTimerEventArgs e)
{
// If liveness hits zero, queue is considered disconnected
if (--_liveness <= 0)
{
Console.WriteLine("{0} - W: heartbeat failure, can't reach queue.", DateTime.Now.ToLongTimeString());
Console.WriteLine("{0} - W: reconnecting in {1} msecs...", DateTime.Now.ToLongTimeString(), _interval);
Thread.Sleep(_interval);
// Exponential back off
if (_interval < INTERVAL_MAX)
{
_interval *= 2;
}
_liveness = Paranoid.HEARTBEAT_LIVENESS;
// Break the while loop and start the connection over
Task.Factory.StartNew(_ =>
{
Console.WriteLine("{0} - I: restart", DateTime.Now.ToLongTimeString());
Stop();
Dispose();
Start();
}, _scheduler);
}
// Send heartbeat to queue if it's time
if (DateTime.Now > _nextHeartbeatAt)
{
_nextHeartbeatAt = DateTime.Now.AddMilliseconds(Paranoid.HEARTBEAT_INTERVAL_MS);
var heartbeatMessage = new NetMQMessage();
heartbeatMessage.Append(new NetMQFrame(Encoding.Unicode.GetBytes(Paranoid.PPP_HEARTBEAT)));
_worker.SendMessage(heartbeatMessage);
}
}
private void ClearDeadNodes(object sender, NetMQTimerEventArgs e)
{
//Console.WriteLine("");
//Console.WriteLine($"ClearDeadNodes");
var now = DateTime.Now;
//DumpNodes(now, "BeforeClearDateNodes");
// create an array with the dead nodes
var deadNodes = nodes.Where(n => now > n.Value + m_deadNodeTimeout)
.Select(n => n.Key)
.ToArray();
// remove all the dead nodes from the nodes list and disconnect from the publisher
foreach (var node in deadNodes)
{
nodes.Remove(node);
publisher.Disconnect(node.Address);
shim.SendMoreFrame(RemovedNodeCommand).SendFrame(node.Address);
}
//DumpNodes(now, "AfterClearDateNodes");
}
private void HouseKeeper_Elapsed(object sender, NetMQTimerEventArgs e)
{
_logger.LogDebug("Performing house keeping");
lock (_server)
{
foreach (var peer in _peerLastContact.Select(x => x.Key).ToList())
{
_server
.SendMoreFrame(peer.ToByteArray())
.SendMoreFrame(_nodeId.ToByteArray())
.SendFrame(ConvertOp(MessageOp.Ping));
}
}
lock (_lockRegistry)
{
var expiredList = _lockRegistry.Where(x => x.Expiry < DateTime.Now).ToList();
foreach (var expired in expiredList)
{
_lockRegistry.Remove(expired);
}
}
}
private void OnReapTimerElapsed(object sender, NetMQTimerEventArgs e)
{
// Ping all peers and reap any expired ones
PingAllPeersAndRemoveExpiredOnes();
}
private void OnTimeoutElapsed(NetMQTimerEventArgs args)
{
_publisherSocket.SendFrame(Heartbeat.CoreHearbeatTopic);
}
private void OnHeartbeatTimerElapsed(object sender, NetMQTimerEventArgs e)
{
// Heartbeat timer elapsed, let's send another heartbeat
_publisherSocket.SendFrame(HeartbeatMessage);
}
private void OnReceiveTimeout(object sender, NetMQTimerEventArgs e)
{
_actor.SendFrame(ReceiveTimeoutCommand);
}
private void OnReapTimerElapsed(object sender, NetMQTimerEventArgs e)
{
// Ping all peers and reap any expired ones
// Don't remove them during the foreach loop
var peersToRemove = new List<ZyrePeer>();
foreach (var peer in _peers.Values)
{
var isToBeRemoved = PingPeer(peer);
if (isToBeRemoved)
{
peersToRemove.Add(peer);
}
}
foreach (var peer in peersToRemove)
{
RemovePeer(peer);
}
}
/*private void DropPublisherSubscriptions(object sender, NetMQSocketEventArgs e)
* {
* // just drop the subscription messages, we have to do that to Welcome message to work
* _publisherSocket.SkipMultipartMessage();
* }*/
private void OnHeartbeatTimerElapsed(object sender, NetMQTimerEventArgs e)
{
_actor.SendFrame(HeartbeatMessage);
}
/// <summary>
/// Timeout timer for no data received notification
/// </summary>
/// <param name="sender"></param>
/// <param name="e"></param>
private void OnTimeoutTimer(object sender, NetMQTimerEventArgs e) =>
_log.Error($"No data received due to HB timeout of {(int)_timeoutTimerSpan.TotalSeconds} seconds occurred, this means we are no longer connected to the data feed. Waiting for a reconnect...");
private void OnReconnectTimer(object sender, NetMQTimerEventArgs e)
{
// try to connect again
Connect();
}
private void OnHeartbeatTimerElapsed(object sender, NetMQTimerEventArgs e)
{
publisherSocket.Send(StreamingProtocol.HeartbeatTopic);
}
private void OnReceiveTimeout(object sender, NetMQTimerEventArgs e)
{
_receiveTimeoutTimer.Enable = false;
_onReceiveTimeout?.Invoke();
}
