private void OnShimReady(NetMQSocketEventArgs args)
{
var msg = args.Socket.ReceiveMultipartMessage();
var cmd = msg.First.ConvertToString();
if (cmd == NetMQActor.EndShimMessage)
{
_poller.Stop();
return;
}
if (msg.FrameCount <= 1)
{
return;
}
if (cmd == "broadcast" && msg.FrameCount == 3)
{
var nmqMsg = new NetMQMessage();
nmqMsg.Append(msg[1].ConvertToString());
nmqMsg.AppendEmptyFrame();
nmqMsg.Append(msg[2].ConvertToString());
_publisherSocket.SendMultipartMessage(nmqMsg);
}
else if (cmd == "response")
{
var nmqMsg = new NetMQMessage();
for (var i = 1; i < msg.FrameCount; i++)
{
nmqMsg.Append(msg[i]);
}
_responseSocket.SendMultipartMessage(nmqMsg);
}
}
private void OnShimReady(object sender, NetMQSocketEventArgs e)
{
var command = _shim.ReceiveFrameString();
switch (command)
{
case NetMQActor.EndShimMessage:
{
_poll.Stop();
break;
}
case PublishCommand:
{
_publisher.SendMultipartMessage(_shim.ReceiveMultipartMessage());
break;
}
case GetHostAddressCommand:
{
_shim.SendFrame($"{_beacon.BoundTo}:{_port}");
break;
}
}
}
private void InitializeQueue()
{
queueSub = queue.GetConsumingEnumerable()
.ToObservable(TaskPoolScheduler.Default)
.Do(_ => CheckQueueBacklog())
.Subscribe(share =>
{
share.Source = clusterConfig.ClusterName;
try
{
var flags = (int)WireFormat.ProtocolBuffers;
var msg = new NetMQMessage(2);
using (var stream = new MemoryStream())
{
Serializer.Serialize(stream, share);
msg.Push(stream.ToArray());
}
msg.Push(flags);
msg.Push(share.PoolId);
pubSocket.SendMultipartMessage(msg);
}
catch (Exception ex)
{
logger.Error(ex);
}
});
}
private void OnShimReady(object sender, NetMQSocketEventArgs e)
{
// new actor command
string command = m_shim.ReceiveFrameString();
// check if we received end shim command
if (command == NetMQActor.EndShimMessage)
{
// we cancel the socket which dispose and exist the shim
m_poller.Stop();
}
else if (command == PublishCommand)
{
// it is a publish command
// we just forward everything to the publisher until end of message
NetMQMessage message = m_shim.ReceiveMultipartMessage();
m_publisher.SendMultipartMessage(message);
}
else if (command == GetHostAddressCommand)
{
var interfaceCollection = new InterfaceCollection();
var bindTo = interfaceCollection.Select(x => x.Address).First();
var address = bindTo + ":" + m_randomPort;
m_shim.SendFrame(address);
}
}
public ZeroMqDeviceService()
{
//We are using NetMQ to retrieve the available devices in the following manner:
// queryRouterSocket -> querySocket -> All Devices In -> All Device out -> commandSocket -> queryRouterSocket
// Router -> Publisher -> Subscriber -> Dealer -> Router -> Router
//This pattern allows any client to request the list of Clients in the commandSocket. So we can send messages directly to clients at another point in time.
queryRouterSocket.ReceiveReady += (o, e) =>
{
//Send a message to all clients with the following fields:
// 1 - Subscription Name
// 2 - queryRouterSocket id
// 3 - command type
var incomingMsg = e.Socket.ReceiveMultipartBytes();
var messageToServer = new NetMQMessage();
messageToServer.Append("All");
incomingMsg.ForEach((val) => { messageToServer.Append(val); });
querySocket.SendMultipartMessage(messageToServer);
};
commandSocket.ReceiveReady += (o, e) =>
{
//Recieve a message from the clients with the following fields:
// 1 - Client Id
// 2 - queryRouterSocket id
// 3 - response
var msg = commandSocket.ReceiveMultipartBytes();
msg.RemoveAt(0); //Remove the Client id
var messageToClient = new NetMQMessage();
msg.ForEach((val) => { messageToClient.Append(val); });
queryRouterSocket.SendMultipartMessage(messageToClient); //forward the message
};
poller = new NetMQPoller {
queryRouterSocket, commandSocket
};
poller.RunAsync();
}
private void InitializeQueue()
{
queueSub = queue.GetConsumingEnumerable()
.ToObservable(TaskPoolScheduler.Default)
.Do(_ => CheckQueueBacklog())
.Subscribe(share =>
{
share.Source = clusterConfig.ClusterName;
try
{
var json = JsonConvert.SerializeObject(share, serializerSettings);
var msg = new NetMQMessage(2);
msg.Push(json);
msg.Push(share.PoolId);
pubSocket.SendMultipartMessage(msg);
}
catch (Exception ex)
{
logger.Error(ex);
}
});
}
public async Task <XtResult <TMessage> > PublishAsync <TMessage>(TMessage message)
where TMessage : class, new()
{
PublisherSocket publisherSocket = null;
try
{
publisherSocket = new PublisherSocket();
publisherSocket.Connect(_configuration.Address());
return(await Task.Run(() =>
{
try
{
var msg = new PubSubMessage <TMessage>(_configuration, message);
publisherSocket.SendMultipartMessage(msg);
}
catch (System.Exception ex)
{
return XtResult <TMessage> .Failed(ex, "publish");
}
return XtResult <TMessage> .Success(message, "publish");
}));
}
catch (System.Exception ex)
{
return(XtResult <TMessage> .Failed(ex, "publish"));
}
finally
{
publisherSocket?.Dispose();
}
}
public static void SendMessage(string connection, NetMQMessage message)
{
using (var pubSocket = new PublisherSocket(connection))
{
pubSocket.Options.SendHighWatermark = 1000;
pubSocket.SendMultipartMessage(message);
}
}
public void Send(byte[] values, double sampleRate)
{
var msg = new NetMQMessage();
msg.Append((long)sampleRate);
msg.Append(values);
Publisher.SendMultipartMessage(msg);
}
/// <summary>
/// Transmits the events to all connected <see cref="ISubscriber"/>s
/// </summary>
/// <param name="topicEvent">Topic event to be sent to all remote <see cref="Subscribers.ISubscriber"/>s</param>
public void Publish(object package)
{
if (disposedValue)
{
throw new InvalidOperationException("NetMQPublisher has been disposed");
}
socket.SendMultipartMessage(messageFactory.CreateTopicMessage(package));
}
public void Send(string topic, Dictionary <string, object> data, byte[] thirdFrame = null)
{
NetMQMessage m = new NetMQMessage();
m.Append(topic);
m.Append(MessagePackSerializer.Serialize <Dictionary <string, object> >(data));
if (thirdFrame != null)
{
m.Append(thirdFrame);
}
publisherSocket.SendMultipartMessage(m);
}
private void OnShimMessage(object sender, NetMQSocketEventArgs e)
{
string command = e.Socket.ReceiveFrameString();
if (command == PublishMessageCommand)
{
var msg = e.Socket.ReceiveMultipartMessage();
Console.WriteLine($"Sending: {msg.Last.ConvertToString()}");
_publisherSocket.SendMultipartMessage(msg);
}
else if (command == NetMQActor.EndShimMessage)
{
_poller.Stop();
}
}
static void Main(string[] args)
{
Console.WriteLine("Type content router");
using (var server = new ResponseSocket())
using (var publish = new PublisherSocket())
{
server.Bind("tcp://*:5555");
publish.Bind("tcp://*:5556");
while (true)
{
var message = server.ReceiveMultipartMessage(2);
Console.WriteLine("responseSocket : Server Received '{0}'-{1}", message[0].ConvertToString(Encoding.UTF8), message[1].ConvertToString(Encoding.UTF8));
server.SendFrameEmpty();
publish.SendMultipartMessage(message);
}
}
}
public void Loop()
{
//Publish message
NetMQMessage msg = new NetMQMessage(1);
msg.Append("32abd/node0974/out0");
pub.SendMultipartMessage(msg);
//At the end of the first run, create another sub to test late joiners
if (!connected)
{
sub2 = new SubscriberSocket();
sub2.Connect("inproc://publisher");
poller.Add(sub2);
sub2.ReceiveReady += (s, a) =>
{
Console.WriteLine("Second: " + a.Socket.ReceiveMultipartMessage()[0].ConvertToString());
};
sub2.Subscribe("32abd/node0974/out0");
connected = true;
}
}
static void Main(string[] args)
{
using (var pubSocket = new PublisherSocket(">tcp://127.0.0.1:5678"))
{
Console.WriteLine("Publisher socket connecting...");
pubSocket.Options.SendHighWatermark = 1000;
Thread.Sleep(1000);
var msg = "This is a test message";
Console.WriteLine("Sending message : {0}", msg);
NetMQMessage message = new NetMQMessage(2);
message.Append(new NetMQFrame("Test"));
message.Append(new NetMQFrame(msg));
pubSocket.SendMultipartMessage(message);
Console.WriteLine("Done");
}
}
void SendFrame(double timestamp)
{
Dictionary <string, object> payload = new Dictionary <string, object> {
{ "topic", topic },
{ "width", width },
{ "height", height },
{ "index", index },
{ "timestamp", timestamp },
{ "format", "rgb" },
{ "projection_matrix", intrinsics }
};
NetMQMessage m = new NetMQMessage();
m.Append(topic);
m.Append(MessagePackSerializer.Serialize <Dictionary <string, object> >(payload));
m.Append(StreamTexture.GetRawTextureData());
pubSocket.SendMultipartMessage(m);
index++;
}
private void OnShimReady(object sender, NetMQSocketEventArgs e)
{
// new actor command
var command = _shim.ReceiveFrameString();
// check if we received end shim command
if (command == NetMQActor.EndShimMessage)
{
// we cancel the socket which dispose and exist the shim
_poller.Stop();
}
else if (command == TaskSchedulerBusCommands.Publish.ToString())
{
// it is a publish command
// we just forward everything to the publisher until end of message
var message = _shim.ReceiveMultipartMessage();
_publisher.SendMultipartMessage(message);
}
else if (command == TaskSchedulerBusCommands.GetHostAddress.ToString())
{
var address = _beacon.HostName + ":" + _randomPort;
_shim.SendFrame(address);
}
}
private void OnShimReady(object sender, NetMQSocketEventArgs e)
{
// new actor command
var command = shim.ReceiveFrameString();
// check if we received end shim command
if (command == NetMQActor.EndShimMessage)
{
// we cancel the socket which dispose and exist the shim
poller.Stop();
}
else if (command == PublishCommand)
{
// it is a publish command
// we just forward everything to the publisher until end of message
var message = shim.ReceiveMultipartMessage();
publisher.SendMultipartMessage(message);
}
else if (command == GetHostAddressCommand)
{
var address = beacon.BoundTo + ":" + randomPort;
shim.SendFrame(address);
}
}
public void Publish(NetMQMessage msg)
{
_publisherSocket.SendMultipartMessage(msg);
}
// added by Huang
private void SendZeroMQMessage(bool success, float fx, float fy, float cx, float cy, double openface_timestamp)
{
Tuple <double, double> gaze_angle = new Tuple <double, double>(0, 0);
List <double> pose = new List <double>();
List <double> non_rigid_params = landmark_detector.GetNonRigidParams();
NetMQMessage output_message = new NetMQMessage();
output_message.Append(topic);
JsonData json_data = new JsonData();
json_data.frame = frame_no++;
DateTime origin = new DateTime(1970, 1, 1, 0, 0, 0, 0, DateTimeKind.Utc);
TimeSpan difference = DateTime.UtcNow - origin;
output_message.Append((long)difference.TotalMilliseconds);
json_data.timestamp = openface_timestamp;
double confidence = landmark_detector.GetConfidence();
if (confidence < 0)
{
confidence = 0;
}
else if (confidence > 1)
{
confidence = 1;
}
json_data.confidence = confidence;
pose = new List <double>();
landmark_detector.GetPose(pose, fx, fy, cx, cy);
json_data.pose.pose_Tx = pose[0];
json_data.pose.pose_Ty = pose[1];
json_data.pose.pose_Tz = pose[2];
json_data.pose.pose_Rx = pose[3];
json_data.pose.pose_Ry = pose[4];
json_data.pose.pose_Rz = pose[5];
gaze_angle = gaze_analyser.GetGazeAngle();
var gaze = gaze_analyser.GetGazeCamera();
json_data.gaze.gaze_angle_x = gaze_angle.Item1;
json_data.gaze.gaze_angle_y = gaze_angle.Item2;
json_data.gaze.gaze_0_x = gaze.Item1.Item1;
json_data.gaze.gaze_0_y = gaze.Item1.Item2;
json_data.gaze.gaze_0_z = gaze.Item1.Item3;
json_data.gaze.gaze_1_x = gaze.Item2.Item1;
json_data.gaze.gaze_1_y = gaze.Item2.Item2;
json_data.gaze.gaze_1_z = gaze.Item2.Item3;
json_data.au_c = face_analyser.GetCurrentAUsClass();
Dictionary <string, double> au_regs = face_analyser.GetCurrentAUsReg();
json_data.au_r = new Dictionary <string, double>();
foreach (KeyValuePair <string, double> au_reg in au_regs)
{
json_data.au_r[au_reg.Key] = au_reg.Value / 5.0;
if (json_data.au_r[au_reg.Key] < 0)
{
json_data.au_r[au_reg.Key] = 0;
}
if (json_data.au_r[au_reg.Key] > 1)
{
json_data.au_r[au_reg.Key] = 1;
}
}
string json_string = JsonConvert.SerializeObject(json_data);
output_message.Append(json_string);
pubSocket.SendMultipartMessage(output_message);
}
/// <summary>
/// the broker setting up the cluster
///
///
/// State 2 ---+ +--- State n
/// | |
/// +----+----+
/// client 1 ---| | |--- worker 1
/// client 2 ---+---- BROKER 1 ----+--- worker 2
/// : | | | :
/// client n ---+ +----+----+ +--- worker n
/// | |
/// BROKER 2 BROKER n
///
/// BROKER 2 and n are not included and must be setup separately
///
/// A minimum of two address must be supplied
/// </summary>
/// <param name="args">[0] = this broker's address
/// [1] = 1st peer's address
/// :
/// [n] = nth peer address</param>
/// <remarks>
/// since "inproc://" is not working in NetMQ we use "tcp://"
/// for each broker we need 5 ports which for this example are
/// assigned as follows (in true life it should be configurable whether
/// they are ports or tcp/ip addresses)
///
/// this brokers address => local frontend binds to tcp://127.0.0.1:5555
/// cloud frontend binds to :5556
/// local backend binds to :5557
/// state backend binds to :5558
/// monitor PULL binds to :5559
///
/// the sockets are connected as follows
///
/// this broker's monitor PUSH connects to tcp://127.0.0.1:5559
///
/// (if peer's address and port is tcp://127.0.0.1:5575)
///
/// this broker's cloud backend connects to :5576
/// this broker's state frontend connects to :5578
///
/// this scheme is fix in this example
/// </remarks>
public static void Main(string[] args)
{
Console.Title = "NetMQ Inter-Broker Router";
const string baseAddress = "tcp://127.0.0.1:";
if (args.Length < 2)
{
Console.WriteLine("usage: program me peer1 [peer]*");
Console.WriteLine("each broker needs 5 port for his sockets!");
Console.WriteLine("place enough distance between multiple broker addresses!");
Environment.Exit(-1);
}
// trapping Ctrl+C as exit signal!
Console.CancelKeyPress += (s, e) =>
{
e.Cancel = true;
s_keepRunning = false;
};
// get random generator for later use
var rnd = new Random();
// get list for registering the clients
var clients = new List<byte[]>(NbrClients);
// get a list of peer addresses
var peers = new List<byte[]>();
// get all peer addresses - first is this broker!
for (var i = 1; i < args.Length; i++)
peers.Add(Encoding.UTF8.GetBytes(args[i]));
// build this broker's address
var me = baseAddress + args[0];
// get the port as integer for later use
var myPort = int.Parse(args[0]);
Console.WriteLine("[BROKER] The broker can be stopped by using CTRL+C!");
Console.WriteLine("[BROKER] setting up sockets ...");
// set up all the addresses needed in the due course
var localFrontendAddress = me;
var cloudFrontendAddress = baseAddress + (myPort + 1);
var localBackendAddress = baseAddress + (myPort + 2);
var stateBackendAddress = baseAddress + (myPort + 3);
var monitorAddress = baseAddress + (myPort + 4);
// create the context and all the sockets
using (var localFrontend = new RouterSocket())
using (var localBackend = new RouterSocket())
using (var cloudFrontend = new RouterSocket())
using (var cloudBackend = new RouterSocket())
using (var stateBackend = new PublisherSocket())
using (var stateFrontend = new SubscriberSocket())
using (var monitor = new PullSocket())
{
// give every socket an unique identity, e.g. LocalFrontend[Port]
SetIdentities(myPort,
localFrontend,
cloudFrontend,
localBackend,
stateBackend,
monitor,
cloudBackend, stateFrontend);
// subscribe to any message on the stateFrontend socket!
stateFrontend.Subscribe("");
// bind the serving sockets
localFrontend.Bind(localFrontendAddress);
cloudFrontend.Bind(cloudFrontendAddress);
localBackend.Bind(localBackendAddress);
stateBackend.Bind(stateBackendAddress);
monitor.Bind(monitorAddress);
// connect sockets to peers
for (var i = 1; i < args.Length; i++)
{
// build the cloud back end address
var peerPort = int.Parse(args[i]);
var address = baseAddress + (peerPort + 1);
Console.WriteLine("[BROKER] connect to cloud peer {0}", address);
// this cloudBackend connects to all peer cloudFrontends
cloudBackend.Connect(address);
// build the state front end address
address = baseAddress + (peerPort + 3);
Console.WriteLine("[BROKER] subscribe to state peer {0}", address);
// this stateFrontend to all peer stateBackends
stateFrontend.Connect(address);
}
// setup the local worker queue for LRU and monitor cloud capacity
var workerQueue = new Queue<byte[]>();
int previousLocalCapacity = 0;
// receive the capacity available from other peer(s)
stateFrontend.ReceiveReady += (s, e) =>
{
// the message should contain the available cloud capacity
var capacity = e.Socket.ReceiveFrameString();
Debug.Assert(string.IsNullOrWhiteSpace(capacity), "StateFrontend: message was empty!");
int couldCapacity;
Debug.Assert(int.TryParse(capacity, out couldCapacity), "StateFrontend: message did not contain a number!");
};
// get the status message and print it
monitor.ReceiveReady += (s, e) =>
{
var msg = e.Socket.ReceiveFrameString();
Console.WriteLine("[MONITOR] {0}", msg);
};
// all local clients are connecting to this socket
// they send a REQ and get a REPLY
localFrontend.ReceiveReady += (s, e) =>
{
// [client adr][empty][message id]
var request = e.Socket.ReceiveMultipartMessage();
// register the local client for later identification if not known
if (!clients.Any(n => AreSame(n, request[0])))
clients.Add(request[0].Buffer);
// if we have local capacity send worker else send to cloud
if (workerQueue.Count > 0)
{
// get the LRU worker adr
var worker = workerQueue.Dequeue();
// wrap message with workers address
var msg = Wrap(worker, request);
// send message to the worker
// [worker adr][empty][client adr][empty][data]
localBackend.SendMultipartMessage(msg);
}
else
{
// get an random index for peers
var peerIdx = rnd.Next(peers.Count - 2) + 2;
// get peers address
var peer = peers[peerIdx];
// wrap message with peer's address
var msg = Wrap(peer, request);
// [peer adr][empty][client adr][empty][data]
cloudBackend.SendMultipartMessage(msg);
}
};
// the workers are connected to this socket
// we get a REPLY either for a cloud client [worker adr][empty][peer adr][empty][peer client adr][empty][data]
// or local client [worker adr][empty][client adr][empty][data]
// or a READY message [worker adr][empty][WORKER_READY]
localBackend.ReceiveReady += (s, e) =>
{
// a worker can send "READY" or a request
// or an REPLAY
var msg = e.Socket.ReceiveMultipartMessage();
// just to make sure we received a proper message
Debug.Assert(msg != null && msg.FrameCount > 0, "[LocalBackend] message was empty or frame count == 0!");
// get the workers identity
var id = Unwrap(msg);
// this worker done in either way so add it to available workers
workerQueue.Enqueue(id);
// if it is NOT a ready message we need to route the message
// it could be a reply to a peer or a local client
// [WORKER_READY] or [client adr][empty][data] or [peer adr][empty][peer client adr][empty][data]
if (msg[0].Buffer[0] != WorkerReady)
{
Debug.Assert(msg.FrameCount > 2, "[LocalBackend] None READY message malformed");
// if the adr (first frame) is any of the clients send the REPLY there
// and send it to the peer otherwise
if (clients.Any(n => AreSame(n, msg.First)))
localFrontend.SendMultipartMessage(msg);
else
cloudFrontend.SendMultipartMessage(msg);
}
};
// this socket is connected to all peers
// we receive either a REQ or a REPLY form a peer
// REQ [peer adr][empty][peer client adr][empty][message id] -> send to peer for processing
// REP [peer adr][empty][client adr][empty][message id] -> send to local client
cloudBackend.ReceiveReady += (s, e) =>
{
var msg = e.Socket.ReceiveMultipartMessage();
// just to make sure we received a message
Debug.Assert(msg != null && msg.FrameCount > 0, "[CloudBackend] message was empty or frame count == 0!");
// we need the peers address for proper addressing
var peerAdr = Unwrap(msg);
// the remaining message must be at least 3 frames!
Debug.Assert(msg.FrameCount > 2, "[CloudBackend] message malformed");
// if the id is any of the local clients it is a REPLY
// and a REQ otherwise
if (clients.Any(n => AreSame(n, msg.First)))
{
// [client adr][empty][message id]
localFrontend.SendMultipartMessage(msg);
}
else
{
// add the peers address to the request
var request = Wrap(peerAdr, msg);
// [peer adr][empty][peer client adr][empty][message id]
cloudFrontend.SendMultipartMessage(request);
}
};
// all peers are binding to this socket
// we receive REPLY or REQ from peers
// REQ [peer adr][empty][peer client adr][empty][data] -> send to local worker for processing
// REP [peer adr][empty][client adr][empty][data] -> send to local client
cloudFrontend.ReceiveReady += (s, e) =>
{
var msg = e.Socket.ReceiveMultipartMessage();
// just to make sure we received a message
Debug.Assert(msg != null && msg.FrameCount > 0, "[CloudFrontend] message was empty or frame count == 0!");
// we may need the peers address for proper addressing
var peerAdr = Unwrap(msg);
// the remaining message must be at least 3 frames!
Debug.Assert(msg.FrameCount > 2, "[CloudFrontend] message malformed");
// if the address is any of the local clients it is a REPLY
// and a REQ otherwise
if (clients.Any(n => AreSame(n, msg.First)))
localFrontend.SendMultipartMessage(msg);
else
{
// in order to know which per to send back the peers adr must be added again
var original = Wrap(peerAdr, msg);
// reduce the capacity to reflect the use of a worker by a cloud request
previousLocalCapacity = workerQueue.Count;
// get the LRU worker
var workerAdr = workerQueue.Dequeue();
// wrap the message with the worker address and send
var request = Wrap(workerAdr, original);
localBackend.SendMultipartMessage(request);
}
};
// in order to reduce chatter we only check to see if we have local capacity to provide to cloud
// periodically every 2 seconds with a timer
var timer = new NetMQTimer((int)TimeSpan.FromSeconds(2).TotalMilliseconds);
timer.Elapsed += (t, e) =>
{
// send message only if the previous send information changed
if (previousLocalCapacity != workerQueue.Count)
{
// set the information
previousLocalCapacity = workerQueue.Count;
// generate the message
var msg = new NetMQMessage();
var data = new NetMQFrame(previousLocalCapacity.ToString());
msg.Append(data);
var stateMessage = Wrap(Encoding.UTF8.GetBytes(me), msg);
// publish info
stateBackend.SendMultipartMessage(stateMessage);
}
// restart the timer
e.Timer.Enable = true;
};
// start all clients and workers as threads
var clientTasks = new Thread[NbrClients];
var workerTasks = new Thread[NbrWorker];
for (var i = 0; i < NbrClients; i++)
{
var client = new Client(localFrontendAddress, monitorAddress, (byte)i);
clientTasks[i] = new Thread(client.Run) { Name = string.Format("Client_{0}", i) };
clientTasks[i].Start();
}
for (var i = 0; i < NbrWorker; i++)
{
var worker = new Worker(localBackendAddress, (byte)i);
workerTasks[i] = new Thread(worker.Run) { Name = string.Format("Worker_{0}", i) };
workerTasks[i].Start();
}
// create poller and add sockets & timer
var poller = new NetMQPoller
{
localFrontend,
localBackend,
cloudFrontend,
cloudBackend,
stateFrontend,
stateBackend,
monitor,
timer
};
// start monitoring the sockets
poller.RunAsync();
// we wait for a CTRL+C to exit
while (s_keepRunning)
Thread.Sleep(100);
Console.WriteLine("Ctrl-C encountered! Exiting the program!");
if (poller.IsRunning)
poller.Stop();
poller.Dispose();
}
}
/// <summary>
/// the broker setting up the cluster
///
///
/// State 2 ---+ +--- State n
/// | |
/// +----+----+
/// client 1 ---| | |--- worker 1
/// client 2 ---+---- BROKER 1 ----+--- worker 2
/// : | | | :
/// client n ---+ +----+----+ +--- worker n
/// | |
/// BROKER 2 BROKER n
///
/// BROKER 2 and n are not included and must be setup separately
///
/// A minimum of two address must be supplied
/// </summary>
/// <param name="args">[0] = this broker's address
/// [1] = 1st peer's address
/// :
/// [n] = nth peer address</param>
/// <remarks>
/// since "inproc://" is not working in NetMQ we use "tcp://"
/// for each broker we need 5 ports which for this example are
/// assigned as follows (in true life it should be configurable whether
/// they are ports or tcp/ip addresses)
///
/// this brokers address => local frontend binds to tcp://127.0.0.1:5555
/// cloud frontend binds to :5556
/// local backend binds to :5557
/// state backend binds to :5558
/// monitor PULL binds to :5559
///
/// the sockets are connected as follows
///
/// this broker's monitor PUSH connects to tcp://127.0.0.1:5559
///
/// (if peer's address and port is tcp://127.0.0.1:5575)
///
/// this broker's cloud backend connects to :5576
/// this broker's state frontend connects to :5578
///
/// this scheme is fix in this example
/// </remarks>
public static void Main(string[] args)
{
Console.Title = "NetMQ Inter-Broker Router";
const string baseAddress = "tcp://127.0.0.1:";
if (args.Length < 2)
{
Console.WriteLine("usage: program me peer1 [peer]*");
Console.WriteLine("each broker needs 5 port for his sockets!");
Console.WriteLine("place enough distance between multiple broker addresses!");
Environment.Exit(-1);
}
// trapping Ctrl+C as exit signal!
Console.CancelKeyPress += (s, e) =>
{
e.Cancel = true;
s_keepRunning = false;
};
// get random generator for later use
var rnd = new Random();
// get list for registering the clients
var clients = new List <byte[]>(NbrClients);
// get a list of peer addresses
var peers = new List <byte[]>();
// get all peer addresses - first is this broker!
for (var i = 1; i < args.Length; i++)
{
peers.Add(Encoding.UTF8.GetBytes(args[i]));
}
// build this broker's address
var me = baseAddress + args[0];
// get the port as integer for later use
var myPort = int.Parse(args[0]);
Console.WriteLine("[BROKER] The broker can be stopped by using CTRL+C!");
Console.WriteLine("[BROKER] setting up sockets ...");
// set up all the addresses needed in the due course
var localFrontendAddress = me;
var cloudFrontendAddress = baseAddress + (myPort + 1);
var localBackendAddress = baseAddress + (myPort + 2);
var stateBackendAddress = baseAddress + (myPort + 3);
var monitorAddress = baseAddress + (myPort + 4);
// create the context and all the sockets
using (var localFrontend = new RouterSocket())
using (var localBackend = new RouterSocket())
using (var cloudFrontend = new RouterSocket())
using (var cloudBackend = new RouterSocket())
using (var stateBackend = new PublisherSocket())
using (var stateFrontend = new SubscriberSocket())
using (var monitor = new PullSocket())
{
// give every socket an unique identity, e.g. LocalFrontend[Port]
SetIdentities(myPort,
localFrontend,
cloudFrontend,
localBackend,
stateBackend,
monitor,
cloudBackend, stateFrontend);
// subscribe to any message on the stateFrontend socket!
stateFrontend.Subscribe("");
// bind the serving sockets
localFrontend.Bind(localFrontendAddress);
cloudFrontend.Bind(cloudFrontendAddress);
localBackend.Bind(localBackendAddress);
stateBackend.Bind(stateBackendAddress);
monitor.Bind(monitorAddress);
// connect sockets to peers
for (var i = 1; i < args.Length; i++)
{
// build the cloud back end address
var peerPort = int.Parse(args[i]);
var address = baseAddress + (peerPort + 1);
Console.WriteLine("[BROKER] connect to cloud peer {0}", address);
// this cloudBackend connects to all peer cloudFrontends
cloudBackend.Connect(address);
// build the state front end address
address = baseAddress + (peerPort + 3);
Console.WriteLine("[BROKER] subscribe to state peer {0}", address);
// this stateFrontend to all peer stateBackends
stateFrontend.Connect(address);
}
// setup the local worker queue for LRU and monitor cloud capacity
var workerQueue = new Queue <byte[]>();
int previousLocalCapacity = 0;
// receive the capacity available from other peer(s)
stateFrontend.ReceiveReady += (s, e) =>
{
// the message should contain the available cloud capacity
var capacity = e.Socket.ReceiveFrameString();
Debug.Assert(string.IsNullOrWhiteSpace(capacity), "StateFrontend: message was empty!");
int couldCapacity;
Debug.Assert(int.TryParse(capacity, out couldCapacity), "StateFrontend: message did not contain a number!");
};
// get the status message and print it
monitor.ReceiveReady += (s, e) =>
{
var msg = e.Socket.ReceiveFrameString();
Console.WriteLine("[MONITOR] {0}", msg);
};
// all local clients are connecting to this socket
// they send a REQ and get a REPLY
localFrontend.ReceiveReady += (s, e) =>
{
// [client adr][empty][message id]
var request = e.Socket.ReceiveMultipartMessage();
// register the local client for later identification if not known
if (!clients.Any(n => AreSame(n, request[0])))
{
clients.Add(request[0].Buffer);
}
// if we have local capacity send worker else send to cloud
if (workerQueue.Count > 0)
{
// get the LRU worker adr
var worker = workerQueue.Dequeue();
// wrap message with workers address
var msg = Wrap(worker, request);
// send message to the worker
// [worker adr][empty][client adr][empty][data]
localBackend.SendMultipartMessage(msg);
}
else
{
// get an random index for peers
var peerIdx = rnd.Next(peers.Count - 2) + 2;
// get peers address
var peer = peers[peerIdx];
// wrap message with peer's address
var msg = Wrap(peer, request);
// [peer adr][empty][client adr][empty][data]
cloudBackend.SendMultipartMessage(msg);
}
};
// the workers are connected to this socket
// we get a REPLY either for a cloud client [worker adr][empty][peer adr][empty][peer client adr][empty][data]
// or local client [worker adr][empty][client adr][empty][data]
// or a READY message [worker adr][empty][WORKER_READY]
localBackend.ReceiveReady += (s, e) =>
{
// a worker can send "READY" or a request
// or an REPLAY
var msg = e.Socket.ReceiveMultipartMessage();
// just to make sure we received a proper message
Debug.Assert(msg != null && msg.FrameCount > 0, "[LocalBackend] message was empty or frame count == 0!");
// get the workers identity
var id = Unwrap(msg);
// this worker done in either way so add it to available workers
workerQueue.Enqueue(id);
// if it is NOT a ready message we need to route the message
// it could be a reply to a peer or a local client
// [WORKER_READY] or [client adr][empty][data] or [peer adr][empty][peer client adr][empty][data]
if (msg[0].Buffer[0] != WorkerReady)
{
Debug.Assert(msg.FrameCount > 2, "[LocalBackend] None READY message malformed");
// if the adr (first frame) is any of the clients send the REPLY there
// and send it to the peer otherwise
if (clients.Any(n => AreSame(n, msg.First)))
{
localFrontend.SendMultipartMessage(msg);
}
else
{
cloudFrontend.SendMultipartMessage(msg);
}
}
};
// this socket is connected to all peers
// we receive either a REQ or a REPLY form a peer
// REQ [peer adr][empty][peer client adr][empty][message id] -> send to peer for processing
// REP [peer adr][empty][client adr][empty][message id] -> send to local client
cloudBackend.ReceiveReady += (s, e) =>
{
var msg = e.Socket.ReceiveMultipartMessage();
// just to make sure we received a message
Debug.Assert(msg != null && msg.FrameCount > 0, "[CloudBackend] message was empty or frame count == 0!");
// we need the peers address for proper addressing
var peerAdr = Unwrap(msg);
// the remaining message must be at least 3 frames!
Debug.Assert(msg.FrameCount > 2, "[CloudBackend] message malformed");
// if the id is any of the local clients it is a REPLY
// and a REQ otherwise
if (clients.Any(n => AreSame(n, msg.First)))
{
// [client adr][empty][message id]
localFrontend.SendMultipartMessage(msg);
}
else
{
// add the peers address to the request
var request = Wrap(peerAdr, msg);
// [peer adr][empty][peer client adr][empty][message id]
cloudFrontend.SendMultipartMessage(request);
}
};
// all peers are binding to this socket
// we receive REPLY or REQ from peers
// REQ [peer adr][empty][peer client adr][empty][data] -> send to local worker for processing
// REP [peer adr][empty][client adr][empty][data] -> send to local client
cloudFrontend.ReceiveReady += (s, e) =>
{
var msg = e.Socket.ReceiveMultipartMessage();
// just to make sure we received a message
Debug.Assert(msg != null && msg.FrameCount > 0, "[CloudFrontend] message was empty or frame count == 0!");
// we may need the peers address for proper addressing
var peerAdr = Unwrap(msg);
// the remaining message must be at least 3 frames!
Debug.Assert(msg.FrameCount > 2, "[CloudFrontend] message malformed");
// if the address is any of the local clients it is a REPLY
// and a REQ otherwise
if (clients.Any(n => AreSame(n, msg.First)))
{
localFrontend.SendMultipartMessage(msg);
}
else
{
// in order to know which per to send back the peers adr must be added again
var original = Wrap(peerAdr, msg);
// reduce the capacity to reflect the use of a worker by a cloud request
previousLocalCapacity = workerQueue.Count;
// get the LRU worker
var workerAdr = workerQueue.Dequeue();
// wrap the message with the worker address and send
var request = Wrap(workerAdr, original);
localBackend.SendMultipartMessage(request);
}
};
// in order to reduce chatter we only check to see if we have local capacity to provide to cloud
// periodically every 2 seconds with a timer
var timer = new NetMQTimer((int)TimeSpan.FromSeconds(2).TotalMilliseconds);
timer.Elapsed += (t, e) =>
{
// send message only if the previous send information changed
if (previousLocalCapacity != workerQueue.Count)
{
// set the information
previousLocalCapacity = workerQueue.Count;
// generate the message
var msg = new NetMQMessage();
var data = new NetMQFrame(previousLocalCapacity.ToString());
msg.Append(data);
var stateMessage = Wrap(Encoding.UTF8.GetBytes(me), msg);
// publish info
stateBackend.SendMultipartMessage(stateMessage);
}
// restart the timer
e.Timer.Enable = true;
};
// start all clients and workers as threads
var clientTasks = new Thread[NbrClients];
var workerTasks = new Thread[NbrWorker];
for (var i = 0; i < NbrClients; i++)
{
var client = new Client(localFrontendAddress, monitorAddress, (byte)i);
clientTasks[i] = new Thread(client.Run)
{
Name = $"Client_{i}"
};
clientTasks[i].Start();
}
for (var i = 0; i < NbrWorker; i++)
{
var worker = new Worker(localBackendAddress, (byte)i);
workerTasks[i] = new Thread(worker.Run)
{
Name = $"Worker_{i}"
};
workerTasks[i].Start();
}
// create poller and add sockets & timer
var poller = new NetMQPoller
{
localFrontend,
localBackend,
cloudFrontend,
cloudBackend,
stateFrontend,
stateBackend,
monitor,
timer
};
// start monitoring the sockets
poller.RunAsync();
// we wait for a CTRL+C to exit
while (s_keepRunning)
{
Thread.Sleep(100);
}
Console.WriteLine("Ctrl-C encountered! Exiting the program!");
if (poller.IsRunning)
{
poller.Stop();
}
poller.Dispose();
}
}
