private void PollerThread() { while (true) { try { if (poller == null || !poller.IsStarted) { SignalService.Logger.Info("Start NetMQ Poller"); poller = new Poller(); poller.AddSocket(router); poller.Start(); } } catch (Exception e) { SignalService.Logger.Error("NetMQ Poller Thread Exception.\n{0}", e.StackTrace); if (poller != null) { poller.Stop(); poller.Dispose(); } } } }
/// <summary> /// Release any contained resources. /// </summary> /// <param name="disposing">set this to true if releasing managed resources</param> protected virtual void Dispose(bool disposing) { if (!disposing) { return; } if (!m_ownPoller && !m_poller.IsStarted) { DisposeSynced(); return; } // disposing on the scheduler thread var task = new Task(DisposeSynced); task.Start(this); task.Wait(); // poller cannot be stopped from poller thread if (m_ownPoller) { m_poller.CancelAndJoin(); m_poller.Dispose(); } m_isSchedulerThread.Dispose(); }
public void Run() { Console.WriteLine("[CLIENT {0}] Starting", m_id); var rnd = new Random(m_id); // each request shall have an unique id in order to recognize an reply for a request var messageId = new byte[5]; // create clientId for messages var clientId = new[] { m_id }; // a flag to signal that an answer has arrived bool messageAnswered = false; // we use a poller because we have a socket and a timer to monitor var clientPoller = new Poller(); using (var context = NetMQContext.Create()) using (var client = context.CreateRequestSocket()) using (var monitor = context.CreatePushSocket()) { client.Connect(m_localFrontendAddress); monitor.Connect(m_monitorAddress); client.Options.Identity = new[] { m_id }; var timer = new NetMQTimer((int)TimeSpan.FromSeconds(10).TotalMilliseconds); // use as flag to indicate exit var exit = false; // every 10 s check if message has been received, if not then send error message and ext // and restart timer otherwise timer.Elapsed += (s, e) => { if (messageAnswered) { e.Timer.Enable = true; } else { var msg = string.Format("[CLIENT {0}] ERR - EXIT - lost message {1}", m_id, messageId); // send an error message monitor.Send(msg); // if poller is started than stop it if (clientPoller.IsStarted) clientPoller.CancelAndJoin(); // mark the required exit exit = true; } }; // process arriving answers client.ReceiveReady += (s, e) => { // mark the arrival of an answer messageAnswered = true; // worker is supposed to answer with our request id var reply = e.Socket.ReceiveMessage(); if (reply.FrameCount == 0) { // something went wrong monitor.Send(string.Format("[CLIENT {0}] Received an empty message!", m_id)); // mark the exit flag to ensure the exit exit = true; } else { var sb = new StringBuilder(); // create success message foreach (var frame in reply) sb.Append("[" + frame.ConvertToString() + "]"); // send the success message monitor.Send(string.Format("[CLIENT {0}] Received answer {1}", m_id, sb.ToString())); } }; // add socket & timer to poller clientPoller.AddSocket(client); clientPoller.AddTimer(timer); // start poller in extra task in order to allow the continued processing // clientPoller.Start() -> blocking call var pollTask = Task.Factory.StartNew(clientPoller.PollTillCancelled); // if true the message has been answered // the 0th message is always answered messageAnswered = true; while (!exit) { // simulate sporadic activity by randomly delaying Thread.Sleep((int)TimeSpan.FromSeconds(rnd.Next(5)).TotalMilliseconds); // only send next message if the previous one has been replied to if (messageAnswered) { // generate random 5 byte as identity for for the message rnd.NextBytes(messageId); messageAnswered = false; // create message [client adr][empty][message id] and send it var msg = new NetMQMessage(); msg.Push(messageId); msg.Push(NetMQFrame.Empty); msg.Push(clientId); client.SendMessage(msg); } } } // stop poller if needed if (clientPoller.IsStarted) clientPoller.CancelAndJoin(); clientPoller.Dispose(); }
/// <summary> /// the broker setting up the cluster /// /// /// State 2 ---+ +--- State n /// | | /// +----+----+ /// client 1 ---| | |--- worker 1 /// client 2 ---+---- BROKER 1 ----+--- worker 2 /// : | | | : /// client n ---+ +----+----+ +--- worker n /// | | /// BROKER 2 BROKER n /// /// BROKER 2 and n are not included and must be setup separately /// /// A minimum of two address must be supplied /// </summary> /// <param name="args">[0] = this broker's address /// [1] = 1st peer's address /// : /// [n] = nth peer address</param> /// <remarks> /// since "inproc://" is not working in NetMQ we use "tcp://" /// for each broker we need 5 ports which for this example are /// assigned as follows (in true life it should be configurable whether /// they are ports or tcp/ip addresses) /// /// this brokers address => local frontend binds to tcp://127.0.0.1:5555 /// cloud frontend binds to :5556 /// local backend binds to :5557 /// state backend binds to :5558 /// monitor PULL binds to :5559 /// /// the sockets are connected as follows /// /// this broker's monitor PUSH connects to tcp://127.0.0.1:5559 /// /// (if peer's address and port is tcp://127.0.0.1:5575) /// /// this broker's cloud backend connects to :5576 /// this broker's state frontend connects to :5578 /// /// this scheme is fix in this example /// </remarks> public static void Main(string[] args) { Console.Title = "NetMQ Inter-Broker Router"; const string baseAddress = "tcp://127.0.0.1:"; if (args.Length < 2) { Console.WriteLine("usage: program me peer1 [peer]*"); Console.WriteLine("each broker needs 5 port for his sockets!"); Console.WriteLine("place enough distance between multiple broker addresses!"); Environment.Exit(-1); } // trapping Ctrl+C as exit signal! Console.CancelKeyPress += (s, e) => { e.Cancel = true; s_keepRunning = false; }; // get random generator for later use var rnd = new Random(); // get list for registering the clients var clients = new List<byte[]>(NbrClients); // get a list of peer addresses var peers = new List<byte[]>(); // get all peer addresses - first is this broker! for (var i = 1; i < args.Length; i++) peers.Add(Encoding.UTF8.GetBytes(args[i])); // build this broker's address var me = baseAddress + args[0]; // get the port as integer for later use var myPort = int.Parse(args[0]); Console.WriteLine("[BROKER] The broker can be stopped by using CTRL+C!"); Console.WriteLine("[BROKER] setting up sockets ..."); // set up all the addresses needed in the due course var localFrontendAddress = me; var cloudFrontendAddress = baseAddress + (myPort + 1); var localBackendAddress = baseAddress + (myPort + 2); var stateBackendAddress = baseAddress + (myPort + 3); var monitorAddress = baseAddress + (myPort + 4); // create the context and all the sockets using (var context = NetMQContext.Create()) using (var localFrontend = context.CreateRouterSocket()) using (var localBackend = context.CreateRouterSocket()) using (var cloudFrontend = context.CreateRouterSocket()) using (var cloudBackend = context.CreateRouterSocket()) using (var stateBackend = context.CreatePublisherSocket()) using (var stateFrontend = context.CreateSubscriberSocket()) using (var monitor = context.CreatePullSocket()) { // give every socket an unique identity, e.g. LocalFrontend[Port] SetIdentities(myPort, localFrontend, cloudFrontend, localBackend, stateBackend, monitor, cloudBackend, stateFrontend); // subscribe to any message on the stateFrontend socket! stateFrontend.Subscribe(""); // bind the serving sockets localFrontend.Bind(localFrontendAddress); cloudFrontend.Bind(cloudFrontendAddress); localBackend.Bind(localBackendAddress); stateBackend.Bind(stateBackendAddress); monitor.Bind(monitorAddress); // connect sockets to peers for (var i = 1; i < args.Length; i++) { // build the cloud back end address var peerPort = int.Parse(args[i]); var address = baseAddress + (peerPort + 1); Console.WriteLine("[BROKER] connect to cloud peer {0}", address); // this cloudBackend connects to all peer cloudFrontends cloudBackend.Connect(address); // build the state front end address address = baseAddress + (peerPort + 3); Console.WriteLine("[BROKER] subscribe to state peer {0}", address); // this stateFrontend to all peer stateBackends stateFrontend.Connect(address); } // setup the local worker queue for LRU and monitor cloud capacity var workerQueue = new Queue<byte[]>(); int previousLocalCapacity = 0; // receive the capacity available from other peer(s) stateFrontend.ReceiveReady += (s, e) => { // the message should contain the available cloud capacity var capacity = e.Socket.ReceiveFrameString(); Debug.Assert(string.IsNullOrWhiteSpace(capacity), "StateFrontend: message was empty!"); int couldCapacity; Debug.Assert(int.TryParse(capacity, out couldCapacity), "StateFrontend: message did not contain a number!"); }; // get the status message and print it monitor.ReceiveReady += (s, e) => { var msg = e.Socket.ReceiveFrameString(); Console.WriteLine("[MONITOR] {0}", msg); }; // all local clients are connecting to this socket // they send a REQ and get a REPLY localFrontend.ReceiveReady += (s, e) => { // [client adr][empty][message id] var request = e.Socket.ReceiveMultipartMessage(); // register the local client for later identification if not known if (!clients.Any(n => AreSame(n, request[0]))) clients.Add(request[0].Buffer); // if we have local capacity send worker else send to cloud if (workerQueue.Count > 0) { // get the LRU worker adr var worker = workerQueue.Dequeue(); // wrap message with workers address var msg = Wrap(worker, request); // send message to the worker // [worker adr][empty][client adr][empty][data] localBackend.SendMessage(msg); } else { // get an random index for peers var peerIdx = rnd.Next(peers.Count - 2) + 2; // get peers address var peer = peers[peerIdx]; // wrap message with peer's address var msg = Wrap(peer, request); // [peer adr][empty][client adr][empty][data] cloudBackend.SendMessage(msg); } }; // the workers are connected to this socket // we get a REPLY either for a cloud client [worker adr][empty][peer adr][empty][peer client adr][empty][data] // or local client [worker adr][empty][client adr][empty][data] // or a READY message [worker adr][empty][WORKER_READY] localBackend.ReceiveReady += (s, e) => { // a worker can send "READY" or a request // or an REPLAY var msg = e.Socket.ReceiveMultipartMessage(); // just to make sure we received a proper message Debug.Assert(msg != null && msg.FrameCount > 0, "[LocalBackend] message was empty or frame count == 0!"); // get the workers identity var id = Unwrap(msg); // this worker done in either way so add it to available workers workerQueue.Enqueue(id); // if it is NOT a ready message we need to route the message // it could be a reply to a peer or a local client // [WORKER_READY] or [client adr][empty][data] or [peer adr][empty][peer client adr][empty][data] if (msg[0].Buffer[0] != WorkerReady) { Debug.Assert(msg.FrameCount > 2, "[LocalBackend] None READY message malformed"); // if the adr (first frame) is any of the clients send the REPLY there // and send it to the peer otherwise if (clients.Any(n => AreSame(n, msg.First))) localFrontend.SendMessage(msg); else cloudFrontend.SendMessage(msg); } }; // this socket is connected to all peers // we receive either a REQ or a REPLY form a peer // REQ [peer adr][empty][peer client adr][empty][message id] -> send to peer for processing // REP [peer adr][empty][client adr][empty][message id] -> send to local client cloudBackend.ReceiveReady += (s, e) => { var msg = e.Socket.ReceiveMultipartMessage(); // just to make sure we received a message Debug.Assert(msg != null && msg.FrameCount > 0, "[CloudBackend] message was empty or frame count == 0!"); // we need the peers address for proper addressing var peerAdr = Unwrap(msg); // the remaining message must be at least 3 frames! Debug.Assert(msg.FrameCount > 2, "[CloudBackend] message malformed"); // if the id is any of the local clients it is a REPLY // and a REQ otherwise if (clients.Any(n => AreSame(n, msg.First))) { // [client adr][empty][message id] localFrontend.SendMessage(msg); } else { // add the peers address to the request var request = Wrap(peerAdr, msg); // [peer adr][empty][peer client adr][empty][message id] cloudFrontend.SendMessage(request); } }; // all peers are binding to this socket // we receive REPLY or REQ from peers // REQ [peer adr][empty][peer client adr][empty][data] -> send to local worker for processing // REP [peer adr][empty][client adr][empty][data] -> send to local client cloudFrontend.ReceiveReady += (s, e) => { var msg = e.Socket.ReceiveMultipartMessage(); // just to make sure we received a message Debug.Assert(msg != null && msg.FrameCount > 0, "[CloudFrontend] message was empty or frame count == 0!"); // we may need need the peers address for proper addressing var peerAdr = Unwrap(msg); // the remaining message must be at least 3 frames! Debug.Assert(msg.FrameCount > 2, "[CloudFrontend] message malformed"); // if the address is any of the local clients it is a REPLY // and a REQ otherwise if (clients.Any(n => AreSame(n, msg.First))) localFrontend.SendMessage(msg); else { // in order to know which per to send back the peers adr must be added again var original = Wrap(peerAdr, msg); // reduce the capacity to reflect the use of a worker by a cloud request previousLocalCapacity = workerQueue.Count; // get the LRU worker var workerAdr = workerQueue.Dequeue(); // wrap the message with the worker address and send var request = Wrap(workerAdr, original); localBackend.SendMessage(request); } }; // in order to reduce chatter we only check to see if we have local capacity to provide to cloud // periodically every 2 seconds with a timer var timer = new NetMQTimer((int)TimeSpan.FromSeconds(2).TotalMilliseconds); timer.Elapsed += (t, e) => { // send message only if the previous send information changed if (previousLocalCapacity != workerQueue.Count) { // set the information previousLocalCapacity = workerQueue.Count; // generate the message var msg = new NetMQMessage(); var data = new NetMQFrame(previousLocalCapacity.ToString()); msg.Append(data); var stateMessage = Wrap(Encoding.UTF8.GetBytes(me), msg); // publish info stateBackend.SendMessage(stateMessage); } // restart the timer e.Timer.Enable = true; }; // start all clients and workers as threads var clientTasks = new Thread[NbrClients]; var workerTasks = new Thread[NbrWorker]; for (var i = 0; i < NbrClients; i++) { var client = new Client(localFrontendAddress, monitorAddress, (byte)i); clientTasks[i] = new Thread(client.Run) { Name = string.Format("Client_{0}", i) }; clientTasks[i].Start(); } for (var i = 0; i < NbrWorker; i++) { var worker = new Worker(localBackendAddress, (byte)i); workerTasks[i] = new Thread(worker.Run) { Name = string.Format("Worker_{0}", i) }; workerTasks[i].Start(); } var sockets = new NetMQSocket[] { localFrontend, localBackend, cloudFrontend, cloudBackend, stateFrontend, stateBackend, monitor }; // create poller and add sockets & timer var poller = new Poller(sockets); poller.AddTimer(timer); // start monitoring the sockets Task.Factory.StartNew(poller.PollTillCancelled); // we wait for a CTRL+C to exit while (s_keepRunning) {} Console.WriteLine("Ctrl-C encountered! Exiting the program!"); if (poller.IsStarted) poller.CancelAndJoin(); poller.Dispose(); } }