public NetMQWorker(int id, string pushAddress, string pullAddress, NetMQContext context) : base(id)
{
_pullSocket = context.CreatePullSocket();
_pushSocket = context.CreatePushSocket();
_pushSocket.Connect(pushAddress);
_pullSocket.Bind(pullAddress+id);
}
public static void Main(string[] args)
{
using (var client = new PullSocket())
{
client.Bind("tcp://127.0.0.1:5488");
while (true)
{
string a = client.ReceiveFrameString(); // To receive a flag from external programs to start the service
if (bool.Parse(a))
{
Console.WriteLine("START");
// Added Begin
CancellationTokenSource source = new CancellationTokenSource();
CancellationToken token = source.Token;
// Added End
Thread t = new Thread(new ParameterizedThreadStart(ThreadProc));
//t.Start(args);
t.Start(source);
//CreateHostBuilder(args).Build().RunAsync(token).Wait(); // This works at the first start
// ATTENTION! CreateHostBuilder(args).Build().Run() will return an error about args.Length < 0
CreateHostBuilder(args).Build().RunAsync(token);
Console.WriteLine("DONE Start");
}
}
}
}
private bool _disposedValue = false; // Для определения избыточных вызовов
#endregion Fields
#region Constructors
public FanBrocker(NetMQContext context, string ventAddress, string sinkAddress, int workersCnt)
{
_logger.Trace("Brocker created");
_ventAddress = ventAddress;
_sinkAddress = sinkAddress;
_sinkSocket = context.CreatePullSocket();
_sinkSocket.Options.ReceiveBuffer = 1;
_sinkSocket.Bind(sinkAddress);
_ventSocket = context.CreatePushSocket();
_ventSocket.Options.SendBuffer = 1;
_ventSocket.Bind(ventAddress);
Task.Run(() =>
{
try
{
while (true)
{
var ba = _sinkSocket.ReceiveFrameString();
_logger.Trace("Brocker received data {0}", ba);
var data = JsonConvert.DeserializeObject<ProcessedEventArgs>(ba);
OnFramesProcessed(data);
}
}
catch (Exception)
{
_logger.Error("EXCEPTION");
}
});
}
public void Start()
{
if (context == null)
{
context = NetMQContext.Create();
publisher = context.CreatePublisherSocket();
publisher.Bind("tcp://*:5252");
receiver = context.CreatePullSocket();
receiver.Bind("tcp://*:5253");
// Start listening..
Task.Run(() =>
{
try
{
while (!tokenSource.IsCancellationRequested)
{
var message = receiver.ReceiveString();
var command = Serialization.DeserializeEnvelope <ContractMarker>(message);
commandReceivedSubject.OnNext(command);
}
}
catch (Exception ex)
{
Tracer.Error(ex);
throw;
}
});
}
}
public Scheduler ()
{
Context = NetMQContext.Create ();
ReceiverSocket = Context.CreatePullSocket ();
ReceiverSocket.Bind ("tcp://0.0.0.0:10000");
new Thread (this.catchAck).Start ();
logger.Info ("Scheduler started");
}
protected override Task ExecuteAsync(CancellationToken stoppingToken)
{
xpubSocket.Bind("tcp://127.0.0.1:5556");
xsubSocket.Bind("tcp://127.0.0.1:5557");
_logger.LogInformation("MQBusService started");
var proxy = new Proxy(xsubSocket, xpubSocket);
return(Task.Run(proxy.Start));
}
public override bool Start()
{
_pullSocket.Bind(GameServerAddress + "out");
_pushSocket.Connect(GameServerAddress + "in");
_receThread2GameServer = new Thread(ReceiveGameServer);
_cancelToken = new CancellationTokenSource();
_receThread2GameServer.Start();
return(base.Start());
}
public void Start(int listenPort)
{
ctx = NetMQContext.Create();
responseSocket = ctx.CreatePullSocket();
responseSocket.Bind("tcp://*:" + listenPort);
isRunning = true;
listenThread = new Thread(ListenForMessages);
listenThread.Name = "ClientMessageListenThread";
listenThread.Start();
logger.Info("Client message puller has started listening on port " + listenPort);
}
public void ServerAsync()
{
using var server = new PullSocket();
var url = ConnectionUrl.Replace("127.0.0.1", "*");
server.Bind($"tcp://{url}");
while (server.TryReceiveFrameString(timeout: TimeSpan.FromMinutes(5), out string responseMessage))
{
_networkAdapter.Receive(responseMessage);
// server.TrySignalOK();
}
}
public void Start(int listenPort)
{
ctx = NetMQContext.Create();
responseSocket = ctx.CreatePullSocket();
responseSocket.Bind("tcp://*:" + listenPort);
isRunning = true;
listenThread = new Thread(ListenForMessages);
listenThread.Name = "ClientMessageListenThread";
listenThread.Start();
logger.Info("Client message puller has started listening on port " + listenPort);
}
private void FanInEntry(object n)
{
FansInRun f;
string name = (string)n;
if (FunInRM.ContainsKey(name))
{
f = FunInRM[name];
}
else
{
Logging.logger.Error("FunInEntry get name failed " + name);
return;
}
PullSocket p = new PullSocket();
string e = "tcp://" + f.point.ip + ":" + f.point.port;
try
{
p.Bind(e);
f.pullsock = p;
}
catch (Exception err)
{
Logging.logger.Error(ModName + " bind funin socket failed " + e + " " + err.Message);
throw (err);
}
string str;
string outdata;
bool result;
f.Working = true;
while (f.Running)
{
str = string.Empty;
result = f.pullsock.TryReceiveFrameString(out str);
outdata = string.Empty;
if (result == true)
{
outdata = Entry4FanInData(name, str);
}
DelayTime();
}
f.Working = false;
p.Close();
return;
}
private static int Main(string[] args)
{
if (args.Length != 3)
{
Console.WriteLine("usage: local_thr <bind-to> <message-size> <message-count>");
return(1);
}
string bindTo = args[0];
int messageSize = int.Parse(args[1]);
int messageCount = int.Parse(args[2]);
using (var pullSocket = new PullSocket())
{
pullSocket.Bind(bindTo);
var msg = new Msg();
msg.InitEmpty();
pullSocket.Receive(ref msg);
var stopWatch = Stopwatch.StartNew();
for (int i = 0; i != messageCount - 1; i++)
{
pullSocket.Receive(ref msg);
if (msg.Size != messageSize)
{
Console.WriteLine("message of incorrect size received. Received: " + msg.Size + " Expected: " + messageSize);
return(-1);
}
}
stopWatch.Stop();
var millisecondsElapsed = stopWatch.ElapsedMilliseconds;
if (millisecondsElapsed == 0)
{
millisecondsElapsed = 1;
}
msg.Close();
double messagesPerSecond = (double)messageCount / millisecondsElapsed * 1000;
double megabits = messagesPerSecond * messageSize * 8 / 1000000;
Console.WriteLine("message size: {0} [B]", messageSize);
Console.WriteLine("message count: {0}", messageCount);
Console.WriteLine("mean throughput: {0:0.000} [msg/s]", messagesPerSecond);
Console.WriteLine("mean throughput: {0:0.000} [Mb/s]", megabits);
}
return(0);
}
private static int Main(string[] args)
{
if (args.Length != 3)
{
Console.WriteLine("usage: local_thr <bind-to> <message-size> <message-count>");
return 1;
}
string bindTo = args[0];
int messageSize = int.Parse(args[1]);
int messageCount = int.Parse(args[2]);
using (var pullSocket = new PullSocket())
{
pullSocket.Bind(bindTo);
var msg = new Msg();
msg.InitEmpty();
pullSocket.Receive(ref msg);
var stopWatch = Stopwatch.StartNew();
for (int i = 0; i != messageCount - 1; i++)
{
pullSocket.Receive(ref msg);
if (msg.Size != messageSize)
{
Console.WriteLine("message of incorrect size received. Received: " + msg.Size + " Expected: " + messageSize);
return -1;
}
}
stopWatch.Stop();
var millisecondsElapsed = stopWatch.ElapsedMilliseconds;
if (millisecondsElapsed == 0)
millisecondsElapsed = 1;
msg.Close();
double messagesPerSecond = (double)messageCount/millisecondsElapsed*1000;
double megabits = messagesPerSecond*messageSize*8/1000000;
Console.WriteLine("message size: {0} [B]", messageSize);
Console.WriteLine("message count: {0}", messageCount);
Console.WriteLine("mean throughput: {0:0.000} [msg/s]", messagesPerSecond);
Console.WriteLine("mean throughput: {0:0.000} [Mb/s]", megabits);
}
return 0;
}
private bool _disposedValue = false; // Для определения избыточных вызовов
#endregion Fields
#region Constructors
public NetMQBrocker(NetMQContext context, string pushAddress, string pullAddress, int workersCnt)
{
_logger.Trace("Brocker created");
_pushAddress = pushAddress;
_pullAddress = pullAddress;
//_pushSocket.Bind(pushAddress);
_pullSocket = context.CreatePullSocket();
_pullSocket.Bind(pullAddress);
_pushSocket = context.CreatePushSocket();
Task.Run(() =>
{
try
{
while (true)
{
var ba = _pullSocket.ReceiveFrameBytes();
if (ba != null)
{
using (var ms = new MemoryStream())
{
ms.Write(ba, 0, ba.Length);
ms.Position = 0;
var data = (ProcessedEventArgs)_formatter.Deserialize(ms);
_logger.Trace("Brocker received result queue {0}", data.QueueId);
OnFramesProcessed(data);
}
}
else
{
_logger.Trace("Brocker not received");
Thread.Sleep(200);
}
}
}
catch (Exception)
{
_logger.Error("EXCEPTION");
}
});
}
void StartServer()
{
Debug.Log("Start Server!");
using (PullSocket pullSocket = new PullSocket())
using (PublisherSocket publisherSocket = new PublisherSocket())
{
pullSocket.Bind("tcp://*:20002");
publisherSocket.Bind("tcp://*:20001");
while (true)
{
string str = pullSocket.ReceiveFrameString();
Debug.Log("[Server] Get! :: " + str);
publisherSocket.SendFrame("Hello Client!");
Debug.Log("[Server] Send Hello Client!");
}
}
}
private static void CreatePullSocket()
{
try
{
var poller = new NetMQPoller();
NetMQMonitor monitor = new NetMQMonitor(pull, "inproc://req.inproc", SocketEvents.All);
monitor.Accepted += Monitor_Accepted;
monitor.EventReceived += Monitor_EventReceived;
pull.ReceiveReady += Pull_ReceiveReady1;
pull.Bind(address);
//monitor.AttachToPoller(poller);
poller.Add(pull);
poller.RunAsync();
monitor.StartAsync();
}
catch (Exception ex)
{
}
}
public void Open()
{
Logger = new NLogLoggingService("roynet");
Logger.Trace("************************");
Logger.Trace("game server is starting");
List <CommandBase> commands = new List <CommandBase>()
{
new LoginCommand(),
new LogoutCommand()
};
//加载模块
OnServerStartup(_modules);
foreach (RoyNetModule module in _modules)
{
module.Configure(commands);
Logger.Trace("load module:{0}", module.Name);
}
//初始化Command
MethodInfo methodSerializer = typeof(ProtoBuf.Serializer).GetMethod("Deserialize");
foreach (CommandBase command in commands)
{
RegisterCommand(methodSerializer, command);
Logger.Trace("load command:{0}", command.Name);
}
_pullSocket.Bind(Address + "in");
_pushSocket.Connect(Address + "out");
_mainThread.Start();
_sendThread.Start();
_receThread.Start();
IsRunning = true;
Logger.Trace("game server start successful");
foreach (RoyNetModule module in _modules)
{
module.Startup();
}
}
public void ContinousReceiveMessages()
{
var pullSocket = new PullSocket();
pullSocket.Bind("tcp://*:" + RemoteServGr.NodePort);
while (RemoteServGr.Running)
{
var msgJson = pullSocket.ReceiveFrameString();
ReceiveMessage(msgJson);
}
var msg = new Message
{
MsgType = MessageType.EndingProgram,
SendingServer = RemoteServGr.NodeAddress
};
var messageWrapper = new RemoteDispatch(msg, null);
RemoteServGr.SendQueue.Add(messageWrapper);
pullSocket.Close();
}
public static void ThreadProc(object obj)
{
//string[] args = ToStringArray(obj);
var source = obj as CancellationTokenSource;
using (var recv = new PullSocket())
{
recv.Bind("tcp://127.0.0.1:5489");
while (true)
{
string temp = recv.ReceiveFrameString(); // To receive a flag from external programs to stop the service
if (!bool.Parse(temp))
{
Console.WriteLine("STOP");
source.Cancel();
source.Dispose();
//CreateHostBuilder(args).Build().StopAsync().Wait();
Console.WriteLine("DONE Stop");
break; // Jump out of the loop as soon as the service is stopped.
}
}
}
}
public void Run()
{
IsRunning = true;
try
{
using (var socket = new PullSocket())
{
socket.Options.Linger = TimeSpan.Zero;
socket.Bind(clientAddress.ZmqAddress + ":" + GlobalConstants.TcpIpPort.Notification);
while (!stopRunning)
{
var notification = socket.ReceiveNetworkMsg(TimeSpan.FromSeconds(1));
if (notification == null)
{
continue;
}
switch (notification.Type)
{
case NetworkMessageType.EventBusNotification:
{
var eventNotification = (EventBusNotification)notification;
if (eventNotification.SessionId == sessionId)
{
NewDomainEventAvailable(eventNotification.NewEvent);
}
break;
}
case NetworkMessageType.PatientAddedNotification:
{
var patientAddedNotification = (PatientAddedNotification)notification;
if (patientAddedNotification.SessionId == sessionId)
{
NewPatientAvailable(patientAddedNotification.Patient);
}
break;
}
case NetworkMessageType.PatientUpdatedNotification:
{
var patientUpdatedNotification = (PatientUpdatedNotification)notification;
if (patientUpdatedNotification.SessionId == sessionId)
{
UpdatedPatientAvailable(patientUpdatedNotification.Patient);
}
break;
}
case NetworkMessageType.TherapyPlaceTypeAddedNotification:
{
var therpyPlaceTypeAddedNotification = (TherapyPlaceTypeAddedNotification)notification;
if (therpyPlaceTypeAddedNotification.SessionId == sessionId)
{
NewTherapyPlaceTypeAvailable(therpyPlaceTypeAddedNotification.TherapyPlaceType);
}
break;
}
case NetworkMessageType.TherapyPlaceTypeUpdatedNotification:
{
var therpyPlaceTypeUpdatedNotification = (TherapyPlaceTypeUpdatedNotification)notification;
if (therpyPlaceTypeUpdatedNotification.SessionId == sessionId)
{
UpdatedTherapyPlaceTypeAvailable(therpyPlaceTypeUpdatedNotification.TherapyPlaceType);
}
break;
}
case NetworkMessageType.LabelAddedNotification:
{
var labelAddedNotification = (LabelAddedNotification)notification;
if (labelAddedNotification.SessionId == sessionId)
{
NewLabelAvailable(labelAddedNotification.Label);
}
break;
}
case NetworkMessageType.LabelUpdatedNotification:
{
var labelUpdatedNotification = (LabelUpdatedNotification)notification;
if (labelUpdatedNotification.SessionId == sessionId)
{
UpdatedLabelAvailable(labelUpdatedNotification.Label);
}
break;
}
default:
throw new ArgumentException();
}
}
}
}
catch
{
// Ignored
}
IsRunning = false;
}
/// <summary>
/// create a socket server to pull message
/// </summary>
/// <param name="port">socket port</param>
public MessagePuller(int port)
{
_puller = new PullSocket();
_puller.Bind($"tcp://*:{port}");
}
public ZeroNotificationDequeue()
{
_context = NetMQContext.Create();
_client = _context.CreatePullSocket();
_client.Bind(QueueAddress);
}
public static void Execute()
{
Console.WriteLine("Executing PushPull test");
_clientData = new byte[DataSize];
_serverData = new byte[DataSize];
var r = new Random();
r.NextBytes(_clientData);
r.NextBytes(_serverData);
var clientThread = new Thread(
() =>
{
var req = new PushSocket();
req.Connect(InprocAddress);
var revreq = new PullSocket();
revreq.Bind(InprocAddressReverse);
byte[] streamOutput = new byte[BufferSize];
while (true)
{
var sw = Stopwatch.StartNew();
for (int i = 0; i < Iter; i++)
{
var result = req.SendImmediate(_clientData);
Trace.Assert(result);
int read = 0;
using (var stream = revreq.ReceiveStream())
while (stream.Length != stream.Position)
{
read += stream.Read(streamOutput, 0, streamOutput.Length);
}
Trace.Assert(read == _serverData.Length);
}
sw.Stop();
var secondsPerSend = sw.Elapsed.TotalSeconds / (double)Iter;
Console.WriteLine("PushPull Time {0} us, {1} per second, {2} mb/s ",
(int)(secondsPerSend * 1000d * 1000d),
(int)(1d / secondsPerSend),
(int)(DataSize * 2d / (1024d * 1024d * secondsPerSend)));
}
});
clientThread.Start();
{
var rep = new PullSocket();
rep.Bind(InprocAddress);
var revrep = new PushSocket();
revrep.Connect(InprocAddressReverse);
byte[] streamOutput = new byte[BufferSize];
var sw = Stopwatch.StartNew();
while (sw.Elapsed.TotalSeconds < 10)
{
int read = 0;
using (var stream = rep.ReceiveStream())
while (stream.Length != stream.Position)
{
read += stream.Read(streamOutput, 0, streamOutput.Length);
}
revrep.SendImmediate(_serverData);
}
clientThread.Abort();
}
}
private void FanInPubEntry(object n)
{
FansInPubRun f;
string name = (string)n;
if (FunInPubRM.ContainsKey(name))
{
f = FunInPubRM[name];
}
else
{
Logging.logger.Error("FunInPubEntry get name failed " + name);
return;
}
PullSocket pull = new PullSocket();
PublisherSocket pub = new PublisherSocket();
string le = "tcp://" + f.PullPoint.ip + ":" + f.PullPoint.port;
string pe = "tcp://*:" + f.PubPoint.port;
try
{
pull.Bind(le);
pub.Bind(pe);
f.pull = pull;
f.pub = pub;
}
catch (Exception err)
{
Logging.logger.Error(ModName + " bind funin socket failed " + le + " " + pe + " " + err.Message);
throw (err);
}
string str;
string outdata;
bool result;
f.Working = true;
while (f.Running)
{
str = string.Empty;
result = pull.TryReceiveFrameString(out str);
outdata = string.Empty;
if (result == true)
{
try
{
outdata = Entry4FanInPubData(name, str);
if (outdata != null)
{
pub.SendFrame(outdata);
}
}
catch (Exception err)
{
Logging.logger.Error("exception occur " + name + err.Message);
continue;
}
}
DelayTime();
}
f.Working = false;
pull.Close();
pub.Close();
return;
}
public ZeroNotificationDequeue(ZeroConfiguration configuration)
{
_client = new PullSocket();
_client.Bind(configuration.ZeroMqAddress);
}
/// <summary>
/// the broker setting up the cluster
///
///
/// State 2 ---+ +--- State n
/// | |
/// +----+----+
/// client 1 ---| | |--- worker 1
/// client 2 ---+---- BROKER 1 ----+--- worker 2
/// : | | | :
/// client n ---+ +----+----+ +--- worker n
/// | |
/// BROKER 2 BROKER n
///
/// BROKER 2 and n are not included and must be setup separately
///
/// A minimum of two address must be supplied
/// </summary>
/// <param name="args">[0] = this broker's address
/// [1] = 1st peer's address
/// :
/// [n] = nth peer address</param>
/// <remarks>
/// since "inproc://" is not working in NetMQ we use "tcp://"
/// for each broker we need 5 ports which for this example are
/// assigned as follows (in true life it should be configurable whether
/// they are ports or tcp/ip addresses)
///
/// this brokers address => local frontend binds to tcp://127.0.0.1:5555
/// cloud frontend binds to :5556
/// local backend binds to :5557
/// state backend binds to :5558
/// monitor PULL binds to :5559
///
/// the sockets are connected as follows
///
/// this broker's monitor PUSH connects to tcp://127.0.0.1:5559
///
/// (if peer's address and port is tcp://127.0.0.1:5575)
///
/// this broker's cloud backend connects to :5576
/// this broker's state frontend connects to :5578
///
/// this scheme is fix in this example
/// </remarks>
public static void Main(string[] args)
{
Console.Title = "NetMQ Inter-Broker Router";
const string baseAddress = "tcp://127.0.0.1:";
if (args.Length < 2)
{
Console.WriteLine("usage: program me peer1 [peer]*");
Console.WriteLine("each broker needs 5 port for his sockets!");
Console.WriteLine("place enough distance between multiple broker addresses!");
Environment.Exit(-1);
}
// trapping Ctrl+C as exit signal!
Console.CancelKeyPress += (s, e) =>
{
e.Cancel = true;
s_keepRunning = false;
};
// get random generator for later use
var rnd = new Random();
// get list for registering the clients
var clients = new List <byte[]>(NbrClients);
// get a list of peer addresses
var peers = new List <byte[]>();
// get all peer addresses - first is this broker!
for (var i = 1; i < args.Length; i++)
{
peers.Add(Encoding.UTF8.GetBytes(args[i]));
}
// build this broker's address
var me = baseAddress + args[0];
// get the port as integer for later use
var myPort = int.Parse(args[0]);
Console.WriteLine("[BROKER] The broker can be stopped by using CTRL+C!");
Console.WriteLine("[BROKER] setting up sockets ...");
// set up all the addresses needed in the due course
var localFrontendAddress = me;
var cloudFrontendAddress = baseAddress + (myPort + 1);
var localBackendAddress = baseAddress + (myPort + 2);
var stateBackendAddress = baseAddress + (myPort + 3);
var monitorAddress = baseAddress + (myPort + 4);
// create the context and all the sockets
using (var localFrontend = new RouterSocket())
using (var localBackend = new RouterSocket())
using (var cloudFrontend = new RouterSocket())
using (var cloudBackend = new RouterSocket())
using (var stateBackend = new PublisherSocket())
using (var stateFrontend = new SubscriberSocket())
using (var monitor = new PullSocket())
{
// give every socket an unique identity, e.g. LocalFrontend[Port]
SetIdentities(myPort,
localFrontend,
cloudFrontend,
localBackend,
stateBackend,
monitor,
cloudBackend, stateFrontend);
// subscribe to any message on the stateFrontend socket!
stateFrontend.Subscribe("");
// bind the serving sockets
localFrontend.Bind(localFrontendAddress);
cloudFrontend.Bind(cloudFrontendAddress);
localBackend.Bind(localBackendAddress);
stateBackend.Bind(stateBackendAddress);
monitor.Bind(monitorAddress);
// connect sockets to peers
for (var i = 1; i < args.Length; i++)
{
// build the cloud back end address
var peerPort = int.Parse(args[i]);
var address = baseAddress + (peerPort + 1);
Console.WriteLine("[BROKER] connect to cloud peer {0}", address);
// this cloudBackend connects to all peer cloudFrontends
cloudBackend.Connect(address);
// build the state front end address
address = baseAddress + (peerPort + 3);
Console.WriteLine("[BROKER] subscribe to state peer {0}", address);
// this stateFrontend to all peer stateBackends
stateFrontend.Connect(address);
}
// setup the local worker queue for LRU and monitor cloud capacity
var workerQueue = new Queue <byte[]>();
int previousLocalCapacity = 0;
// receive the capacity available from other peer(s)
stateFrontend.ReceiveReady += (s, e) =>
{
// the message should contain the available cloud capacity
var capacity = e.Socket.ReceiveFrameString();
Debug.Assert(string.IsNullOrWhiteSpace(capacity), "StateFrontend: message was empty!");
int couldCapacity;
Debug.Assert(int.TryParse(capacity, out couldCapacity), "StateFrontend: message did not contain a number!");
};
// get the status message and print it
monitor.ReceiveReady += (s, e) =>
{
var msg = e.Socket.ReceiveFrameString();
Console.WriteLine("[MONITOR] {0}", msg);
};
// all local clients are connecting to this socket
// they send a REQ and get a REPLY
localFrontend.ReceiveReady += (s, e) =>
{
// [client adr][empty][message id]
var request = e.Socket.ReceiveMultipartMessage();
// register the local client for later identification if not known
if (!clients.Any(n => AreSame(n, request[0])))
{
clients.Add(request[0].Buffer);
}
// if we have local capacity send worker else send to cloud
if (workerQueue.Count > 0)
{
// get the LRU worker adr
var worker = workerQueue.Dequeue();
// wrap message with workers address
var msg = Wrap(worker, request);
// send message to the worker
// [worker adr][empty][client adr][empty][data]
localBackend.SendMultipartMessage(msg);
}
else
{
// get an random index for peers
var peerIdx = rnd.Next(peers.Count - 2) + 2;
// get peers address
var peer = peers[peerIdx];
// wrap message with peer's address
var msg = Wrap(peer, request);
// [peer adr][empty][client adr][empty][data]
cloudBackend.SendMultipartMessage(msg);
}
};
// the workers are connected to this socket
// we get a REPLY either for a cloud client [worker adr][empty][peer adr][empty][peer client adr][empty][data]
// or local client [worker adr][empty][client adr][empty][data]
// or a READY message [worker adr][empty][WORKER_READY]
localBackend.ReceiveReady += (s, e) =>
{
// a worker can send "READY" or a request
// or an REPLAY
var msg = e.Socket.ReceiveMultipartMessage();
// just to make sure we received a proper message
Debug.Assert(msg != null && msg.FrameCount > 0, "[LocalBackend] message was empty or frame count == 0!");
// get the workers identity
var id = Unwrap(msg);
// this worker done in either way so add it to available workers
workerQueue.Enqueue(id);
// if it is NOT a ready message we need to route the message
// it could be a reply to a peer or a local client
// [WORKER_READY] or [client adr][empty][data] or [peer adr][empty][peer client adr][empty][data]
if (msg[0].Buffer[0] != WorkerReady)
{
Debug.Assert(msg.FrameCount > 2, "[LocalBackend] None READY message malformed");
// if the adr (first frame) is any of the clients send the REPLY there
// and send it to the peer otherwise
if (clients.Any(n => AreSame(n, msg.First)))
{
localFrontend.SendMultipartMessage(msg);
}
else
{
cloudFrontend.SendMultipartMessage(msg);
}
}
};
// this socket is connected to all peers
// we receive either a REQ or a REPLY form a peer
// REQ [peer adr][empty][peer client adr][empty][message id] -> send to peer for processing
// REP [peer adr][empty][client adr][empty][message id] -> send to local client
cloudBackend.ReceiveReady += (s, e) =>
{
var msg = e.Socket.ReceiveMultipartMessage();
// just to make sure we received a message
Debug.Assert(msg != null && msg.FrameCount > 0, "[CloudBackend] message was empty or frame count == 0!");
// we need the peers address for proper addressing
var peerAdr = Unwrap(msg);
// the remaining message must be at least 3 frames!
Debug.Assert(msg.FrameCount > 2, "[CloudBackend] message malformed");
// if the id is any of the local clients it is a REPLY
// and a REQ otherwise
if (clients.Any(n => AreSame(n, msg.First)))
{
// [client adr][empty][message id]
localFrontend.SendMultipartMessage(msg);
}
else
{
// add the peers address to the request
var request = Wrap(peerAdr, msg);
// [peer adr][empty][peer client adr][empty][message id]
cloudFrontend.SendMultipartMessage(request);
}
};
// all peers are binding to this socket
// we receive REPLY or REQ from peers
// REQ [peer adr][empty][peer client adr][empty][data] -> send to local worker for processing
// REP [peer adr][empty][client adr][empty][data] -> send to local client
cloudFrontend.ReceiveReady += (s, e) =>
{
var msg = e.Socket.ReceiveMultipartMessage();
// just to make sure we received a message
Debug.Assert(msg != null && msg.FrameCount > 0, "[CloudFrontend] message was empty or frame count == 0!");
// we may need the peers address for proper addressing
var peerAdr = Unwrap(msg);
// the remaining message must be at least 3 frames!
Debug.Assert(msg.FrameCount > 2, "[CloudFrontend] message malformed");
// if the address is any of the local clients it is a REPLY
// and a REQ otherwise
if (clients.Any(n => AreSame(n, msg.First)))
{
localFrontend.SendMultipartMessage(msg);
}
else
{
// in order to know which per to send back the peers adr must be added again
var original = Wrap(peerAdr, msg);
// reduce the capacity to reflect the use of a worker by a cloud request
previousLocalCapacity = workerQueue.Count;
// get the LRU worker
var workerAdr = workerQueue.Dequeue();
// wrap the message with the worker address and send
var request = Wrap(workerAdr, original);
localBackend.SendMultipartMessage(request);
}
};
// in order to reduce chatter we only check to see if we have local capacity to provide to cloud
// periodically every 2 seconds with a timer
var timer = new NetMQTimer((int)TimeSpan.FromSeconds(2).TotalMilliseconds);
timer.Elapsed += (t, e) =>
{
// send message only if the previous send information changed
if (previousLocalCapacity != workerQueue.Count)
{
// set the information
previousLocalCapacity = workerQueue.Count;
// generate the message
var msg = new NetMQMessage();
var data = new NetMQFrame(previousLocalCapacity.ToString());
msg.Append(data);
var stateMessage = Wrap(Encoding.UTF8.GetBytes(me), msg);
// publish info
stateBackend.SendMultipartMessage(stateMessage);
}
// restart the timer
e.Timer.Enable = true;
};
// start all clients and workers as threads
var clientTasks = new Thread[NbrClients];
var workerTasks = new Thread[NbrWorker];
for (var i = 0; i < NbrClients; i++)
{
var client = new Client(localFrontendAddress, monitorAddress, (byte)i);
clientTasks[i] = new Thread(client.Run)
{
Name = $"Client_{i}"
};
clientTasks[i].Start();
}
for (var i = 0; i < NbrWorker; i++)
{
var worker = new Worker(localBackendAddress, (byte)i);
workerTasks[i] = new Thread(worker.Run)
{
Name = $"Worker_{i}"
};
workerTasks[i].Start();
}
// create poller and add sockets & timer
var poller = new NetMQPoller
{
localFrontend,
localBackend,
cloudFrontend,
cloudBackend,
stateFrontend,
stateBackend,
monitor,
timer
};
// start monitoring the sockets
poller.RunAsync();
// we wait for a CTRL+C to exit
while (s_keepRunning)
{
Thread.Sleep(100);
}
Console.WriteLine("Ctrl-C encountered! Exiting the program!");
if (poller.IsRunning)
{
poller.Stop();
}
poller.Dispose();
}
}
/// <summary>
/// the broker setting up the cluster
///
///
/// State 2 ---+ +--- State n
/// | |
/// +----+----+
/// client 1 ---| | |--- worker 1
/// client 2 ---+---- BROKER 1 ----+--- worker 2
/// : | | | :
/// client n ---+ +----+----+ +--- worker n
/// | |
/// BROKER 2 BROKER n
///
/// BROKER 2 and n are not included and must be setup separately
///
/// A minimum of two address must be supplied
/// </summary>
/// <param name="args">[0] = this broker's address
/// [1] = 1st peer's address
/// :
/// [n] = nth peer address</param>
/// <remarks>
/// since "inproc://" is not working in NetMQ we use "tcp://"
/// for each broker we need 5 ports which for this example are
/// assigned as follows (in true life it should be configurable whether
/// they are ports or tcp/ip addresses)
///
/// this brokers address => local frontend binds to tcp://127.0.0.1:5555
/// cloud frontend binds to :5556
/// local backend binds to :5557
/// state backend binds to :5558
/// monitor PULL binds to :5559
///
/// the sockets are connected as follows
///
/// this broker's monitor PUSH connects to tcp://127.0.0.1:5559
///
/// (if peer's address and port is tcp://127.0.0.1:5575)
///
/// this broker's cloud backend connects to :5576
/// this broker's state frontend connects to :5578
///
/// this scheme is fix in this example
/// </remarks>
public static void Main(string[] args)
{
Console.Title = "NetMQ Inter-Broker Router";
const string baseAddress = "tcp://127.0.0.1:";
if (args.Length < 2)
{
Console.WriteLine("usage: program me peer1 [peer]*");
Console.WriteLine("each broker needs 5 port for his sockets!");
Console.WriteLine("place enough distance between multiple broker addresses!");
Environment.Exit(-1);
}
// trapping Ctrl+C as exit signal!
Console.CancelKeyPress += (s, e) =>
{
e.Cancel = true;
s_keepRunning = false;
};
// get random generator for later use
var rnd = new Random();
// get list for registering the clients
var clients = new List<byte[]>(NbrClients);
// get a list of peer addresses
var peers = new List<byte[]>();
// get all peer addresses - first is this broker!
for (var i = 1; i < args.Length; i++)
peers.Add(Encoding.UTF8.GetBytes(args[i]));
// build this broker's address
var me = baseAddress + args[0];
// get the port as integer for later use
var myPort = int.Parse(args[0]);
Console.WriteLine("[BROKER] The broker can be stopped by using CTRL+C!");
Console.WriteLine("[BROKER] setting up sockets ...");
// set up all the addresses needed in the due course
var localFrontendAddress = me;
var cloudFrontendAddress = baseAddress + (myPort + 1);
var localBackendAddress = baseAddress + (myPort + 2);
var stateBackendAddress = baseAddress + (myPort + 3);
var monitorAddress = baseAddress + (myPort + 4);
// create the context and all the sockets
using (var localFrontend = new RouterSocket())
using (var localBackend = new RouterSocket())
using (var cloudFrontend = new RouterSocket())
using (var cloudBackend = new RouterSocket())
using (var stateBackend = new PublisherSocket())
using (var stateFrontend = new SubscriberSocket())
using (var monitor = new PullSocket())
{
// give every socket an unique identity, e.g. LocalFrontend[Port]
SetIdentities(myPort,
localFrontend,
cloudFrontend,
localBackend,
stateBackend,
monitor,
cloudBackend, stateFrontend);
// subscribe to any message on the stateFrontend socket!
stateFrontend.Subscribe("");
// bind the serving sockets
localFrontend.Bind(localFrontendAddress);
cloudFrontend.Bind(cloudFrontendAddress);
localBackend.Bind(localBackendAddress);
stateBackend.Bind(stateBackendAddress);
monitor.Bind(monitorAddress);
// connect sockets to peers
for (var i = 1; i < args.Length; i++)
{
// build the cloud back end address
var peerPort = int.Parse(args[i]);
var address = baseAddress + (peerPort + 1);
Console.WriteLine("[BROKER] connect to cloud peer {0}", address);
// this cloudBackend connects to all peer cloudFrontends
cloudBackend.Connect(address);
// build the state front end address
address = baseAddress + (peerPort + 3);
Console.WriteLine("[BROKER] subscribe to state peer {0}", address);
// this stateFrontend to all peer stateBackends
stateFrontend.Connect(address);
}
// setup the local worker queue for LRU and monitor cloud capacity
var workerQueue = new Queue<byte[]>();
int previousLocalCapacity = 0;
// receive the capacity available from other peer(s)
stateFrontend.ReceiveReady += (s, e) =>
{
// the message should contain the available cloud capacity
var capacity = e.Socket.ReceiveFrameString();
Debug.Assert(string.IsNullOrWhiteSpace(capacity), "StateFrontend: message was empty!");
int couldCapacity;
Debug.Assert(int.TryParse(capacity, out couldCapacity), "StateFrontend: message did not contain a number!");
};
// get the status message and print it
monitor.ReceiveReady += (s, e) =>
{
var msg = e.Socket.ReceiveFrameString();
Console.WriteLine("[MONITOR] {0}", msg);
};
// all local clients are connecting to this socket
// they send a REQ and get a REPLY
localFrontend.ReceiveReady += (s, e) =>
{
// [client adr][empty][message id]
var request = e.Socket.ReceiveMultipartMessage();
// register the local client for later identification if not known
if (!clients.Any(n => AreSame(n, request[0])))
clients.Add(request[0].Buffer);
// if we have local capacity send worker else send to cloud
if (workerQueue.Count > 0)
{
// get the LRU worker adr
var worker = workerQueue.Dequeue();
// wrap message with workers address
var msg = Wrap(worker, request);
// send message to the worker
// [worker adr][empty][client adr][empty][data]
localBackend.SendMultipartMessage(msg);
}
else
{
// get an random index for peers
var peerIdx = rnd.Next(peers.Count - 2) + 2;
// get peers address
var peer = peers[peerIdx];
// wrap message with peer's address
var msg = Wrap(peer, request);
// [peer adr][empty][client adr][empty][data]
cloudBackend.SendMultipartMessage(msg);
}
};
// the workers are connected to this socket
// we get a REPLY either for a cloud client [worker adr][empty][peer adr][empty][peer client adr][empty][data]
// or local client [worker adr][empty][client adr][empty][data]
// or a READY message [worker adr][empty][WORKER_READY]
localBackend.ReceiveReady += (s, e) =>
{
// a worker can send "READY" or a request
// or an REPLAY
var msg = e.Socket.ReceiveMultipartMessage();
// just to make sure we received a proper message
Debug.Assert(msg != null && msg.FrameCount > 0, "[LocalBackend] message was empty or frame count == 0!");
// get the workers identity
var id = Unwrap(msg);
// this worker done in either way so add it to available workers
workerQueue.Enqueue(id);
// if it is NOT a ready message we need to route the message
// it could be a reply to a peer or a local client
// [WORKER_READY] or [client adr][empty][data] or [peer adr][empty][peer client adr][empty][data]
if (msg[0].Buffer[0] != WorkerReady)
{
Debug.Assert(msg.FrameCount > 2, "[LocalBackend] None READY message malformed");
// if the adr (first frame) is any of the clients send the REPLY there
// and send it to the peer otherwise
if (clients.Any(n => AreSame(n, msg.First)))
localFrontend.SendMultipartMessage(msg);
else
cloudFrontend.SendMultipartMessage(msg);
}
};
// this socket is connected to all peers
// we receive either a REQ or a REPLY form a peer
// REQ [peer adr][empty][peer client adr][empty][message id] -> send to peer for processing
// REP [peer adr][empty][client adr][empty][message id] -> send to local client
cloudBackend.ReceiveReady += (s, e) =>
{
var msg = e.Socket.ReceiveMultipartMessage();
// just to make sure we received a message
Debug.Assert(msg != null && msg.FrameCount > 0, "[CloudBackend] message was empty or frame count == 0!");
// we need the peers address for proper addressing
var peerAdr = Unwrap(msg);
// the remaining message must be at least 3 frames!
Debug.Assert(msg.FrameCount > 2, "[CloudBackend] message malformed");
// if the id is any of the local clients it is a REPLY
// and a REQ otherwise
if (clients.Any(n => AreSame(n, msg.First)))
{
// [client adr][empty][message id]
localFrontend.SendMultipartMessage(msg);
}
else
{
// add the peers address to the request
var request = Wrap(peerAdr, msg);
// [peer adr][empty][peer client adr][empty][message id]
cloudFrontend.SendMultipartMessage(request);
}
};
// all peers are binding to this socket
// we receive REPLY or REQ from peers
// REQ [peer adr][empty][peer client adr][empty][data] -> send to local worker for processing
// REP [peer adr][empty][client adr][empty][data] -> send to local client
cloudFrontend.ReceiveReady += (s, e) =>
{
var msg = e.Socket.ReceiveMultipartMessage();
// just to make sure we received a message
Debug.Assert(msg != null && msg.FrameCount > 0, "[CloudFrontend] message was empty or frame count == 0!");
// we may need the peers address for proper addressing
var peerAdr = Unwrap(msg);
// the remaining message must be at least 3 frames!
Debug.Assert(msg.FrameCount > 2, "[CloudFrontend] message malformed");
// if the address is any of the local clients it is a REPLY
// and a REQ otherwise
if (clients.Any(n => AreSame(n, msg.First)))
localFrontend.SendMultipartMessage(msg);
else
{
// in order to know which per to send back the peers adr must be added again
var original = Wrap(peerAdr, msg);
// reduce the capacity to reflect the use of a worker by a cloud request
previousLocalCapacity = workerQueue.Count;
// get the LRU worker
var workerAdr = workerQueue.Dequeue();
// wrap the message with the worker address and send
var request = Wrap(workerAdr, original);
localBackend.SendMultipartMessage(request);
}
};
// in order to reduce chatter we only check to see if we have local capacity to provide to cloud
// periodically every 2 seconds with a timer
var timer = new NetMQTimer((int)TimeSpan.FromSeconds(2).TotalMilliseconds);
timer.Elapsed += (t, e) =>
{
// send message only if the previous send information changed
if (previousLocalCapacity != workerQueue.Count)
{
// set the information
previousLocalCapacity = workerQueue.Count;
// generate the message
var msg = new NetMQMessage();
var data = new NetMQFrame(previousLocalCapacity.ToString());
msg.Append(data);
var stateMessage = Wrap(Encoding.UTF8.GetBytes(me), msg);
// publish info
stateBackend.SendMultipartMessage(stateMessage);
}
// restart the timer
e.Timer.Enable = true;
};
// start all clients and workers as threads
var clientTasks = new Thread[NbrClients];
var workerTasks = new Thread[NbrWorker];
for (var i = 0; i < NbrClients; i++)
{
var client = new Client(localFrontendAddress, monitorAddress, (byte)i);
clientTasks[i] = new Thread(client.Run) { Name = string.Format("Client_{0}", i) };
clientTasks[i].Start();
}
for (var i = 0; i < NbrWorker; i++)
{
var worker = new Worker(localBackendAddress, (byte)i);
workerTasks[i] = new Thread(worker.Run) { Name = string.Format("Worker_{0}", i) };
workerTasks[i].Start();
}
// create poller and add sockets & timer
var poller = new NetMQPoller
{
localFrontend,
localBackend,
cloudFrontend,
cloudBackend,
stateFrontend,
stateBackend,
monitor,
timer
};
// start monitoring the sockets
poller.RunAsync();
// we wait for a CTRL+C to exit
while (s_keepRunning)
Thread.Sleep(100);
Console.WriteLine("Ctrl-C encountered! Exiting the program!");
if (poller.IsRunning)
poller.Stop();
poller.Dispose();
}
}